def test_download_source_get_content_cache_existent_md5_unmatch(self, exists_mock, makedirs_mock, open_mock,
urlopen_mock):
"""
Testing DownloadSource.get_content with cache on cached resource with md5 check
"""
exists_mock.side_effect = [True, True, False]
fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
file_mock = MagicMock(name = 'file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'cached_content'
open_mock.return_value = file_mock
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
urlopen_mock.return_value = web_file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source", cache=True, md5sum=fake_md5)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(open_mock.call_count, 2)
open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
open_mock.assert_once_called('/var/tmp/downloads/source')
self.assertEqual(file_mock.read.call_count, 1)
self.assertEqual(urlopen_mock.call_count, 1)
urlopen_mock.assert_called_with('http://download/source')
def download_database_connector_if_needed():
"""
Downloads the database connector to use when connecting to the metadata storage
"""
import params
if params.metadata_storage_type != 'mysql' or not params.jdbc_driver_jar:
return
File(params.check_db_connection_jar,
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")))
target_jar_with_directory = params.connector_download_dir + os.path.sep + params.jdbc_driver_jar
if not os.path.exists(target_jar_with_directory):
File(params.downloaded_custom_connector,
content=DownloadSource(params.connector_curl_source))
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
target_jar_with_directory),
path=["/bin", "/usr/bin/"],
sudo=True)
File(target_jar_with_directory,
owner=params.druid_user,
group=params.user_group)
def test_download_source_get_content_cache_existent_md5_unmatch(
self, exists_mock, makedirs_mock, open_mock, urlopen_mock):
"""
Testing DownloadSource.get_content with cache on cached resource with md5 check
"""
exists_mock.side_effect = [True, True, False]
fake_md5 = "144c9defac04969c7bfad8efaa8ea194"
file_mock = MagicMock(name='file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'cached_content'
open_mock.return_value = file_mock
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
urlopen_mock.return_value = web_file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source",
cache=True,
md5sum=fake_md5)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(open_mock.call_count, 2)
open_mock.assert_once_called('/var/tmp/downloads/source', 'w')
open_mock.assert_once_called('/var/tmp/downloads/source')
self.assertEqual(file_mock.read.call_count, 1)
self.assertEqual(urlopen_mock.call_count, 1)
urlopen_mock.assert_called_with('http://download/source')
def test_download_source_get_content_cache_new(self, exists_mock,
makedirs_mock, opener_mock,
request_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on non-cached resource
"""
exists_mock.side_effect = [True, False]
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
file_mock = MagicMock(name='file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'content'
open_mock.return_value = file_mock
with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
download_source = DownloadSource("http://download/source",
redownload_files=False)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(opener_mock.call_count, 1)
request_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)
def patch_mysql_driver(self):
import params
Logger.info("Patching mysql driver")
Logger.info("Mysql Jar source :" + params.driver_source)
Logger.info("Mysql Jar target :" + params.mysql_driver_target)
if params.jdbc_jar_name is None:
raise Fail("Mysql JDBC driver not installed on ambari-server")
File(params.mysql_driver_target,
content=DownloadSource(params.driver_source),
mode=0644)
start_script = params.dpprofiler_home + "/bin/profiler-agent"
Logger.info("Updating script : " + start_script)
search_string = "declare -r app_classpath=\""
replace_string = "declare -r app_classpath=\"" + params.mysql_driver_target + ":"
data = open(start_script).read()
if replace_string not in data:
Logger.info("Adding mysql in classpath")
data = data.replace(search_string, replace_string)
f = open(start_script, 'w')
f.write(data)
f.close()
def jdbc_connector():
import params
from urllib2 import HTTPError
from resource_management import Fail
for jar_name in params.sqoop_jdbc_drivers_dict:
if 'mysql-connector-java.jar' in jar_name:
continue
downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
jdbc_symlink_remote = params.sqoop_jdbc_drivers_dict[jar_name]
jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
driver_curl_source = format("{jdk_location}/{jdbc_symlink_remote}")
environment = {
"no_proxy": format("{ambari_server_hostname}")
}
try:
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644,
)
except HTTPError:
error_string = format("Could not download {driver_curl_source}\n\
Please upload jdbc driver to server by run command:\n\
ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
at {ambari_server_hostname}")
raise Fail(error_string)
def setup_jce(self):
import params
if not params.jdk_name:
return
jce_curl_target = None
if params.jce_policy_zip is not None:
jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
Directory(params.artifact_dir,
create_parents = True,
)
File(jce_curl_target,
content = DownloadSource(format("{jce_location}/{jce_policy_zip}")),
)
elif params.security_enabled:
# Something weird is happening
raise Fail("Security is enabled, but JCE policy zip is not specified.")
# The extraction will occur only after the security flag is set
if params.security_enabled:
security_dir = format("{java_home}/jre/lib/security")
File([format("{security_dir}/US_export_policy.jar"), format("{security_dir}/local_policy.jar")],
action = "delete",
)
extract_cmd = ("unzip", "-o", "-j", "-q", jce_curl_target, "-d", security_dir)
Execute(extract_cmd,
only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
path = ['/bin/','/usr/bin'],
sudo = True
)
def download_database_library_if_needed():
"""
Downloads the library to use when connecting to the Oozie database, if
necessary. The library will be downloaded to 'params.target' unless
otherwise specified.
:param target_directory: the location where the database library will be
downloaded to.
:return:
"""
import params
# check to see if the JDBC driver name is in the list of ones that need to
# be downloaded
if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
File(
params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.target, owner=params.oozie_user, group=params.user_group)
def jdbc_connector():
import params
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {
"no_proxy": format("{ambari_server_hostname}")
}
# TODO: should be removed after ranger_hive_plugin will not provide jdbc
Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
path=["/bin", "/usr/bin/"],
sudo = True)
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
)
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo = True)
else:
#for default hive db (Mysql)
Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True
)
File(params.target,
mode = 0644,
)
def download_database_library_if_needed(target_directory = None):
"""
Downloads the library to use when connecting to the Oozie database, if
necessary. The library will be downloaded to 'params.target' unless
otherwise specified.
:param target_directory: the location where the database library will be
downloaded to.
:return:
"""
import params
jdbc_drivers = ["com.mysql.jdbc.Driver",
"com.microsoft.sqlserver.jdbc.SQLServerDriver",
"oracle.jdbc.driver.OracleDriver"]
# check to see if the JDBC driver name is in the list of ones that need to
# be downloaded
if params.jdbc_driver_name not in jdbc_drivers:
return
# if the target directory is not specified
if target_directory is None:
target_jar_with_directory = params.target
else:
# create the full path using the supplied target directory and the JDBC JAR
target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source))
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
path=["/bin", "/usr/bin/"],
sudo = True)
File(target_jar_with_directory, owner = params.oozie_user,
group = params.user_group)
def setup_ranger_admin(rolling_upgrade=False):
import params
ranger_home = params.ranger_home
ranger_conf = params.ranger_conf
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
ranger_conf = format("/usr/hdp/{version}/ranger-admin/conf")
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
)
cp = format("{check_db_connection_jar}")
cp = cp + os.pathsep + format("{driver_curl_target}")
cp = cp + os.pathsep + format("{ranger_home}/ews/webapp/WEB-INF/lib/*")
db_connection_check_command = format(
"{java_home}/bin/java -cp {cp} org.apache.ambari.server.DBConnectionVerification '{ranger_jdbc_connection_url}' {ranger_db_user} {ranger_db_password!p} {ranger_jdbc_driver}")
Execute(db_connection_check_command, path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', tries=5, try_sleep=10)
Execute(('ln','-sf', format('{ranger_home}/ews/webapp/WEB-INF/classes/conf'), format('{ranger_home}/conf')),
not_if=format("ls {ranger_home}/conf"),
only_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes/conf"),
sudo=True)
if rolling_upgrade:
src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/ranger-admin-default-site.xml')
dst_file = format('{ranger_home}/conf/ranger-admin-default-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{ranger_home}/ews/webapp/WEB-INF/classes/conf.dist/security-applicationContext.xml')
dst_file = format('{ranger_home}/conf/security-applicationContext.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
Execute(('chown','-R',format('{unix_user}:{unix_group}'), format('{ranger_home}/')), sudo=True)
Execute(('ln','-sf', format('{ranger_home}/ews/ranger-admin-services.sh'),'/usr/bin/ranger-admin'),
not_if=format("ls /usr/bin/ranger-admin"),
only_if=format("ls {ranger_home}/ews/ranger-admin-services.sh"),
sudo=True)
XmlConfig("ranger-admin-site.xml",
conf_dir=ranger_conf,
configurations=params.config['configurations']['ranger-admin-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-admin-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
Directory(os.path.join(ranger_conf,'ranger_jaas'),
mode=0700,
owner=params.unix_user,
group=params.unix_group,
)
do_keystore_setup(rolling_upgrade=rolling_upgrade)
def jdbc_connector():
import params
from urllib2 import HTTPError
from resource_management import Fail
for jar_name in params.sqoop_jdbc_drivers_dict:
if not jar_name or 'mysql' in jar_name:
continue
downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
custom_connector_to_remove = format(
"{sqoop_lib}/" +
str(params.sqoop_jdbc_drivers_to_remove[jar_name]))
jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
driver_curl_source = format("{jdk_location}/{jar_name}")
environment = {"no_proxy": format("{ambari_server_hostname}")}
try:
if custom_connector_to_remove and os.path.isfile(
custom_connector_to_remove):
File(custom_connector_to_remove, action='delete')
File(
downloaded_custom_connector,
content=DownloadSource(driver_curl_source),
mode=0644,
)
except HTTPError:
error_string = format("Could not download {driver_curl_source}\n\
Please upload jdbc driver to server by run command:\n\
ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
at {ambari_server_hostname}")
raise Fail(error_string)
def copy_jdbc_connector(kms_home):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = "Error! Sorry, but we can't find jdbc driver related to {0} database to download from {1}. \
Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'".format(params.db_flavor, params.jdk_location)
Logger.error(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
driver_curl_target = format("{kms_home}/ews/webapp/lib/{jdbc_jar_name}")
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
Directory(os.path.join(kms_home, 'ews', 'lib'),
mode=0755
)
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', 'sajdbc4.jar'), mode=0644)
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(kms_home, 'ews', 'webapp', 'lib', params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = params.config['configurations']['kms-properties'],
owner = params.kms_user
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{kms_home}/ews/webapp/lib/sajdbc4.jar')},
owner = params.kms_user,
)
else:
ModifyPropertiesFile(format("{kms_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.kms_user,
)
def setup_kms_jce():
import params
if params.jce_name is not None:
Directory(params.jce_source_dir, create_parents=True)
jce_target = format('{jce_source_dir}/{jce_name}')
File(
jce_target,
content=DownloadSource(format('{jdk_location}/{jce_name}')),
mode=0644,
)
File(
[
format("{java_home}/jre/lib/security/local_policy.jar"),
format("{java_home}/jre/lib/security/US_export_policy.jar")
],
action="delete",
)
unzip_cmd = ("unzip", "-o", "-j", "-q", jce_target, "-d",
format("{java_home}/jre/lib/security"))
Execute(
unzip_cmd,
only_if=format(
"test -e {java_home}/jre/lib/security && test -f {jce_target}"
),
path=['/bin/', '/usr/bin'],
sudo=True)
else:
Logger.warning(
"Required jce policy zip is not available, need to setup manually")
def jdbc_connector(target, hive_previous_jdbc_jar):
"""
Shared by Hive Batch, Hive Metastore, and Hive Interactive
:param target: Target of jdbc jar name, which could be for any of the components above.
"""
import params
if not params.jdbc_jar_name:
return
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {"no_proxy": format("{ambari_server_hostname}")}
if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
File(hive_previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
Execute(('cp', '-r', params.downloaded_custom_connector, target),
path=["/bin", "/usr/bin/"],
sudo=True)
else:
# for default hive db (Mysql)
Execute(
('cp', '-r', format('/usr/share/java/{jdbc_jar_name}'), target),
path=["/bin", "/usr/bin/"],
sudo=True)
pass
File(
target,
mode=0644,
)
def jdbc_connector(target, hive_previous_jdbc_jar):
"""
Shared by Hive Batch, Hive Metastore, and Hive Interactive
:param target: Target of jdbc jar name, which could be for any of the components above.
"""
import params
if not params.jdbc_jar_name:
return
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {
"no_proxy": format("{ambari_server_hostname}")
}
if hive_previous_jdbc_jar and os.path.isfile(hive_previous_jdbc_jar):
File(hive_previous_jdbc_jar, action='delete')
# TODO: should be removed after ranger_hive_plugin will not provide jdbc
if params.prepackaged_jdbc_name != params.jdbc_jar_name:
Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
path=["/bin", "/usr/bin/"],
sudo = True)
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source))
# maybe it will be more correcvly to use db type
if params.sqla_db_used:
untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir)
Execute(untar_sqla_type2_driver, sudo = True)
Execute(format("yes | {sudo} cp {jars_path_in_archive} {hive_lib}"))
Directory(params.jdbc_libs_dir,
create_parents = True)
Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))
Execute(format("{sudo} chown -R {hive_user}:{user_group} {hive_lib}/*"))
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target),
#creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo = True)
else:
#for default hive db (Mysql)
Execute(('cp', '--remove-destination', format('/usr/share/java/{jdbc_jar_name}'), target),
#creates=target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True
)
pass
File(target,
mode = 0644,
)
def copy_jdbc_connector(stack_version=None):
import params
if params.jdbc_jar_name is None and params.driver_curl_source.endswith("/None"):
error_message = format("{db_flavor} jdbc driver cannot be downloaded from {jdk_location}\nPlease run 'ambari-server setup --jdbc-db={db_flavor} --jdbc-driver={{path_to_jdbc}}' on ambari-server host.")
raise Fail(error_message)
if params.driver_curl_source and not params.driver_curl_source.endswith("/None"):
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source),
mode = 0644
)
ranger_home = params.ranger_home
if stack_version is not None:
ranger_home = format("{stack_root}/{stack_version}/ranger-admin")
driver_curl_target = format("{ranger_home}/ews/lib/{jdbc_jar_name}")
if params.db_flavor.lower() == 'sqla':
Execute(('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir), sudo = True)
Execute(('cp', '--remove-destination', params.jar_path_in_archive, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib', 'sajdbc4.jar'), mode=0644)
Directory(params.jdbc_libs_dir,
cd_access="a",
create_parents=True)
Execute(as_sudo(['yes', '|', 'cp', params.libs_path_in_archive, params.jdbc_libs_dir], auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, os.path.join(ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(ranger_home, 'ews', 'lib',params.jdbc_jar_name), mode=0644)
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = params.config['configurations']['admin-properties'],
owner = params.unix_user,
)
if params.db_flavor.lower() == 'sqla':
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{ranger_home}/ews/lib/sajdbc4.jar')},
owner = params.unix_user,
)
else:
ModifyPropertiesFile(format("{ranger_home}/install.properties"),
properties = {'SQL_CONNECTOR_JAR': format('{driver_curl_target}')},
owner = params.unix_user,
)
def test_download_source_get_content_nocache(self, exists_mock, request_mock, opener_mock):
"""
Testing DownloadSource.get_content without cache
"""
exists_mock.return_value = True
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
download_source = DownloadSource("http://download/source", redownload_files=True)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(opener_mock.call_count, 1)
request_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)
def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, urlopen_mock):
"""
Testing DownloadSource.get_content with cache on non-cached resource
"""
exists_mock.side_effect = [True, False]
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
urlopen_mock.return_value = web_file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source", cache=True)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(urlopen_mock.call_count, 1)
urlopen_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)
def jdbc_connector():
import params
if params.hive_jdbc_driver in params.hive_jdbc_drivers_list and params.hive_use_existing_db:
environment = {"no_proxy": format("{ambari_server_hostname}")}
# TODO: should be removed after ranger_hive_plugin will not provide jdbc
Execute(('rm', '-f', params.prepackaged_ojdbc_symlink),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
# maybe it will be more correcvly to use db type
if params.sqla_db_used:
untar_sqla_type2_driver = ('tar', '-xvf',
params.downloaded_custom_connector,
'-C', params.tmp_dir)
Execute(untar_sqla_type2_driver, sudo=True)
Execute(as_sudo([
'yes', '|', 'cp', params.jars_path_in_archive, params.hive_lib
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
Directory(params.jdbc_libs_dir, recursive=True)
Execute(as_sudo([
'yes', '|', 'cp', params.libs_path_in_archive,
params.jdbc_libs_dir
],
auto_escape=False),
path=["/bin", "/usr/bin/"])
else:
Execute(
('cp', '--remove-destination',
params.downloaded_custom_connector, params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
else:
#for default hive db (Mysql)
Execute(
('cp', '--remove-destination',
format('/usr/share/java/{jdbc_jar_name}'), params.target),
#creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
path=["/bin", "/usr/bin/"],
sudo=True)
File(
params.target,
mode=0644,
)
def download_mysql_driver():
import params
if params.jdbc_jar_name is None:
raise Fail("Mysql JDBC driver not installed on ambari-server")
File(params.mysql_driver_target,
content=DownloadSource(params.driver_source),
mode=0644)
def downloadjar(cs_lib_path, jdk_location):
# Try to download CredentialUtil.jar from ambari-server resources
credential_util_dir = cs_lib_path.split('*')[0].split(":")[-1] # Remove the trailing '*' and get the last directory if an entire path is passed
credential_util_path = os.path.join(credential_util_dir, credential_util_jar)
credential_util_url = jdk_location + '/' + credential_util_jar
File(credential_util_path,
content = DownloadSource(credential_util_url),
mode = 0755,
)
def hive(name=None):
import params
Directory(params.hive_etc_dir_prefix,
mode=0755
)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
Logger.info("Directories to fill with configs: %s" % str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig("hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
# Generate atlas-application.properties.xml file
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_config_dir, params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.hive_user, params.user_group)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template)
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents = True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2")
)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
mode = 0644,
)
if name != "client":
setup_non_client()
if name == 'hiveserver2':
setup_hiveserver2()
if name == 'metastore':
setup_metastore() # schematool work
def test_download_source_get_content_cache_existent(self, exists_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on cached resource
"""
exists_mock.side_effect = [True, True]
file_mock = MagicMock(name="file_mock")
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = "cached_content"
open_mock.return_value = file_mock
with Environment("/base", tmp_dir="/var/tmp/downloads") as env:
download_source = DownloadSource("http://download/source", redownload_files=False)
content = download_source.get_content()
self.assertEqual("cached_content", content)
self.assertEqual(open_mock.call_count, 1)
self.assertEqual(file_mock.read.call_count, 1)
def test_download_source_get_content_cache_existent(self, exists_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on cached resource
"""
exists_mock.side_effect = [True, True]
file_mock = MagicMock(name = 'file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'cached_content'
open_mock.return_value = file_mock
with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
download_source = DownloadSource("http://download/source", redownload_files=False)
content = download_source.get_content()
self.assertEqual('cached_content', content)
self.assertEqual(open_mock.call_count, 1)
self.assertEqual(file_mock.read.call_count, 1)
def test_download_source_get_content_nocache(self, exists_mock,
makedirs_mock, urlopen_mock):
"""
Testing DownloadSource.get_content without cache
"""
exists_mock.return_value = True
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
urlopen_mock.return_value = web_file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source",
cache=False)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(urlopen_mock.call_count, 1)
urlopen_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
custom_plugin_properties = dict()
custom_plugin_properties['CUSTOM_USER'] = component_user
custom_plugin_properties['CUSTOM_GROUP'] = component_group
ModifyPropertiesFile(file_path,properties = custom_plugin_properties)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def setup_ranger_db(rolling_upgrade=False):
import params
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source))
Directory(params.java_share_dir, mode=0755)
if not os.path.isfile(params.driver_curl_target):
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
not_if=format("test -f {driver_curl_target}"),
sudo=True)
ranger_home = params.ranger_home
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
if not os.path.isfile(
os.path.join(params.ranger_home, 'ews', 'lib',
params.jdbc_jar_name)):
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.ranger_home, 'ews', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
ModifyPropertiesFile(
format("{ranger_home}/install.properties"),
properties=params.config['configurations']['admin-properties'])
# User wants us to setup the DB user and DB?
if params.create_db_dbuser:
Logger.info('Setting up Ranger DB and DB User')
dba_setup = format('python {ranger_home}/dba_script.py -q')
Execute(dba_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
else:
Logger.info(
'Separate DBA property not set. Assuming Ranger DB and DB User exists!'
)
db_setup = format('python {ranger_home}/db_setup.py')
Execute(db_setup,
environment={
'RANGER_ADMIN_HOME': ranger_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
def test_download_source_get_content_cache_existent(self, exists_mock, makedirs_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on cached resource
"""
exists_mock.side_effect = [True, True, False]
file_mock = MagicMock(name = 'file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'cached_content'
open_mock.return_value = file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source", cache=True)
content = download_source.get_content()
self.assertEqual('cached_content', content)
self.assertEqual(open_mock.call_count, 1)
open_mock.assert_called_with('/var/tmp/downloads/source')
self.assertEqual(file_mock.read.call_count, 1)
def download_database_library_if_needed(target_directory = None):
"""
Downloads the library to use when connecting to the Oozie database, if
necessary. The library will be downloaded to 'params.target' unless
otherwise specified.
:param target_directory: the location where the database library will be
downloaded to.
:return:
"""
import params
jdbc_drivers = ["com.mysql.jdbc.Driver",
"com.microsoft.sqlserver.jdbc.SQLServerDriver",
"oracle.jdbc.driver.OracleDriver","sap.jdbc4.sqlanywhere.IDriver"]
# check to see if the JDBC driver name is in the list of ones that need to
# be downloaded
if params.jdbc_driver_name not in jdbc_drivers or not params.jdbc_driver_jar:
return
if params.previous_jdbc_jar and os.path.isfile(params.previous_jdbc_jar):
File(params.previous_jdbc_jar, action='delete')
# if the target directory is not specified
if target_directory is None:
target_jar_with_directory = params.target
else:
# create the full path using the supplied target directory and the JDBC JAR
target_jar_with_directory = target_directory + os.path.sep + params.jdbc_driver_jar
if not os.path.exists(target_jar_with_directory):
File(params.downloaded_custom_connector,
content = DownloadSource(params.driver_curl_source))
if params.sqla_db_used:
untar_sqla_type2_driver = ('tar', '-xvf', params.downloaded_custom_connector, '-C', params.tmp_dir)
Execute(untar_sqla_type2_driver, sudo = True)
Execute(format("yes | {sudo} cp {jars_path_in_archive} {oozie_libext_dir}"))
Directory(params.jdbc_libs_dir,
create_parents = True)
Execute(format("yes | {sudo} cp {libs_path_in_archive} {jdbc_libs_dir}"))
Execute(format("{sudo} chown -R {oozie_user}:{user_group} {oozie_libext_dir}/*"))
else:
Execute(('cp', '--remove-destination', params.downloaded_custom_connector, target_jar_with_directory),
path=["/bin", "/usr/bin/"],
sudo = True)
File(target_jar_with_directory, owner = params.oozie_user,
group = params.user_group)
def test_download_source_get_content_cache_new(self, exists_mock, makedirs_mock, opener_mock, request_mock, open_mock):
"""
Testing DownloadSource.get_content with cache on non-cached resource
"""
exists_mock.side_effect = [True, False]
web_file_mock = MagicMock()
web_file_mock.read.return_value = 'web_content'
opener_mock.return_value.open = MagicMock(return_value=web_file_mock)
file_mock = MagicMock(name = 'file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'content'
open_mock.return_value = file_mock
with Environment("/base", tmp_dir='/var/tmp/downloads') as env:
download_source = DownloadSource("http://download/source", redownload_files=False)
content = download_source.get_content()
self.assertEqual('web_content', content)
self.assertEqual(opener_mock.call_count, 1)
request_mock.assert_called_with('http://download/source')
self.assertEqual(web_file_mock.read.call_count, 1)
def test_download_source_get_content_cache_existent_md5_match(
self, exists_mock, makedirs_mock, open_mock, urlopen_mock):
"""
Testing DownloadSource.get_content with cache on cached resource with md5 check
"""
exists_mock.side_effect = [True, True, False]
file_mock = MagicMock(name='file_mock')
file_mock.__enter__.return_value = file_mock
file_mock.read.return_value = 'cached_content'
open_mock.return_value = file_mock
with Environment("/base") as env:
download_source = DownloadSource("http://download/source",
cache=True)
content = download_source.get_content()
self.assertEqual('cached_content', content)
self.assertEqual(open_mock.call_count, 1)
open_mock.assert_called_with('/var/tmp/downloads/source')
self.assertEqual(file_mock.read.call_count, 1)
self.assertEqual(urlopen_mock.call_count, 0)
def test_download_source_init_existent_download_directory(
self, exists_mock, makedirs_mock):
"""
Testing DownloadSource without cache with existent download directory
"""
exists_mock.return_value = True
with Environment("/base") as env:
static_file = DownloadSource("http://download/source")
self.assertEqual(makedirs_mock.call_count, 0)
self.assertEqual(exists_mock.call_count, 1)
pass
def test_download_source_init_nonexistent_download_directory(
self, exists_mock, makedirs_mock):
"""
Testing DownloadSource without cache with non-existent download directory
"""
exists_mock.return_value = False
with Environment("/base") as env:
static_file = DownloadSource("http://download/source")
self.assertEqual(makedirs_mock.call_count, 1)
makedirs_mock.assert_called_with("/var/tmp/downloads")
self.assertEqual(exists_mock.call_count, 1)
pass
def setup_kms_db():
import params
if params.has_ranger_admin:
File(params.downloaded_custom_connector,
content=DownloadSource(params.driver_curl_source),
mode=0644)
Directory(params.java_share_dir, mode=0755)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
params.driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(params.driver_curl_target, mode=0644)
Directory(os.path.join(params.kms_home, 'ews', 'lib'), mode=0755)
Execute(
('cp', '--remove-destination', params.downloaded_custom_connector,
os.path.join(params.kms_home, 'ews', 'webapp', 'lib')),
path=["/bin", "/usr/bin/"],
sudo=True)
File(os.path.join(params.kms_home, 'ews', 'webapp', 'lib',
params.jdbc_jar_name),
mode=0644)
ModifyPropertiesFile(
format("/usr/hdp/current/ranger-kms/install.properties"),
properties=params.config['configurations']['kms-properties'])
dba_setup = format('python {kms_home}/dba_script.py -q')
db_setup = format('python {kms_home}/db_setup.py')
Execute(dba_setup,
environment={
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
Execute(db_setup,
environment={
'RANGER_KMS_HOME': params.kms_home,
'JAVA_HOME': params.java_home
},
logoutput=True)
