def action_delayed(self, action_name, main_resource):
main_resource.assert_parameter_is_set('user')
if main_resource.resource.security_enabled:
main_resource.kinit()
if main_resource.resource.nameservices is None:
nameservices = namenode_ha_utils.get_nameservices(
main_resource.resource.hdfs_site)
else:
nameservices = main_resource.resource.nameservices
if not nameservices:
self.action_delayed_for_nameservice(None, action_name,
main_resource)
else:
for nameservice in nameservices:
try:
self.action_delayed_for_nameservice(
nameservice, action_name, main_resource)
except namenode_ha_utils.NoActiveNamenodeException as ex:
# one of ns can be down (during initial start forexample) no need to worry for federated cluster
if len(nameservices) > 1:
Logger.exception(
"Cannot run HdfsResource for nameservice {0}. Due to no active namenode present"
.format(nameservice))
else:
raise
def action_delayed(self, action_name, main_resource):
dfs_type = main_resource.resource.dfs_type
if main_resource.resource.nameservices is None: # all nameservices
nameservices = namenode_ha_utils.get_nameservices(
main_resource.resource.hdfs_site)
else:
nameservices = main_resource.resource.nameservices
# non-federated cluster
if not nameservices or len(nameservices) < 2:
self.action_delayed_for_nameservice(None, action_name,
main_resource)
else:
default_fs_protocol = urlparse(
main_resource.resource.default_fs).scheme
if not default_fs_protocol or default_fs_protocol == "viewfs":
protocol = dfs_type.lower()
else:
protocol = default_fs_protocol
for nameservice in nameservices:
try:
nameservice = protocol + "://" + nameservice
self.action_delayed_for_nameservice(
nameservice, action_name, main_resource)
except namenode_ha_utils.NoActiveNamenodeException as ex:
# one of ns can be down (during initial start forexample) no need to worry for federated cluster
if len(nameservices) > 1:
Logger.exception(
"Cannot run HdfsResource for nameservice {0}. Due to no active namenode present"
.format(nameservice))
else:
raise
def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs):
import params
if not is_empty(dir_path):
dir_path = HdfsResourceProvider.parse_path(dir_path)
if dir_path in ignored_dfs_dirs:
Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.")
return
Logger.info("Verifying if DFS directory '" + dir_path + "' exists.")
dir_exists = None
nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site)
nameservice = None if not nameservices else nameservices[-1]
if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type):
# check with webhdfs is much faster than executing hdfs dfs -test
util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled)
list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)
dir_exists = ('FileStatus' in list_status)
else:
# have to do time expensive hdfs dfs -d check.
dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy2_user)[0]
dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists
if not dir_exists:
raise Fail("DFS directory '" + dir_path + "' does not exist !")
else:
Logger.info("DFS directory '" + dir_path + "' exists.")
def action_delayed(self, action_name, main_resource):
dfs_type = main_resource.resource.dfs_type
if main_resource.resource.nameservices is None: # all nameservices
nameservices = namenode_ha_utils.get_nameservices(
main_resource.resource.hdfs_site)
else:
nameservices = main_resource.resource.nameservices
# non-federated cluster
if not nameservices:
self.action_delayed_for_nameservice(None, action_name,
main_resource)
else:
for nameservice in nameservices:
try:
if not dfs_type:
raise Fail(
"<serviceType> for fileSystem service should be set in metainfo.xml"
)
nameservice = dfs_type.lower() + "://" + nameservice
self.action_delayed_for_nameservice(
nameservice, action_name, main_resource)
except namenode_ha_utils.NoActiveNamenodeException as ex:
# one of ns can be down (during initial start forexample) no need to worry for federated cluster
if len(nameservices) > 1:
Logger.exception(
"Cannot run HdfsResource for nameservice {0}. Due to no active namenode present"
.format(nameservice))
else:
raise
def test_get_nameservice(self):
# our cluster is HAA
# dfs.internal.nameservices in hdfs-site
hdfs_site = {
"dfs.internal.nameservices": "HAA",
"dfs.nameservices": "HAA,HAB",
"dfs.ha.namenodes.HAA": "nn1,nn2",
"dfs.ha.namenodes.HAB": "nn1,nn2",
"dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020",
"dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020",
"dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020",
"dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020",
}
self.assertEqual(["HAA"], get_nameservices(hdfs_site))
# dfs.internal.nameservices not in hdfs-site
hdfs_site = {
"dfs.nameservices": "HAA,HAB",
"dfs.ha.namenodes.HAA": "nn1,nn2",
"dfs.ha.namenodes.HAB": "nn1,nn2",
"dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020",
"dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020",
"dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020",
"dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020",
}
self.assertEqual(["HAA"], get_nameservices(hdfs_site))
# Non HA
hdfs_site = {}
self.assertEqual([], get_nameservices(hdfs_site))
# federated config dfs.internal.nameservices in hdfs-site
hdfs_site = {
"dfs.internal.nameservices": "ns1,ns2",
"dfs.nameservices": "ns1,ns2,exns1,exns2"
}
self.assertEqual(["ns1", "ns2"], get_nameservices(hdfs_site))
def action_delayed(self, action_name, main_resource):
main_resource.assert_parameter_is_set('user')
if main_resource.resource.security_enabled:
main_resource.kinit()
nameservices = namenode_ha_utils.get_nameservices(
main_resource.resource.hdfs_site)
if not nameservices:
self.action_delayed_for_nameservice(None, action_name,
main_resource)
else:
for nameservice in nameservices:
self.action_delayed_for_nameservice(nameservice, action_name,
main_resource)
def oozie_service(action='start', upgrade_type=None):
"""
Starts or stops the Oozie service
:param action: 'start' or 'stop'
:param upgrade_type: type of upgrade, either "rolling" or "non_rolling"
skipped since a variation of them was performed during the rolling upgrade
:return:
"""
import params
environment = {'OOZIE_CONFIG': params.conf_dir}
if params.security_enabled:
if params.oozie_principal is None:
oozie_principal_with_host = 'missing_principal'
else:
oozie_principal_with_host = params.oozie_principal.replace(
"_HOST", params.hostname)
kinit_if_needed = format(
"{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host};"
)
else:
kinit_if_needed = ""
no_op_test = as_user(format(
"ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"
),
user=params.oozie_user)
if action == 'start':
start_cmd = format(
"cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-start.sh")
path_to_jdbc = params.target
if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \
params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
params.jdbc_driver_name == "org.postgresql.Driver" or \
params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver":
if not params.jdbc_driver_jar:
path_to_jdbc = format("{oozie_libext_dir}/") + \
params.default_connectors_map[params.jdbc_driver_name] if params.jdbc_driver_name in params.default_connectors_map else None
if not os.path.isfile(path_to_jdbc):
path_to_jdbc = format("{oozie_libext_dir}/") + "*"
error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.jdbc_driver_name] + \
" in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'"
Logger.error(error_message)
db_connection_check_command = format(
"{java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}"
)
else:
db_connection_check_command = None
if upgrade_type is None:
if not os.path.isfile(
path_to_jdbc
) and params.jdbc_driver_name == "org.postgresql.Driver":
print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \
"1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \
"3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \
"{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \
"/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n")
exit(1)
if db_connection_check_command:
sudo.chmod(params.check_db_connection_jar, 0755)
Execute(
db_connection_check_command,
tries=5,
try_sleep=10,
user=params.oozie_user,
)
if params.sysprep_skip_oozie_schema_create:
Logger.info(
"Skipping creation of oozie schema as host is sys prepped")
else:
Execute(format(
"cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run"
),
user=params.oozie_user,
not_if=no_op_test,
ignore_failures=True)
if params.security_enabled:
Execute(
kinit_if_needed,
user=params.oozie_user,
)
nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site)
nameservice = None if not nameservices else nameservices[-1]
if params.sysprep_skip_copy_oozie_share_lib_to_hdfs:
Logger.info(
"Skipping creation of oozie sharelib as host is sys prepped"
)
# Copy current hive-site to hdfs:/user/oozie/share/lib/spark/
params.HdfsResource(
format("{hdfs_share_dir}/lib/spark/hive-site.xml"),
action="create_on_execute",
type='file',
mode=0444,
owner=params.oozie_user,
group=params.user_group,
source=format("{hive_conf_dir}/hive-site.xml"),
)
params.HdfsResource(None, action="execute")
hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check
elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled,
params.dfs_type):
# check with webhdfs is much faster than executing hadoop fs -ls.
util = WebHDFSUtil(params.hdfs_site, nameservice,
params.hdfs_user, params.security_enabled)
list_status = util.run_command(params.hdfs_share_dir,
'GETFILESTATUS',
method='GET',
ignore_status_codes=['404'],
assertable_result=False)
hdfs_share_dir_exists = ('FileStatus' in list_status)
else:
# have to do time expensive hadoop fs -ls check.
hdfs_share_dir_exists = not shell.call(format(
"{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls {hdfs_share_dir} | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'"
),
user=params.oozie_user
)[0]
if not hdfs_share_dir_exists:
Execute(params.put_shared_lib_to_hdfs_cmd,
user=params.oozie_user,
path=params.execute_path)
params.HdfsResource(
format("{oozie_hdfs_user_dir}/share"),
type="directory",
action="create_on_execute",
mode=0755,
recursive_chmod=True,
)
params.HdfsResource(None, action="execute")
try:
# start oozie
Execute(start_cmd,
environment=environment,
user=params.oozie_user,
not_if=no_op_test)
copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type,
params.upgrade_direction)
except:
show_logs(params.oozie_log_dir, params.oozie_user)
raise
elif action == 'stop':
Directory(
params.oozie_tmp_dir,
owner=params.oozie_user,
create_parents=True,
)
stop_cmd = format(
"cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 -force")
try:
# stop oozie
Execute(stop_cmd,
environment=environment,
only_if=no_op_test,
user=params.oozie_user)
except:
show_logs(params.oozie_log_dir, params.oozie_user)
raise
File(params.pid_file, action="delete")
