def action_delayed(self, action_name, main_resource): main_resource.assert_parameter_is_set('user') if main_resource.resource.security_enabled: main_resource.kinit() if main_resource.resource.nameservices is None: nameservices = namenode_ha_utils.get_nameservices( main_resource.resource.hdfs_site) else: nameservices = main_resource.resource.nameservices if not nameservices: self.action_delayed_for_nameservice(None, action_name, main_resource) else: for nameservice in nameservices: try: self.action_delayed_for_nameservice( nameservice, action_name, main_resource) except namenode_ha_utils.NoActiveNamenodeException as ex: # one of ns can be down (during initial start forexample) no need to worry for federated cluster if len(nameservices) > 1: Logger.exception( "Cannot run HdfsResource for nameservice {0}. Due to no active namenode present" .format(nameservice)) else: raise
def action_delayed(self, action_name, main_resource): dfs_type = main_resource.resource.dfs_type if main_resource.resource.nameservices is None: # all nameservices nameservices = namenode_ha_utils.get_nameservices( main_resource.resource.hdfs_site) else: nameservices = main_resource.resource.nameservices # non-federated cluster if not nameservices or len(nameservices) < 2: self.action_delayed_for_nameservice(None, action_name, main_resource) else: default_fs_protocol = urlparse( main_resource.resource.default_fs).scheme if not default_fs_protocol or default_fs_protocol == "viewfs": protocol = dfs_type.lower() else: protocol = default_fs_protocol for nameservice in nameservices: try: nameservice = protocol + "://" + nameservice self.action_delayed_for_nameservice( nameservice, action_name, main_resource) except namenode_ha_utils.NoActiveNamenodeException as ex: # one of ns can be down (during initial start forexample) no need to worry for federated cluster if len(nameservices) > 1: Logger.exception( "Cannot run HdfsResource for nameservice {0}. Due to no active namenode present" .format(nameservice)) else: raise
def wait_for_dfs_directory_created(self, dir_path, ignored_dfs_dirs): import params if not is_empty(dir_path): dir_path = HdfsResourceProvider.parse_path(dir_path) if dir_path in ignored_dfs_dirs: Logger.info("Skipping DFS directory '" + dir_path + "' as it's marked to be ignored.") return Logger.info("Verifying if DFS directory '" + dir_path + "' exists.") dir_exists = None nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) nameservice = None if not nameservices else nameservices[-1] if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hdfs dfs -test util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) dir_exists = ('FileStatus' in list_status) else: # have to do time expensive hdfs dfs -d check. dfs_ret_code = shell.call(format("hdfs --config {hadoop_conf_dir} dfs -test -d " + dir_path), user=params.livy2_user)[0] dir_exists = not dfs_ret_code #dfs -test -d returns 0 in case the dir exists if not dir_exists: raise Fail("DFS directory '" + dir_path + "' does not exist !") else: Logger.info("DFS directory '" + dir_path + "' exists.")
def action_delayed(self, action_name, main_resource): dfs_type = main_resource.resource.dfs_type if main_resource.resource.nameservices is None: # all nameservices nameservices = namenode_ha_utils.get_nameservices( main_resource.resource.hdfs_site) else: nameservices = main_resource.resource.nameservices # non-federated cluster if not nameservices: self.action_delayed_for_nameservice(None, action_name, main_resource) else: for nameservice in nameservices: try: if not dfs_type: raise Fail( "<serviceType> for fileSystem service should be set in metainfo.xml" ) nameservice = dfs_type.lower() + "://" + nameservice self.action_delayed_for_nameservice( nameservice, action_name, main_resource) except namenode_ha_utils.NoActiveNamenodeException as ex: # one of ns can be down (during initial start forexample) no need to worry for federated cluster if len(nameservices) > 1: Logger.exception( "Cannot run HdfsResource for nameservice {0}. Due to no active namenode present" .format(nameservice)) else: raise
def test_get_nameservice(self): # our cluster is HAA # dfs.internal.nameservices in hdfs-site hdfs_site = { "dfs.internal.nameservices": "HAA", "dfs.nameservices": "HAA,HAB", "dfs.ha.namenodes.HAA": "nn1,nn2", "dfs.ha.namenodes.HAB": "nn1,nn2", "dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020", "dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020", "dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020", "dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020", } self.assertEqual(["HAA"], get_nameservices(hdfs_site)) # dfs.internal.nameservices not in hdfs-site hdfs_site = { "dfs.nameservices": "HAA,HAB", "dfs.ha.namenodes.HAA": "nn1,nn2", "dfs.ha.namenodes.HAB": "nn1,nn2", "dfs.namenode.rpc-address.HAA.nn1": "hosta1:8020", "dfs.namenode.rpc-address.HAA.nn2": "hosta2:8020", "dfs.namenode.rpc-address.HAB.nn1": "hostb1:8020", "dfs.namenode.rpc-address.HAB.nn2": "hostb2:8020", } self.assertEqual(["HAA"], get_nameservices(hdfs_site)) # Non HA hdfs_site = {} self.assertEqual([], get_nameservices(hdfs_site)) # federated config dfs.internal.nameservices in hdfs-site hdfs_site = { "dfs.internal.nameservices": "ns1,ns2", "dfs.nameservices": "ns1,ns2,exns1,exns2" } self.assertEqual(["ns1", "ns2"], get_nameservices(hdfs_site))
def action_delayed(self, action_name, main_resource): main_resource.assert_parameter_is_set('user') if main_resource.resource.security_enabled: main_resource.kinit() nameservices = namenode_ha_utils.get_nameservices( main_resource.resource.hdfs_site) if not nameservices: self.action_delayed_for_nameservice(None, action_name, main_resource) else: for nameservice in nameservices: self.action_delayed_for_nameservice(nameservice, action_name, main_resource)
def oozie_service(action='start', upgrade_type=None): """ Starts or stops the Oozie service :param action: 'start' or 'stop' :param upgrade_type: type of upgrade, either "rolling" or "non_rolling" skipped since a variation of them was performed during the rolling upgrade :return: """ import params environment = {'OOZIE_CONFIG': params.conf_dir} if params.security_enabled: if params.oozie_principal is None: oozie_principal_with_host = 'missing_principal' else: oozie_principal_with_host = params.oozie_principal.replace( "_HOST", params.hostname) kinit_if_needed = format( "{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host};" ) else: kinit_if_needed = "" no_op_test = as_user(format( "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1" ), user=params.oozie_user) if action == 'start': start_cmd = format( "cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-start.sh") path_to_jdbc = params.target if params.jdbc_driver_name == "com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ params.jdbc_driver_name == "org.postgresql.Driver" or \ params.jdbc_driver_name == "oracle.jdbc.driver.OracleDriver": if not params.jdbc_driver_jar: path_to_jdbc = format("{oozie_libext_dir}/") + \ params.default_connectors_map[params.jdbc_driver_name] if params.jdbc_driver_name in params.default_connectors_map else None if not os.path.isfile(path_to_jdbc): path_to_jdbc = format("{oozie_libext_dir}/") + "*" error_message = "Error! Sorry, but we can't find jdbc driver with default name " + params.default_connectors_map[params.jdbc_driver_name] + \ " in oozie lib dir. So, db connection check can fail. Please run 'ambari-server setup --jdbc-db={db_name} --jdbc-driver={path_to_jdbc} on server host.'" Logger.error(error_message) db_connection_check_command = format( "{java_home}/bin/java -cp {check_db_connection_jar}:{path_to_jdbc} org.apache.ambari.server.DBConnectionVerification '{oozie_jdbc_connection_url}' {oozie_metastore_user_name} {oozie_metastore_user_passwd!p} {jdbc_driver_name}" ) else: db_connection_check_command = None if upgrade_type is None: if not os.path.isfile( path_to_jdbc ) and params.jdbc_driver_name == "org.postgresql.Driver": print format("ERROR: jdbc file {target} is unavailable. Please, follow next steps:\n" \ "1) Download postgresql-9.0-801.jdbc4.jar.\n2) Create needed directory: mkdir -p {oozie_home}/libserver/\n" \ "3) Copy postgresql-9.0-801.jdbc4.jar to newly created dir: cp /path/to/jdbc/postgresql-9.0-801.jdbc4.jar " \ "{oozie_home}/libserver/\n4) Copy postgresql-9.0-801.jdbc4.jar to libext: cp " \ "/path/to/jdbc/postgresql-9.0-801.jdbc4.jar {oozie_home}/libext/\n") exit(1) if db_connection_check_command: sudo.chmod(params.check_db_connection_jar, 0755) Execute( db_connection_check_command, tries=5, try_sleep=10, user=params.oozie_user, ) if params.sysprep_skip_oozie_schema_create: Logger.info( "Skipping creation of oozie schema as host is sys prepped") else: Execute(format( "cd {oozie_tmp_dir} && {oozie_home}/bin/ooziedb.sh create -sqlfile oozie.sql -run" ), user=params.oozie_user, not_if=no_op_test, ignore_failures=True) if params.security_enabled: Execute( kinit_if_needed, user=params.oozie_user, ) nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site) nameservice = None if not nameservices else nameservices[-1] if params.sysprep_skip_copy_oozie_share_lib_to_hdfs: Logger.info( "Skipping creation of oozie sharelib as host is sys prepped" ) # Copy current hive-site to hdfs:/user/oozie/share/lib/spark/ params.HdfsResource( format("{hdfs_share_dir}/lib/spark/hive-site.xml"), action="create_on_execute", type='file', mode=0444, owner=params.oozie_user, group=params.user_group, source=format("{hive_conf_dir}/hive-site.xml"), ) params.HdfsResource(None, action="execute") hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.dfs_type): # check with webhdfs is much faster than executing hadoop fs -ls. util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled) list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False) hdfs_share_dir_exists = ('FileStatus' in list_status) else: # have to do time expensive hadoop fs -ls check. hdfs_share_dir_exists = not shell.call(format( "{kinit_if_needed} hadoop --config {hadoop_conf_dir} dfs -ls {hdfs_share_dir} | awk 'BEGIN {{count=0;}} /share/ {{count++}} END {{if (count > 0) {{exit 0}} else {{exit 1}}}}'" ), user=params.oozie_user )[0] if not hdfs_share_dir_exists: Execute(params.put_shared_lib_to_hdfs_cmd, user=params.oozie_user, path=params.execute_path) params.HdfsResource( format("{oozie_hdfs_user_dir}/share"), type="directory", action="create_on_execute", mode=0755, recursive_chmod=True, ) params.HdfsResource(None, action="execute") try: # start oozie Execute(start_cmd, environment=environment, user=params.oozie_user, not_if=no_op_test) copy_atlas_hive_hook_to_dfs_share_lib(upgrade_type, params.upgrade_direction) except: show_logs(params.oozie_log_dir, params.oozie_user) raise elif action == 'stop': Directory( params.oozie_tmp_dir, owner=params.oozie_user, create_parents=True, ) stop_cmd = format( "cd {oozie_tmp_dir} && {oozie_home}/bin/oozied.sh stop 60 -force") try: # stop oozie Execute(stop_cmd, environment=environment, only_if=no_op_test, user=params.oozie_user) except: show_logs(params.oozie_log_dir, params.oozie_user) raise File(params.pid_file, action="delete")