def configure_static(env): import params env.set_params(params) generate_logfeeder_input_config('hdfs', Template("input.config-hdfs.json.j2", extra_imports=[default])) hdfs("zkfc_slave") utils.set_up_zkfc_security(params) pass
def format(self, env): import params env.set_params(params) utils.set_up_zkfc_security(params) Execute("hdfs zkfc -formatZK -nonInteractive", returns=[0, 2], # Returns 0 on success ; Returns 2 if zkfc is already formatted user=params.hdfs_user, logoutput=True )
def namenode(action=None, hdfs_binary=None, do_format=True, upgrade_type=None, upgrade_suspended=False, env=None): if action is None: raise Fail('"action" parameter is required for function namenode().') if action in ["start", "stop"] and hdfs_binary is None: raise Fail( '"hdfs_binary" parameter is required for function namenode().') if action == "configure": import params #we need this directory to be present before any action(HA manual steps for #additional namenode) create_name_dirs(params.dfs_name_dir) # set up failover / secure zookeper ACLs, this feature is supported from HDP 2.6 ownwards set_up_zkfc_security(params) elif action == "start": Logger.info("Called service {0} with upgrade_type: {1}".format( action, str(upgrade_type))) setup_ranger_hdfs(upgrade_type=upgrade_type) import params File(params.exclude_file_path, content=Template("exclude_hosts_list.j2"), owner=params.hdfs_user, group=params.user_group) if do_format and not params.hdfs_namenode_format_disabled: format_namenode() pass if params.dfs_ha_enabled and \ params.dfs_ha_namenode_standby is not None and \ params.hostname == params.dfs_ha_namenode_standby: # if the current host is the standby NameNode in an HA deployment # run the bootstrap command, to start the NameNode in standby mode # this requires that the active NameNode is already up and running, # so this execute should be re-tried upon failure, up to a timeout success = bootstrap_standby_namenode(params) if not success: raise Fail("Could not bootstrap standby namenode") if upgrade_type == constants.UPGRADE_TYPE_ROLLING and params.dfs_ha_enabled: # Most likely, ZKFC is up since RU will initiate the failover command. However, if that failed, it would have tried # to kill ZKFC manually, so we need to start it if not already running. safe_zkfc_op(action, env) options = "" if upgrade_type == constants.UPGRADE_TYPE_ROLLING: if params.upgrade_direction == Direction.UPGRADE: options = "-rollingUpgrade started" elif params.upgrade_direction == Direction.DOWNGRADE: options = "-rollingUpgrade downgrade" elif upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: is_previous_image_dir = is_previous_fs_image() Logger.info("Previous file system image dir present is {0}".format( str(is_previous_image_dir))) if params.upgrade_direction == Direction.UPGRADE: options = "-rollingUpgrade started" elif params.upgrade_direction == Direction.DOWNGRADE: options = "-rollingUpgrade downgrade" elif upgrade_type == constants.UPGRADE_TYPE_HOST_ORDERED: # nothing special to do for HOU - should be very close to a normal restart pass elif upgrade_type is None and upgrade_suspended is True: # the rollingUpgrade flag must be passed in during a suspended upgrade when starting NN if os.path.exists( namenode_upgrade.get_upgrade_in_progress_marker()): options = "-rollingUpgrade started" else: Logger.info( "The NameNode upgrade marker file {0} does not exist, yet an upgrade is currently suspended. " "Assuming that the upgrade of NameNode has not occurred yet." .format(namenode_upgrade.get_upgrade_in_progress_marker())) Logger.info("Options for start command are: {0}".format(options)) service(action="start", name="namenode", user=params.hdfs_user, options=options, create_pid_dir=True, create_log_dir=True) if params.security_enabled: Execute(format( "{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_principal_name}" ), user=params.hdfs_user) # ___Scenario___________|_Expected safemode state__|_Wait for safemode OFF____| # no-HA | ON -> OFF | Yes | # HA and active | ON -> OFF | Yes | # HA and standby | no change | No | # RU with HA on active | ON -> OFF | Yes | # RU with HA on standby | ON -> OFF | Yes | # EU with HA on active | ON -> OFF | No | # EU with HA on standby | ON -> OFF | No | # EU non-HA | ON -> OFF | No | # because we do things like create directories after starting NN, # the vast majority of the time this should be True - it should only # be False if this is HA and we are the Standby NN ensure_safemode_off = True # True if this is the only NameNode (non-HA) or if its the Active one in HA is_active_namenode = True if params.dfs_ha_enabled: Logger.info( "Waiting for the NameNode to broadcast whether it is Active or Standby..." ) if is_this_namenode_active() is False: # we are the STANDBY NN is_active_namenode = False # we are the STANDBY NN and this restart is not part of an upgrade if upgrade_type is None: ensure_safemode_off = False # During an Express Upgrade, NameNode will not leave SafeMode until the DataNodes are started, # so always disable the Safemode check if upgrade_type == constants.UPGRADE_TYPE_NON_ROLLING: ensure_safemode_off = False # some informative logging separate from the above logic to keep things a little cleaner if ensure_safemode_off: Logger.info( "Waiting for this NameNode to leave Safemode due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}" .format(params.dfs_ha_enabled, is_active_namenode, upgrade_type)) else: Logger.info( "Skipping Safemode check due to the following conditions: HA: {0}, isActive: {1}, upgradeType: {2}" .format(params.dfs_ha_enabled, is_active_namenode, upgrade_type)) # wait for Safemode to end if ensure_safemode_off: if params.rolling_restart and params.rolling_restart_safemode_exit_timeout: calculated_retries = int( params.rolling_restart_safemode_exit_timeout) / 30 wait_for_safemode_off(hdfs_binary, afterwait_sleep=30, retries=calculated_retries, sleep_seconds=30) else: wait_for_safemode_off(hdfs_binary) # Always run this on the "Active" NN unless Safemode has been ignored # in the case where safemode was ignored (like during an express upgrade), then # NN will be in SafeMode and cannot have directories created if is_active_namenode and ensure_safemode_off: create_hdfs_directories() create_ranger_audit_hdfs_directories() else: Logger.info( "Skipping creation of HDFS directories since this is either not the Active NameNode or we did not wait for Safemode to finish." ) elif action == "stop": import params service(action="stop", name="namenode", user=params.hdfs_user) elif action == "status": import status_params check_process_status(status_params.namenode_pid_file) elif action == "decommission": decommission()
def configure_static(env): import params env.set_params(params) hdfs("zkfc_slave") utils.set_up_zkfc_security(params) pass