def configure_atlas_user_for_tagsync(self, env): Logger.info("Configuring Atlas user for Tagsync service.") import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_name = upgrade_stack[0] stack_version = upgrade_stack[1] stack_select.select("ranger-tagsync", stack_version) conf_select.select(stack_name, "ranger-tagsync", stack_version) if params.stack_supports_ranger_tagsync_ssl_xml_support: Logger.info( "Upgrading Tagsync, stack support Atlas user for Tagsync, creating keystore for same." ) self.create_atlas_user_keystore(env) else: Logger.info( "Upgrading Tagsync, stack does not support Atlas user for Tagsync, skipping keystore creation for same." ) Logger.info("Configuring Atlas user for Tagsync service done.")
def upgrade_oozie_database_and_sharelib(self, env): """ Performs the creation and upload of the sharelib and the upgrade of the database. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :return: """ import params env.set_params(params) Logger.info("Will upgrade the Oozie database") # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace( "_HOST", params.hostname) command = format( "{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}" ) Execute(command, user=params.oozie_user, logoutput=True) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # upgrade oozie DB Logger.info( format( 'Upgrading the Oozie database, using version {stack_version}')) # the database upgrade requires the db driver JAR, but since we have # not yet run <stack-selector-tool> to upgrade the current points, we have to use # the versioned libext directory as the location[[-vufdtffr, versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format( stack_version) oozie.download_database_library_if_needed( target_directory=versioned_libext_dir) database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format( stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) # install new sharelib to HDFS self.create_sharelib(env)
def set_pre_start(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_select.select_packages(params.version)
def setup_ranger_kms_database(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] Logger.info(format('Setting Ranger KMS database schema, using version {stack_version}')) setup_kms_db(stack_version=stack_version)
def set_pre_start(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_name = upgrade_stack[0] stack_version = upgrade_stack[1] stack_select.select("ranger-admin", stack_version) conf_select.select(stack_name, "ranger-admin", stack_version)
def setup_ranger_java_patches(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.xml_configurations_supported: Logger.info(format('Applying Ranger java patches, using version {stack_version}')) from setup_ranger_xml import setup_java_patch setup_java_patch(stack_version=stack_version)
def setup_ranger_database(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.xml_configurations_supported: Logger.info(format('Setting Ranger database schema, using version {stack_version}')) from setup_ranger_xml import setup_ranger_db setup_ranger_db(stack_version=stack_version)
def setup_ranger_java_patches(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.upgrade_direction == Direction.UPGRADE: target_version = upgrade_summary.get_target_version( "RANGER", default_version=stack_version) Logger.info( format( 'Applying Ranger java patches, using version {target_version}' )) setup_ranger_xml.setup_java_patch(stack_version=target_version)
def create_sharelib(self, env): """ Performs the creation and upload of the sharelib. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :param env: :return: """ import params env.set_params(params) Logger.info('Creating a new sharelib and uploading it to HDFS...') # ensure the oozie directory exists for the sharelib params.HdfsResource(format("{oozie_hdfs_user_dir}/share"), action="create_on_execute", type="directory", owner="oozie", group="hadoop", mode=0755, recursive_chmod=True) params.HdfsResource(None, action="execute") upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # install new sharelib to HDFS sharelib_command = "/usr/hdp/{0}/oozie/bin/oozie-setup.sh sharelib create -fs {1}".format( stack_version, params.fs_root) Execute(sharelib_command, user=params.oozie_user, logoutput=True)
def setup_tls_toolkit_upgrade(self,env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') if params.upgrade_direction == Direction.UPGRADE and params.nifi_ssl_enabled and params.nifi_ca_host: version_file = params.nifi_config_dir + '/config_version' client_json_file = params.nifi_config_dir+ '/nifi-certificate-authority-client.json' if sudo.path_isfile(version_file): Logger.info(format('Remove config version file if does exist')) sudo.unlink(version_file) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def setup_ranger_database(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.xml_configurations_supported and params.upgrade_direction == Direction.UPGRADE: target_version = upgrade_summary.get_target_version( "RANGER", default_version=stack_version) Logger.info( format( 'Setting Ranger database schema, using version {target_version}' )) from setup_ranger_xml import setup_ranger_db setup_ranger_db(stack_version=target_version)
def setup_tls_toolkit_upgrade(self,env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') if params.upgrade_direction == Direction.UPGRADE and params.nifi_registry_ssl_enabled and params.nifi_ca_host: version_file = params.nifi_registry_config_dir + '/config_version' client_json_file = params.nifi_registry_config_dir+ '/nifi-certificate-authority-client.json' # if not sudo.path_isfile(version_file): # Logger.info(format('Create config version file if it does not exist')) # version_num = params.config['configurationTags']['nifi-registry-ambari-ssl-config']['tag'] # nifi_toolkit_util.save_config_version(version_file,'ssl',version_num,params.nifi_registry_user,params.nifi_registry_group) if sudo.path_isfile(client_json_file): Logger.info(format('Remove client json file')) sudo.unlink(client_json_file)
def upgrade_oozie_database_and_sharelib(): """ Performs the creation and upload of the sharelib and the upgrade of the database. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :return: """ import params # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace( "_HOST", params.hostname) command = format( "{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}" ) Execute(command, user=params.oozie_user) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # upgrade oozie DB Logger.info('Upgrading the Oozie database...') oozie.download_database_library_if_needed() database_upgrade_command = "/usr/iop/{0}/oozie/bin/ooziedb.sh upgrade -run".format( stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) create_sharelib()
stack_version_formatted = status_params.stack_version_formatted major_stack_version = get_major_version(stack_version_formatted) version_for_stack_feature_checks = get_stack_feature_version(config) hadoop_conf_dir = conf_select.get_hadoop_conf_dir() hadoop_bin_dir = stack_select.get_hadoop_dir("bin") hadoop_lib_home = stack_select.get_hadoop_dir("lib") #spark_conf spark_conf_dir = format("{stack_root}/current/spark-client/conf") #hadoop params if stack_version_formatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_formatted): stack_version = None upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is not None and len( upgrade_stack) == 2 and upgrade_stack[1] is not None: stack_version = upgrade_stack[1] # oozie-server or oozie-client, depending on role oozie_root = status_params.component_directory # using the correct oozie root dir, format the correct location oozie_lib_dir = format("{stack_root}/current/{oozie_root}") oozie_setup_sh = format( "{stack_root}/current/{oozie_root}/bin/oozie-setup.sh") oozie_webapps_dir = format( "{stack_root}/current/{oozie_root}/oozie-server/webapps") oozie_webapps_conf_dir = format( "{stack_root}/current/{oozie_root}/oozie-server/conf")
def prepare_libext_directory(): """ Performs the following actions on libext: - creates <stack-root>/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode=0777) # get all hadooplzo* JAR files # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use # <stack-root>/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and ( params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob( os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format( str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format( oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format( stack_version) Logger.info("Copying {0} to {1}".format( versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute( format( '{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}' )) Execute( format( '{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar' ))
def prepare_libext_directory(upgrade_type=None): """ Performs the following actions on libext: - creates <stack-root>/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some stack versions don't need the lzo compression libraries target_version_needs_compression_libraries = params.version and check_stack_feature( StackFeature.LZO, params.version) # ensure the directory exists Directory(params.oozie_libext_dir, mode=0777) # get all hadooplzo* JAR files # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use # <stack-root>/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present. # # This can also be called during a Downgrade. # # When a version is Installed, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. # # This block is just copying around files - there is no assumption about installation # if params.lzo_enabled and ( params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format( "{stack_root}/{version}/hadoop/lib") files = glob.iglob( os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format( str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) # Copy ext ZIP to libext dir # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths( upgrade_type, params) found_at_least_one_oozie_ext_file = False # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): found_at_least_one_oozie_ext_file = True Logger.info("Copying {0} to {1}".format( source_ext_zip_path, params.oozie_libext_dir)) Execute( ("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644) break if not found_at_least_one_oozie_ext_file: raise Fail( "Unable to find any Oozie source extension files from the following paths {0}" .format(source_ext_zip_paths)) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format( params.stack_root, stack_version) Logger.info("Copying {0} to {1}".format( versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute( format( '{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}' )) Execute( format( '{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar' ))