def prepare_libext_directory(): """ Creates /usr/hdp/current/oozie/libext-customer and recursively sets 777 permissions on it and its parents. Also, downloads jdbc driver and provides other staff """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 if not os.path.isdir(params.oozie_libext_customer_dir): os.makedirs(params.oozie_libext_customer_dir, 0o777) # ensure that it's rwx for all os.chmod(params.oozie_libext_customer_dir, 0o777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir)) shutil.copy2(file, params.oozie_libext_customer_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to customer dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir)) shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed()
def upgrade_oozie_database_and_sharelib(self, env): """ Performs the creation and upload of the sharelib and the upgrade of the database. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :return: """ import params env.set_params(params) Logger.info("Will upgrade the Oozie database") # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace( "_HOST", params.hostname) command = format( "{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}" ) Execute(command, user=params.oozie_user, logoutput=True) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # upgrade oozie DB Logger.info( format( 'Upgrading the Oozie database, using version {stack_version}')) # the database upgrade requires the db driver JAR, but since we have # not yet run <stack-selector-tool> to upgrade the current points, we have to use # the versioned libext directory as the location[[-vufdtffr, versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format( stack_version) oozie.download_database_library_if_needed( target_directory=versioned_libext_dir) database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format( stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) # install new sharelib to HDFS self.create_sharelib(env)
def upgrade_oozie_database_and_sharelib(self, env): """ Performs the creation and upload of the sharelib and the upgrade of the database. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :return: """ import params env.set_params(params) Logger.info("Will upgrade the Oozie database") # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}") Execute(command, user=params.oozie_user, logoutput=True) upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") stack_version = upgrade_stack[1] # upgrade oozie DB Logger.info(format('Upgrading the Oozie database, using version {stack_version}')) # the database upgrade requires the db driver JAR, but since we have # not yet run hdp-select to upgrade the current points, we have to use # the versioned libext directory as the location[[-vufdtffr, versioned_libext_dir = "/usr/hdp/{0}/oozie/libext".format(stack_version) oozie.download_database_library_if_needed(target_directory=versioned_libext_dir) database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) # install new sharelib to HDFS self.create_sharelib(env)
def upgrade_oozie_database_and_sharelib(): """ Performs the creation and upload of the sharelib and the upgrade of the database. This method will also perform a kinit if necessary. It is run before the upgrade of oozie begins exactly once as part of the upgrade orchestration. Since this runs before the upgrade has occurred, it should not use any "current" directories since they will still be pointing to the older version of Oozie. Instead, it should use versioned directories to ensure that the commands running are from the oozie version about to be upgraded to. :return: """ import params # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace( "_HOST", params.hostname) command = format( "{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}" ) Execute(command, user=params.oozie_user) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # upgrade oozie DB Logger.info('Upgrading the Oozie database...') oozie.download_database_library_if_needed() database_upgrade_command = "/usr/iop/{0}/oozie/bin/ooziedb.sh upgrade -run".format( stack_version) Execute(database_upgrade_command, user=params.oozie_user, logoutput=True) create_sharelib()
def prepare_libext_directory(): """ Performs the following actions on libext: - creates /usr/hdp/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644 ) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory(): """ Performs the following actions on libext: - creates /usr/hdp/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) sudo.chmod(oozie_ext_zip_target_path, 0644) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory(upgrade_type=None): """ Performs the following actions on libext: - creates <stack-root>/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some stack versions don't need the lzo compression libraries target_version_needs_compression_libraries = params.version and check_stack_feature( StackFeature.LZO, params.version) # ensure the directory exists Directory(params.oozie_libext_dir, mode=0777) # get all hadooplzo* JAR files # <stack-selector-tool> set hadoop-client has not run yet, therefore we cannot use # <stack-root>/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present. # # This can also be called during a Downgrade. # # When a version is Installed, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. # # This block is just copying around files - there is no assumption about installation # if params.lzo_enabled and ( params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format( "{stack_root}/{version}/hadoop/lib") files = glob.iglob( os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format( str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # something like <stack-root>/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, params.ext_js_file) # Copy ext ZIP to libext dir # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = oozie.get_oozie_ext_zip_source_paths( upgrade_type, params) found_at_least_one_oozie_ext_file = False # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): found_at_least_one_oozie_ext_file = True Logger.info("Copying {0} to {1}".format( source_ext_zip_path, params.oozie_libext_dir)) Execute( ("cp", source_ext_zip_path, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644) break if not found_at_least_one_oozie_ext_file: raise Fail( "Unable to find any Oozie source extension files from the following paths {0}" .format(source_ext_zip_paths)) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None or len( upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail( "Unable to determine the stack that is being upgraded to or downgraded to." ) stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "{0}/{1}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format( params.stack_root, stack_version) Logger.info("Copying {0} to {1}".format( versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute( format( '{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}' )) Execute( format( '{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar' ))