def pre_upgrade_restart(self, env, upgrade_type=None): """ Performs the tasks that should be done before an upgrade of oozie. This includes: - backing up configurations - running hdp-select and conf-select - restoring configurations - preparing the libext directory :param env: :return: """ import params env.set_params(params) # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: return Logger.info("Executing Oozie Server Stack Upgrade pre-restart") OozieUpgrade.backup_configuration() if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "oozie", params.version) hdp_select.select("oozie-server", params.version) OozieUpgrade.restore_configuration() OozieUpgrade.prepare_libext_directory()
def configure(self, env, upgrade_type=None): import params # The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary if upgrade_type is None: restart_type = default("/commandParams/restart_type", "") if restart_type.lower() == "rolling_upgrade": upgrade_type = UPGRADE_TYPE_ROLLING elif restart_type.lower() == "nonrolling_upgrade": upgrade_type = UPGRADE_TYPE_NON_ROLLING if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None: Logger.info(format("Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}")) if compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of # oozie, we need to create the symlinks both for server and client. # This is required as both need to be pointing to new installed oozie version. # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-client", params.version) # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-server", params.version) if compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') >= 0: conf_select.select(params.stack_name, "oozie", params.version) env.set_params(params) oozie(is_server=True)
def configure(self, env, upgrade_type=None): import params # The configure command doesn't actually receive the upgrade_type from Script.py, so get it from the config dictionary if upgrade_type is None: restart_type = default("/commandParams/restart_type", "") if restart_type.lower() == "rolling_upgrade": upgrade_type = UPGRADE_TYPE_ROLLING elif restart_type.lower() == "nonrolling_upgrade": upgrade_type = UPGRADE_TYPE_NON_ROLLING if upgrade_type is not None and params.upgrade_direction == Direction.UPGRADE and params.version is not None: Logger.info( format( "Configuring Oozie during upgrade type: {upgrade_type}, direction: {params.upgrade_direction}, and version {params.version}" )) if compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of # oozie, we need to create the symlinks both for server and client. # This is required as both need to be pointing to new installed oozie version. # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-client", params.version) # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-server", params.version) if compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') >= 0: conf_select.select(params.stack_name, "oozie", params.version) env.set_params(params) oozie(is_server=True)
def pre_upgrade_restart(self, env, upgrade_type=None): """ Performs the tasks that should be done before an upgrade of oozie. This includes: - backing up configurations - running hdp-select and conf-select - restoring configurations - preparing the libext directory :param env: :return: """ import params env.set_params(params) # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 if not params.version or compare_versions( format_hdp_stack_version(params.version), '2.2.0.0') < 0: return Logger.info("Executing Oozie Server Stack Upgrade pre-restart") OozieUpgrade.backup_configuration() if params.version and compare_versions( format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "oozie", params.version) hdp_select.select("oozie-server", params.version) OozieUpgrade.restore_configuration() OozieUpgrade.prepare_libext_directory()
def pre_rolling_restart(self, env): """ Performs the tasks surrounding the Oozie startup when a rolling upgrade is in progress. This includes backing up the configuration, updating the database, preparing the WAR, and installing the sharelib in HDFS. :param env: :return: """ import params env.set_params(params) # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: return Logger.info("Executing Oozie Server Rolling Upgrade pre-restart") oozie_server_upgrade.backup_configuration() conf_select.select(params.stack_name, "oozie", params.version) hdp_select.select("oozie-server", params.version) oozie_server_upgrade.restore_configuration() oozie_server_upgrade.prepare_libext_directory() oozie_server_upgrade.upgrade_oozie()
def pre_rolling_restart(self, env): """ Performs the tasks surrounding the Oozie startup when a rolling upgrade is in progress. This includes backing up the configuration, updating the database, preparing the WAR, and installing the sharelib in HDFS. :param env: :return: """ import params env.set_params(params) # this function should not execute if the version can't be determined or # is not at least HDP 2.2.0.0 if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0: return Logger.info("Executing Oozie Server Rolling Upgrade pre-restart") oozie_server_upgrade.backup_configuration() Execute(format("hdp-select set oozie-server {version}")) oozie_server_upgrade.restore_configuration() oozie_server_upgrade.prepare_libext_directory() oozie_server_upgrade.upgrade_oozie()
def prepare_libext_directory(): """ Creates /usr/hdp/current/oozie/libext-customer and recursively sets 777 permissions on it and its parents. :return: """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 if not os.path.isdir(params.oozie_libext_customer_dir): os.makedirs(params.oozie_libext_customer_dir, 0o777) # ensure that it's rwx for all os.chmod(params.oozie_libext_customer_dir, 0o777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob( os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format( str(file), params.oozie_libext_customer_dir)) shutil.copy2(file, params.oozie_libext_customer_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to customer dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format( oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir)) shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
def prepare_libext_directory(): """ Creates /usr/hdp/current/oozie/libext-customer and recursively sets 777 permissions on it and its parents. :return: """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 if not os.path.isdir(params.oozie_libext_customer_dir): os.makedirs(params.oozie_libext_customer_dir, 0o777) # ensure that it's rwx for all os.chmod(params.oozie_libext_customer_dir, 0o777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir)) shutil.copy2(file, params.oozie_libext_customer_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to customer dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir)) shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
def configure(self, env, upgrade_type=None): import params if upgrade_type == "nonrolling" and params.upgrade_direction == Direction.UPGRADE and \ params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0: conf_select.select(params.stack_name, "oozie", params.version) # In order for the "/usr/hdp/current/oozie-<client/server>" point to the new version of # oozie, we need to create the symlinks both for server and client. # This is required as both need to be pointing to new installed oozie version. # Sets the symlink : eg: /usr/hdp/current/oozie-client -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-client", params.version) # Sets the symlink : eg: /usr/hdp/current/oozie-server -> /usr/hdp/2.3.x.y-<version>/oozie hdp_select.select("oozie-server", params.version) env.set_params(params) oozie(is_server=True)
def prepare_libext_directory(): """ Performs the following actions on libext: - creates /usr/hdp/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) File(oozie_ext_zip_target_path, mode=0644 ) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory(): """ Performs the following actions on libext: - creates /usr/hdp/current/oozie/libext and recursively - set 777 permissions on it and its parents. - downloads JDBC driver JAR if needed - copies Falcon JAR for the Oozie WAR if needed """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 # ensure the directory exists Directory(params.oozie_libext_dir, mode = 0777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_dir)) shutil.copy2(file, params.oozie_libext_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to libext dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' # something like /usr/hdp/current/oozie-server/libext/ext-2.2.zip oozie_ext_zip_target_path = os.path.join(params.oozie_libext_dir, "ext-2.2.zip") if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_dir)) Execute(("cp", oozie_ext_zip_file, params.oozie_libext_dir), sudo=True) Execute(("chown", format("{oozie_user}:{user_group}"), oozie_ext_zip_target_path), sudo=True) sudo.chmod(oozie_ext_zip_target_path, 0644) # Redownload jdbc driver to a new current location oozie.download_database_library_if_needed() # get the upgrade version in the event that it's needed upgrade_stack = hdp_select._get_upgrade_stack() if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None: raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.") # something like 2.3.0.0-1234 stack_version = upgrade_stack[1] # copy the Falcon JAR if needed; falcon has not upgraded yet, so we must # use the versioned falcon directory if params.has_falcon_host: versioned_falcon_jar_directory = "/usr/hdp/{0}/falcon/oozie/ext/falcon-oozie-el-extension-*.jar".format(stack_version) Logger.info("Copying {0} to {1}".format(versioned_falcon_jar_directory, params.oozie_libext_dir)) Execute(format('{sudo} cp {versioned_falcon_jar_directory} {oozie_libext_dir}')) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'))
def prepare_libext_directory(): """ Creates /usr/hdp/current/oozie/libext-customer and recursively sets 777 permissions on it and its parents. Also, downloads jdbc driver and provides other staff """ import params # some versions of HDP don't need the lzo compression libraries target_version_needs_compression_libraries = compare_versions( format_hdp_stack_version(params.version), '2.2.1.0') >= 0 if not os.path.isdir(params.oozie_libext_customer_dir): os.makedirs(params.oozie_libext_customer_dir, 0o777) # ensure that it's rwx for all os.chmod(params.oozie_libext_customer_dir, 0o777) # get all hadooplzo* JAR files # hdp-select set hadoop-client has not run yet, therefore we cannot use # /usr/hdp/current/hadoop-client ; we must use params.version directly # however, this only works when upgrading beyond 2.2.0.0; don't do this # for downgrade to 2.2.0.0 since hadoop-lzo will not be present # This can also be called during a Downgrade. # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages # if lzo is enabled. if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries): hadoop_lzo_pattern = 'hadoop-lzo*.jar' hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib") files = glob.iglob( os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern)) if not files: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy files into libext files_copied = False for file in files: if os.path.isfile(file): Logger.info("Copying {0} to {1}".format( str(file), params.oozie_libext_customer_dir)) shutil.copy2(file, params.oozie_libext_customer_dir) files_copied = True if not files_copied: raise Fail("There are no files at {0} matching {1}".format( hadoop_client_new_lib_dir, hadoop_lzo_pattern)) # copy ext ZIP to customer dir oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip' if not os.path.isfile(oozie_ext_zip_file): raise Fail("Unable to copy {0} because it does not exist".format( oozie_ext_zip_file)) Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir)) shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir) # Redownload jdbc driver to a new current location if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \ params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \ params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver": File( params.downloaded_custom_connector, content=DownloadSource(params.driver_curl_source), ) Execute( ('cp', '--remove-destination', params.downloaded_custom_connector, params.target), #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc path=["/bin", "/usr/bin/"], sudo=True) File(params.target, owner=params.oozie_user, group=params.user_group)