def prepare_warfile(): """ Invokes the 'prepare-war' command in Oozie in order to create the WAR. The prepare-war command uses the input WAR from ${OOZIE_HOME}/oozie.war and outputs the prepared WAR to ${CATALINA_BASE}/webapps/oozie.war - because of this, both of these environment variables must point to the upgraded oozie-server path and not oozie-client since it was not yet updated. This method will also perform a kinit if necessary. :return: """ import params # get the kerberos token if necessary to execute commands as oozie if params.security_enabled: oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname) command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}") Execute(command, user=params.oozie_user, logoutput=True) prepare_war(params)
def oozie_server_specific(upgrade_type): import params no_op_test = as_user(format("ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1"), user=params.oozie_user) File(params.pid_file, action="delete", not_if=no_op_test ) oozie_server_directories = [format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir] Directory( oozie_server_directories, owner = params.oozie_user, group = params.user_group, mode = 0755, create_parents = True, cd_access="a", ) Directory(params.oozie_libext_dir, create_parents = True, ) hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar','-xvf',format('{oozie_home}/oozie-sharelib.tar.gz'),'-C',params.oozie_home) Execute( untar_sharelib, # time-expensive not_if = format("{no_op_test} || {skip_recreate_sharelib}"), sudo = True, ) configure_cmds = [] # Default to /usr/share/$TARGETSTACK-oozie/ext-2.2.zip as the first path source_ext_zip_paths = get_oozie_ext_zip_source_paths(upgrade_type, params) # Copy the first oozie ext-2.2.zip file that is found. # This uses a list to handle the cases when migrating from some versions of BigInsights to HDP. if source_ext_zip_paths is not None: for source_ext_zip_path in source_ext_zip_paths: if os.path.isfile(source_ext_zip_path): configure_cmds.append(('cp', source_ext_zip_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute(configure_cmds, not_if=no_op_test, sudo=True, ) break Directory(params.oozie_webapps_conf_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, recursion_follow_links = True, ) # download the database JAR download_database_library_if_needed() #falcon el extension if params.has_falcon_host: Execute(format('{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}'), not_if = no_op_test) Execute(format('{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar'), not_if = no_op_test) if params.lzo_enabled and len(params.all_lzo_packages) > 0: Package(params.all_lzo_packages, retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute(format('{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if = no_op_test, ) prepare_war(params) File(hashcode_file, mode = 0644, ) if params.stack_version_formatted and check_stack_feature(StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents = True, owner = params.oozie_user, group = params.user_group ) if 'hive-site' in params.config['configurations']: hive_site_config = update_credential_provider_path(params.config['configurations']['hive-site'], 'hive-site', os.path.join(params.hive_conf_dir, 'hive-site.jceks'), params.oozie_user, params.user_group ) XmlConfig("hive-site.xml", conf_dir=params.hive_conf_dir, configurations=hive_site_config, configuration_attributes=params.config['configuration_attributes']['hive-site'], owner=params.oozie_user, group=params.user_group, mode=0644 ) if 'tez-site' in params.config['configurations']: XmlConfig( "tez-site.xml", conf_dir = params.hive_conf_dir, configurations = params.config['configurations']['tez-site'], configuration_attributes=params.config['configuration_attributes']['tez-site'], owner = params.oozie_user, group = params.user_group, mode = 0664 ) # If Atlas is also installed, need to generate Atlas Hive hook (hive-atlas-application.properties file) in directory # {stack_root}/{current_version}/atlas/hook/hive/ # Because this is a .properties file instead of an xml file, it will not be read automatically by Oozie. # However, should still save the file on this host so that can upload it to the Oozie Sharelib in DFS. if has_atlas_in_cluster(): atlas_hook_filepath = os.path.join(params.hive_conf_dir, params.atlas_hook_filename) Logger.info("Has atlas in cluster, will save Atlas Hive hook into location %s" % str(atlas_hook_filepath)) setup_atlas_hook(SERVICE.HIVE, params.hive_atlas_application_properties, atlas_hook_filepath, params.oozie_user, params.user_group) Directory(params.oozie_server_dir, owner = params.oozie_user, group = params.user_group, recursive_ownership = True, ) if params.security_enabled: File(os.path.join(params.conf_dir, 'zkmigrator_jaas.conf'), owner=params.oozie_user, group=params.user_group, content=Template("zkmigrator_jaas.conf.j2") )
hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format( "test -f {hashcode_file} && test -d {oozie_home}/share") Execute( configure_cmds, not_if=format("{no_op_test} || {skip_recreate_sharelib}"), sudo=True, ) File( hashcode_file, mode=0644, ) prepare_war(params) oozie_shared_lib = format("/usr/hdp/{stack_version}/oozie/share") oozie_user = '******' oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}") kinit_if_needed = '' create_yarn_service_tarball() if options.upgrade: Logger.info("Skipping uploading oozie shared lib during upgrade") else: params.HdfsResource(format("{oozie_hdfs_user_dir}/share/"), action="delete_on_execute", type='directory')
def oozie_server_specific(): import params no_op_test = as_user(format( "ls {pid_file} >/dev/null 2>&1 && ps -p `cat {pid_file}` >/dev/null 2>&1" ), user=params.oozie_user) File(params.pid_file, action="delete", not_if=no_op_test) oozie_server_directories = [ format("{oozie_home}/{oozie_tmp_dir}"), params.oozie_pid_dir, params.oozie_log_dir, params.oozie_tmp_dir, params.oozie_data_dir, params.oozie_lib_dir, params.oozie_webapps_dir, params.oozie_webapps_conf_dir, params.oozie_server_dir ] Directory( oozie_server_directories, owner=params.oozie_user, group=params.user_group, mode=0755, create_parents=True, cd_access="a", ) Directory( params.oozie_libext_dir, create_parents=True, ) hashcode_file = format("{oozie_home}/.hashcode") skip_recreate_sharelib = format( "test -f {hashcode_file} && test -d {oozie_home}/share") untar_sharelib = ('tar', '-xvf', format('{oozie_home}/oozie-sharelib.tar.gz'), '-C', params.oozie_home) Execute( untar_sharelib, # time-expensive not_if=format("{no_op_test} || {skip_recreate_sharelib}"), sudo=True, ) configure_cmds = [] configure_cmds.append(('cp', params.ext_js_path, params.oozie_libext_dir)) configure_cmds.append(('chown', format('{oozie_user}:{user_group}'), format('{oozie_libext_dir}/{ext_js_file}'))) Execute( configure_cmds, not_if=no_op_test, sudo=True, ) Directory( params.oozie_webapps_conf_dir, owner=params.oozie_user, group=params.user_group, recursive_ownership=True, recursion_follow_links=True, ) # download the database JAR download_database_library_if_needed() #falcon el extension if params.has_falcon_host: Execute(format( '{sudo} cp {falcon_home}/oozie/ext/falcon-oozie-el-extension-*.jar {oozie_libext_dir}' ), not_if=no_op_test) Execute(format( '{sudo} chown {oozie_user}:{user_group} {oozie_libext_dir}/falcon-oozie-el-extension-*.jar' ), not_if=no_op_test) if params.lzo_enabled and len(params.all_lzo_packages) > 0: Package(params.all_lzo_packages, retry_on_repo_unavailability=params. agent_stack_retry_on_unavailability, retry_count=params.agent_stack_retry_count) Execute( format( '{sudo} cp {hadoop_lib_home}/hadoop-lzo*.jar {oozie_lib_dir}'), not_if=no_op_test, ) prepare_war(params) File( hashcode_file, mode=0644, ) if params.stack_version_formatted and check_stack_feature( StackFeature.OOZIE_CREATE_HIVE_TEZ_CONFIGS, params.stack_version_formatted): # Create hive-site and tez-site configs for oozie Directory(params.hive_conf_dir, create_parents=True, owner=params.oozie_user, group=params.user_group) if 'hive-site' in params.config['configurations']: XmlConfig( "hive-site.xml", conf_dir=params.hive_conf_dir, configurations=params.config['configurations']['hive-site'], configuration_attributes=params. config['configuration_attributes']['hive-site'], owner=params.oozie_user, group=params.user_group, mode=0644) if 'tez-site' in params.config['configurations']: XmlConfig( "tez-site.xml", conf_dir=params.hive_conf_dir, configurations=params.config['configurations']['tez-site'], configuration_attributes=params. config['configuration_attributes']['tez-site'], owner=params.oozie_user, group=params.user_group, mode=0664) Directory( params.oozie_server_dir, owner=params.oozie_user, group=params.user_group, recursive_ownership=True, )