def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if params.version and compare_versions( format_stack_version(params.version), '4.1.0.0') >= 0: stack_select.select_packages(params.version) # This is extremely important since it should only be called if crossing the IOP 4.2 boundary. # This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary. if params.version and params.upgrade_direction: src_version = dst_version = None if params.upgrade_direction == Direction.UPGRADE: src_version = upgrade_summary.get_source_version( "KAFKA", default_version=params.version) dst_version = upgrade_summary.get_target_version( "KAFKA", default_version=params.version) else: # These represent the original values during the UPGRADE direction src_version = upgrade_summary.get_target_version( "KAFKA", default_version=params.version) dst_version = upgrade_summary.get_source_version( "KAFKA", default_version=params.version) if compare_versions(src_version, '4.2.0.0') < 0 and compare_versions( dst_version, '4.2.0.0') >= 0: # Upgrade from IOP 4.1 to 4.2, Calling the acl migration script requires the configs to be present. self.configure(env, upgrade_type=upgrade_type) upgrade.run_migration(env, upgrade_type)
def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) # grab the current version of the component pre_upgrade_version = stack_select.get_role_component_current_stack_version( ) if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version): stack_select.select_packages(params.version) # This is extremely important since it should only be called if crossing the HDP 2.3.4.0 boundary. if params.version and params.upgrade_direction: src_version = dst_version = None if params.upgrade_direction == Direction.UPGRADE: src_version = upgrade_summary.get_source_version( "KAFKA", default_version=params.version) dst_version = upgrade_summary.get_target_version( "KAFKA", default_version=params.version) else: # These represent the original values during the UPGRADE direction src_version = upgrade_summary.get_target_version( "KAFKA", default_version=params.version) dst_version = upgrade_summary.get_source_version( "KAFKA", default_version=params.version) if not check_stack_feature( StackFeature.KAFKA_ACL_MIGRATION_SUPPORT, src_version) and check_stack_feature( StackFeature.KAFKA_ACL_MIGRATION_SUPPORT, dst_version): # Calling the acl migration script requires the configs to be present. self.configure(env, upgrade_type=upgrade_type) upgrade.run_migration(env, upgrade_type)
def test_get_stack_feature_version_missing_params(self): """ Tests that simple upgrade information can be extracted from JSON :return: """ command_json = TestUpgradeSummary._get_cluster_simple_upgrade_json() Script.config = command_json summary = upgrade_summary.get_upgrade_summary() self.assertEqual(False, summary.is_revert) self.assertEqual("UPGRADE", summary.direction) self.assertEqual("STANDARD", summary.orchestration) self.assertEqual("rolling_upgrade", summary.type) services = summary.services self.assertEqual("2.4.0.0-1234", services["HDFS"].source_version) self.assertEqual("2.5.9.9-9999", services["HDFS"].target_version) self.assertEqual("2.4.0.0-1234", upgrade_summary.get_source_version("HDFS")) self.assertEqual("2.5.9.9-9999", upgrade_summary.get_target_version("HDFS")) self.assertTrue( upgrade_summary.get_downgrade_from_version("HDFS") is None)
def convert_tables(self, env): import params env.set_params(params) source_version = upgrade_summary.get_source_version( service_name="HIVE") target_version = upgrade_summary.get_target_version( service_name="HIVE") source_dir = format("/usr/hdp/{source_version}") target_dir = format("/usr/hdp/{target_version}") if params.security_enabled: hive_kinit_cmd = format( "{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; " ) Execute(hive_kinit_cmd, user=params.hive_user) # in the M10 release PreUpgradeTool was fixed to use Hive1 instead of Hive2 if target_version >= "3.0.3": hive_lib_dir = format("{source_dir}/hive/lib") else: hive_lib_dir = format("{source_dir}/hive2/lib") classpath = format( "{hive_lib_dir}/*:{source_dir}/hadoop/*:{source_dir}/hadoop/lib/*:{source_dir}/hadoop-mapreduce/*:{source_dir}/hadoop-mapreduce/lib/*:{source_dir}/hadoop-hdfs/*:{source_dir}/hadoop-hdfs/lib/*:{source_dir}/hadoop/etc/hadoop/:{target_dir}/hive/lib/hive-pre-upgrade.jar:{source_dir}/hive/conf/conf.server" ) # hack to avoid derby cp issue we want derby-10.10.2.0.jar to appear first in cp, if its available, note other derby jars are derbyclient-10.11.1.1.jar derbynet-10.11.1.1.jar derby_jars = glob.glob(source_dir + "/hive2/lib/*derby-*.jar") if len(derby_jars) == 1: classpath = derby_jars[0] + ":" + classpath cmd = format( "{java64_home}/bin/java -Djavax.security.auth.useSubjectCredsOnly=false -cp {classpath} org.apache.hadoop.hive.upgrade.acid.PreUpgradeTool -execute" ) Execute(cmd, user=params.hive_user)
def convert_tables(self, env): import params env.set_params(params) source_version = upgrade_summary.get_source_version( service_name="HIVE") target_version = upgrade_summary.get_target_version( service_name="HIVE") source_dir = format("/usr/hdp/{source_version}") target_dir = format("/usr/hdp/{target_version}") if params.security_enabled: hive_kinit_cmd = format( "{kinit_path_local} -kt {hive_server2_keytab} {hive_principal}; " ) Execute(hive_kinit_cmd, user=params.hive_user) classpath = format( "{source_dir}/hive2/lib/*:{source_dir}/hadoop/*:{source_dir}/hadoop/lib/*:{source_dir}/hadoop-mapreduce/*:{source_dir}/hadoop-mapreduce/lib/*:{target_dir}/hive/lib/hive-pre-upgrade.jar:{source_dir}/hive/conf" ) cmd = format( "{java64_home}/bin/java -Djavax.security.auth.useSubjectCredsOnly=false -cp {classpath} org.apache.hadoop.hive.upgrade.acid.PreUpgradeTool -execute" ) Execute(cmd, user=params.hive_user)
def get_current_version(service=None, use_upgrading_version_during_upgrade=True): """ Get the effective version to use to copy the tarballs to. :param service: the service name when checking for an upgrade. made optional for unknown \ code bases that may be using this function :param use_upgrading_version_during_upgrade: True, except when the RU/EU hasn't started yet. :return: Version, or False if an error occurred. """ from resource_management.libraries.functions import upgrade_summary # get the version for this command version = stack_features.get_stack_feature_version(Script.get_config()) if service is not None: version = upgrade_summary.get_target_version(service_name=service, default_version=version) # if there is no upgrade, then use the command's version if not Script.in_stack_upgrade() or use_upgrading_version_during_upgrade: Logger.info("Tarball version was calcuated as {0}. Use Command Version: {1}".format( version, use_upgrading_version_during_upgrade)) return version # we're in an upgrade and we need to use an older version current_version = stack_select.get_role_component_current_stack_version() if service is not None: current_version = upgrade_summary.get_source_version(service_name=service, default_version=current_version) if current_version is None: Logger.warning("Unable to determine the current version of the component for this command; unable to copy the tarball") return False return current_version;
def update_atlas_simple_authz(self, env): import params env.set_params(params) if params.upgrade_direction == Direction.UPGRADE: orchestration = stack_select.PACKAGE_SCOPE_STANDARD summary = upgrade_summary.get_upgrade_summary() if summary is not None: orchestration = summary.orchestration if orchestration is None: raise Fail( "The upgrade summary does not contain an orchestration type" ) if orchestration.upper( ) in stack_select._PARTIAL_ORCHESTRATION_SCOPES: orchestration = stack_select.PACKAGE_SCOPE_PATCH stack_select_packages = stack_select.get_packages( orchestration, service_name="ATLAS", component_name="ATLAS_SERVER") if stack_select_packages is None: raise Fail("Unable to get packages for stack-select") Logger.info( "ATLAS_SERVER component will be stack-selected to version {0} using a {1} orchestration" .format(params.version, orchestration.upper())) for stack_select_package_name in stack_select_packages: stack_select.select(stack_select_package_name, params.version) Directory( format('{metadata_home}/'), owner=params.metadata_user, group=params.user_group, recursive_ownership=True, ) target_version = upgrade_summary.get_target_version('ATLAS') update_atlas_simple_authz_script = os.path.join( format('{stack_root}'), target_version, 'atlas', 'bin', 'atlas_update_simple_auth_json.py') update_atlas_simple_authz_command = format( 'source {params.conf_dir}/atlas-env.sh ; {update_atlas_simple_authz_script} {conf_dir}' ) Execute( update_atlas_simple_authz_command, only_if=format("test -e {update_atlas_simple_authz_script}"), user=params.metadata_user) atlas_simple_auth_policy_file = os.path.join( format('{conf_dir}'), 'atlas-simple-authz-policy.json') File(atlas_simple_auth_policy_file, group=params.user_group, owner=params.metadata_user, only_if=format("test -e {atlas_simple_auth_policy_file}"), mode=0644)
def move_tables(self, env): import params env.set_params(params) create_hive_hdfs_dirs() target_version = upgrade_summary.get_target_version( service_name="HIVE") hive_script = format("{install_dir}/bin/hive") cmd = format( "{hive_script} --config /etc/hive --service strictmanagedmigration --hiveconf hive.strict.managed.tables=true -m automatic --modifyManagedTables --oldWarehouseRoot /apps/hive/warehouse" ) Execute(cmd, environment={'JAVA_HOME': params.java64_home}, user=params.hdfs_user)
def setup_ranger_kms_database(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] target_version = upgrade_summary.get_target_version( "RANGER_KMS", default_version=stack_version) Logger.info( format( 'Setting Ranger KMS database schema, using version {target_version}' )) kms.setup_kms_db(stack_version=target_version)
def setup_ranger_java_patches(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.upgrade_direction == Direction.UPGRADE: target_version = upgrade_summary.get_target_version( "RANGER", default_version=stack_version) Logger.info( format( 'Applying Ranger java patches, using version {target_version}' )) setup_ranger_xml.setup_java_patch(stack_version=target_version)
def convert_tables(self, env): import params env.set_params(params) source_version = upgrade_summary.get_source_version( service_name="HIVE") target_version = upgrade_summary.get_target_version( service_name="HIVE") source_dir = format("/usr/hdp/{source_version}") target_dir = format("/usr/hdp/{target_version}") classpath = format( "{source_dir}/hive2/lib/*:{source_dir}/hadoop/*:{source_dir}/hadoop/lib/*:{source_dir}/hadoop-mapreduce/*:{source_dir}/hadoop-mapreduce/lib/*:{target_dir}/hive/lib/hive-pre-upgrade.jar:{source_dir}/hive/conf" ) cmd = format( "{java64_home}/bin/java -cp {classpath} org.apache.hadoop.hive.upgrade.acid.PreUpgradeTool -execute" ) Execute(cmd, user="******")
def setup_ranger_database(self, env): import params env.set_params(params) upgrade_stack = stack_select._get_upgrade_stack() if upgrade_stack is None: raise Fail('Unable to determine the stack and stack version') stack_version = upgrade_stack[1] if params.xml_configurations_supported and params.upgrade_direction == Direction.UPGRADE: target_version = upgrade_summary.get_target_version( "RANGER", default_version=stack_version) Logger.info( format( 'Setting Ranger database schema, using version {target_version}' )) from setup_ranger_xml import setup_ranger_db setup_ranger_db(stack_version=target_version)
def configure_atlas_for_upgrade(self, env): import params env.set_params(params) if params.upgrade_direction == Direction.UPGRADE: if params.stack_supports_atlas_ranger_plugin and params.enable_ranger_atlas: atlas_policycache_file_v1 = os.path.join( '/etc', 'ranger', params.repo_name, 'policycache', format('atlas_{repo_name}.json')) atlas_policycache_file_v1_rename = os.path.join( '/etc', 'ranger', params.repo_name, 'policycache', format('atlas_{repo_name}_v1.json')) if os.path.isfile(atlas_policycache_file_v1): Execute(format( 'mv {atlas_policycache_file_v1} {atlas_policycache_file_v1_rename}' ), user=params.metadata_user, logoutput=True) target_version = upgrade_summary.get_target_version('ATLAS') atlas_simple_auth_policy_file_source = os.path.join( format('{stack_root}'), target_version, 'etc', 'atlas', 'conf.dist', 'atlas-simple-authz-policy.json') atlas_simple_auth_policy_file_target = os.path.join( format('{conf_dir}'), 'atlas-simple-authz-policy.json') Execute(('cp', '-f', atlas_simple_auth_policy_file_source, atlas_simple_auth_policy_file_target), not_if=format( 'test -e {atlas_simple_auth_policy_file_target}'), only_if=format( 'test -e {atlas_simple_auth_policy_file_source}'), sudo=True) File(atlas_simple_auth_policy_file_target, group=params.user_group, owner=params.metadata_user, only_if=format( "test -e {atlas_simple_auth_policy_file_target}"), mode=0644)