def _run_main(self, parsed_args, parsed_globals): parameters = {'ClusterId': parsed_args.cluster_id} describe_cluster_result = self._call(self._session, 'describe_cluster', parameters, parsed_globals) list_instance_groups_result = self._call(self._session, 'list_instance_groups', parameters, parsed_globals) list_bootstrap_actions_result = self._call(self._session, 'list_bootstrap_actions', parameters, parsed_globals) master_public_dns = self._find_master_public_dns( cluster_id=parsed_args.cluster_id, parsed_globals=parsed_globals) constructed_result = self._construct_result( describe_cluster_result, list_instance_groups_result, list_bootstrap_actions_result, master_public_dns) emrutils.display_response(self._session, 'describe_cluster', constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service('emr') describe_cluster = emr.get_operation('DescribeCluster') parameters = {'ClusterId': parsed_args.cluster_id} describe_cluster_result = self._call(describe_cluster, parameters, parsed_globals) list_instance_groups_result = self._call( emr.get_operation('ListInstanceGroups'), parameters, parsed_globals) list_bootstrap_actions_result = self._call( emr.get_operation('ListBootstrapActions'), parameters, parsed_globals) master_public_dns = self._find_master_public_dns( cluster_id=parsed_args.cluster_id, parsed_globals=parsed_globals) constructed_result = self._construct_result( describe_cluster_result, list_instance_groups_result, list_bootstrap_actions_result, master_public_dns) emrutils.display_response(self._session, describe_cluster, constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): parameters = {'ClusterId': parsed_args.cluster_id} describe_cluster_result = self._call( self._session, 'describe_cluster', parameters, parsed_globals) list_instance_groups_result = self._call( self._session, 'list_instance_groups', parameters, parsed_globals) list_bootstrap_actions_result = self._call( self._session, 'list_bootstrap_actions', parameters, parsed_globals) master_public_dns = self._find_master_public_dns( cluster_id=parsed_args.cluster_id, parsed_globals=parsed_globals) constructed_result = self._construct_result( describe_cluster_result, list_instance_groups_result, list_bootstrap_actions_result, master_public_dns) emrutils.display_response(self._session, 'describe_cluster', constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): ec2_result = None emr_result = None self.iam_endpoint_url = parsed_args.iam_endpoint self._check_for_iam_endpoint(self.region, self.iam_endpoint_url) self.emr_endpoint_url = \ self._session.create_client( 'emr', region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl).meta.endpoint_url LOG.debug('elasticmapreduce endpoint used for resolving' ' service principal: ' + self.emr_endpoint_url) # Check if the default EC2 Role for EMR exists. role_name = EC2_ROLE_NAME if self._check_if_role_exists(role_name, parsed_globals): LOG.debug('Role ' + role_name + ' exists.') else: LOG.debug('Role ' + role_name + ' does not exist.' ' Creating default role for EC2: ' + role_name) ec2_result = self._create_role_with_role_policy( role_name, role_name, EC2, emrutils.dict_to_string(EC2_ROLE_POLICY), parsed_globals) # Check if the default EC2 Instance Profile for EMR exists. instance_profile_name = EC2_ROLE_NAME if self._check_if_instance_profile_exists(instance_profile_name, parsed_globals): LOG.debug('Instance Profile ' + instance_profile_name + ' exists.') else: LOG.debug('Instance Profile ' + instance_profile_name + 'does not exist. Creating default Instance Profile ' + instance_profile_name) self._create_instance_profile_with_role(instance_profile_name, instance_profile_name, parsed_globals) # Check if the default EMR Role exists. role_name = EMR_ROLE_NAME if self._check_if_role_exists(role_name, parsed_globals): LOG.debug('Role ' + role_name + ' exists.') else: LOG.debug('Role ' + role_name + ' does not exist.' ' Creating default role for EMR: ' + role_name) emr_result = self._create_role_with_role_policy( role_name, role_name, EMR, emrutils.dict_to_string(EMR_ROLE_POLICY), parsed_globals) configutils.update_roles(self._session) emrutils.display_response( self._session, 'create_role', self._construct_result(ec2_result, emr_result), parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): ec2_result = None ec2_policy = None emr_result = None emr_policy = None self.iam_endpoint_url = parsed_args.iam_endpoint self._check_for_iam_endpoint(self.region, self.iam_endpoint_url) self.emr_endpoint_url = self._session.create_client( "emr", region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl ).meta.endpoint_url LOG.debug("elasticmapreduce endpoint used for resolving" " service principal: " + self.emr_endpoint_url) # Check if the default EC2 Role for EMR exists. role_name = EC2_ROLE_NAME if self._check_if_role_exists(role_name, parsed_globals): LOG.debug("Role " + role_name + " exists.") else: LOG.debug("Role " + role_name + " does not exist." " Creating default role for EC2: " + role_name) role_arn = get_ec2_role_policy_arn(self.region) ec2_result = self._create_role_with_role_policy(role_name, EC2, role_arn, parsed_globals) ec2_policy = self._get_role_policy(role_arn, parsed_globals) # Check if the default EC2 Instance Profile for EMR exists. instance_profile_name = EC2_ROLE_NAME if self._check_if_instance_profile_exists(instance_profile_name, parsed_globals): LOG.debug("Instance Profile " + instance_profile_name + " exists.") else: LOG.debug( "Instance Profile " + instance_profile_name + "does not exist. Creating default Instance Profile " + instance_profile_name ) self._create_instance_profile_with_role(instance_profile_name, instance_profile_name, parsed_globals) # Check if the default EMR Role exists. role_name = EMR_ROLE_NAME if self._check_if_role_exists(role_name, parsed_globals): LOG.debug("Role " + role_name + " exists.") else: LOG.debug("Role " + role_name + " does not exist." " Creating default role for EMR: " + role_name) role_arn = get_service_role_policy_arn(self.region) emr_result = self._create_role_with_role_policy(role_name, EMR, role_arn, parsed_globals) emr_policy = self._get_role_policy(role_arn, parsed_globals) configutils.update_roles(self._session) emrutils.display_response( self._session, "create_role", self._construct_result(ec2_result, ec2_policy, emr_result, emr_policy), parsed_globals, ) return 0
def _run_main_command(self, parsed_args, parsed_globals): self.iam_endpoint_url = parsed_args.iam_endpoint self._check_for_iam_endpoint(self.region, self.iam_endpoint_url) self.emr_endpoint_url = \ self._session.create_client( 'emr', region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl).meta.endpoint_url LOG.debug('elasticmapreduce endpoint used for resolving' ' service principal: ' + self.emr_endpoint_url) # Create default EC2 Role for EMR if it does not exist. ec2_result, ec2_policy = self._create_role_if_not_exists(parsed_globals, EC2_ROLE_NAME, EC2_ROLE_POLICY_NAME, [EC2]) # Create default EC2 Instance Profile for EMR if it does not exist. instance_profile_name = EC2_ROLE_NAME if self.check_if_instance_profile_exists(instance_profile_name, parsed_globals): LOG.debug('Instance Profile ' + instance_profile_name + ' exists.') else: LOG.debug('Instance Profile ' + instance_profile_name + 'does not exist. Creating default Instance Profile ' + instance_profile_name) self._create_instance_profile_with_role(instance_profile_name, instance_profile_name, parsed_globals) # Create default EMR Role if it does not exist. emr_result, emr_policy = self._create_role_if_not_exists(parsed_globals, EMR_ROLE_NAME, EMR_ROLE_POLICY_NAME, [EMR]) # Create default EMR AutoScaling Role if it does not exist. emr_autoscaling_result, emr_autoscaling_policy = \ self._create_role_if_not_exists(parsed_globals, EMR_AUTOSCALING_ROLE_NAME, EMR_AUTOSCALING_ROLE_POLICY_NAME, [EMR, APPLICATION_AUTOSCALING]) configutils.update_roles(self._session) emrutils.display_response( self._session, 'create_role', self._construct_result(ec2_result, ec2_policy, emr_result, emr_policy, emr_autoscaling_result, emr_autoscaling_policy), parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): ec2_result = None emr_result = None self.iam = self._session.get_service('iam') self.iam_endpoint_url = parsed_args.iam_endpoint region = self._get_region(parsed_globals) self._check_for_iam_endpoint(region, self.iam_endpoint_url) self.emr_endpoint_url = \ self._session.get_service('emr').get_endpoint( region_name=parsed_globals.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl).host LOG.debug('elasticmapreduce endpoint used for resolving' ' service principal: ' + self.emr_endpoint_url) # Check if the default EC2 Role for EMR exists. role_name = EC2_ROLE_NAME if self._check_if_role_exists(role_name, parsed_globals): LOG.debug('Role ' + role_name + ' exists.') else: LOG.debug('Role ' + role_name + ' does not exist.' ' Creating default role ' + role_name) ec2_result = self._create_role_with_role_policy( role_name, role_name, constants.EC2, emrutils.dict_to_string(EC2_ROLE_POLICY), parsed_globals) # Check if the default EC2 Instance Profile for EMR exists. instance_profile_name = EC2_ROLE_NAME if self._check_if_instance_profile_exists(instance_profile_name, parsed_globals): LOG.debug('Instance Profile ' + instance_profile_name + ' exists.') else: LOG.debug('Instance Profile ' + instance_profile_name + 'does not exist. Creating default Instance Profile ' + instance_profile_name) self._create_instance_profile_with_role(instance_profile_name, instance_profile_name, parsed_globals) emrutils.display_response( self._session, self._session.get_service('iam').get_operation('CreateRole'), self._construct_result(ec2_result), parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): parameters = {'JobFlowId': parsed_args.cluster_id} parameters['InstanceGroups'] = \ instancegroupsutils.build_instance_groups( parsed_args.instance_groups) add_instance_groups_response = emrutils.call( self._session, 'add_instance_groups', parameters, self.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result( add_instance_groups_response) emrutils.display_response(self._session, 'add_instance_groups', constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): parameters = {'ClusterId': parsed_args.cluster_id} list_instance_fleets_result = None list_instance_groups_result = None is_fleet_based_cluster = False describe_cluster_result = self._call(self._session, 'describe_cluster', parameters, parsed_globals) if 'Cluster' in describe_cluster_result: describe_cluster = describe_cluster_result['Cluster'] if describe_cluster.get( 'InstanceCollectionType') == constants.INSTANCE_FLEET_TYPE: is_fleet_based_cluster = True if 'Ec2InstanceAttributes' in describe_cluster: ec2_instance_attr_keys = \ describe_cluster['Ec2InstanceAttributes'].keys() ec2_instance_attr = \ describe_cluster['Ec2InstanceAttributes'] else: ec2_instance_attr_keys = {} if is_fleet_based_cluster: list_instance_fleets_result = self._call(self._session, 'list_instance_fleets', parameters, parsed_globals) else: list_instance_groups_result = self._call(self._session, 'list_instance_groups', parameters, parsed_globals) list_bootstrap_actions_result = self._call(self._session, 'list_bootstrap_actions', parameters, parsed_globals) constructed_result = self._construct_result( describe_cluster_result, list_instance_fleets_result, list_instance_groups_result, list_bootstrap_actions_result) emrutils.display_response(self._session, 'describe_cluster', constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): parameters = {'ClusterId': parsed_args.cluster_id} list_instance_fleets_result = None list_instance_groups_result = None is_fleet_based_cluster = False describe_cluster_result = self._call( self._session, 'describe_cluster', parameters, parsed_globals) if 'Cluster' in describe_cluster_result: describe_cluster = describe_cluster_result['Cluster'] if describe_cluster.get('InstanceCollectionType') == constants.INSTANCE_FLEET_TYPE: is_fleet_based_cluster = True if 'Ec2InstanceAttributes' in describe_cluster: ec2_instance_attr_keys = \ describe_cluster['Ec2InstanceAttributes'].keys() ec2_instance_attr = \ describe_cluster['Ec2InstanceAttributes'] else: ec2_instance_attr_keys = {} if is_fleet_based_cluster: list_instance_fleets_result = self._call( self._session, 'list_instance_fleets', parameters, parsed_globals) else: list_instance_groups_result = self._call( self._session, 'list_instance_groups', parameters, parsed_globals) list_bootstrap_actions_result = self._call( self._session, 'list_bootstrap_actions', parameters, parsed_globals) constructed_result = self._construct_result( describe_cluster_result, list_instance_fleets_result, list_instance_groups_result, list_bootstrap_actions_result) emrutils.display_response(self._session, 'describe_cluster', constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service('emr') add_instance_groups = emr.get_operation('AddInstanceGroups') parameters = {'JobFlowId': parsed_args.cluster_id} parameters['InstanceGroups'] = \ instancegroupsutils.build_instance_groups( parsed_args.instance_groups) add_instance_groups_response = emrutils.call( self._session, add_instance_groups, parameters, parsed_globals.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result( add_instance_groups_response[1]) emrutils.display_response(self._session, add_instance_groups, constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service("emr") add_instance_groups = emr.get_operation("AddInstanceGroups") parameters = {"JobFlowId": parsed_args.cluster_id} parameters["InstanceGroups"] = instancegroupsutils.build_instance_groups(parsed_args.instance_groups) add_instance_groups_response = emrutils.call( self._session, add_instance_groups, parameters, parsed_globals.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl, ) constructed_result = self._construct_result(add_instance_groups_response[1]) emrutils.display_response(self._session, add_instance_groups, constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): params = {} params['Name'] = parsed_args.name self._validate_release_label_ami_version(parsed_args) service_role_validation_message = ( " Either choose --use-default-roles or use both --service-role " "<roleName> and --ec2-attributes InstanceProfile=<profileName>.") if parsed_args.use_default_roles is True and \ parsed_args.service_role is not None: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--service-role", message=service_role_validation_message) if parsed_args.use_default_roles is True and \ parsed_args.ec2_attributes is not None and \ 'InstanceProfile' in parsed_args.ec2_attributes: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--ec2-attributes InstanceProfile", message=service_role_validation_message) if parsed_args.instance_groups is not None and \ parsed_args.instance_fleets is not None: raise exceptions.MutualExclusiveOptionError( option1="--instance-groups", option2="--instance-fleets") instances_config = {} if parsed_args.instance_fleets is not None: instances_config['InstanceFleets'] = \ instancefleetsutils.validate_and_build_instance_fleets( parsed_args.instance_fleets) else: instances_config['InstanceGroups'] = \ instancegroupsutils.validate_and_build_instance_groups( instance_groups=parsed_args.instance_groups, instance_type=parsed_args.instance_type, instance_count=parsed_args.instance_count) if parsed_args.release_label is not None: params["ReleaseLabel"] = parsed_args.release_label if parsed_args.configurations is not None: try: params["Configurations"] = json.loads( parsed_args.configurations) except ValueError: raise ValueError('aws: error: invalid json argument for ' 'option --configurations') if (parsed_args.release_label is None and parsed_args.ami_version is not None): is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version) if is_valid_ami_version is None: raise exceptions.InvalidAmiVersionError( ami_version=parsed_args.ami_version) params['AmiVersion'] = parsed_args.ami_version emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) if parsed_args.log_encryption_kms_key_id is not None: emrutils.apply_dict(params, 'LogEncryptionKmsKeyId', parsed_args.log_encryption_kms_key_id) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) if parsed_args.instance_groups is not None: for instance_group in instances_config['InstanceGroups']: if 'AutoScalingPolicy' in instance_group.keys(): if parsed_args.auto_scaling_role is None: raise exceptions.MissingAutoScalingRoleError() emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role) if parsed_args.scale_down_behavior is not None: emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior) if ( parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False): parsed_args.no_auto_terminate = True instances_config['KeepJobFlowAliveWhenNoSteps'] = \ emrutils.apply_boolean_options( parsed_args.no_auto_terminate, '--no-auto-terminate', parsed_args.auto_terminate, '--auto-terminate') instances_config['TerminationProtected'] = \ emrutils.apply_boolean_options( parsed_args.termination_protected, '--termination-protected', parsed_args.no_termination_protected, '--no-termination-protected') if (parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False): parsed_args.visible_to_all_users = True params['VisibleToAllUsers'] = \ emrutils.apply_boolean_options( parsed_args.visible_to_all_users, '--visible-to-all-users', parsed_args.no_visible_to_all_users, '--no-visible-to-all-users') params['Tags'] = emrutils.parse_tags(parsed_args.tags) params['Instances'] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes( cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, '--enable-debugging', parsed_args.no_enable_debugging, '--no-enable-debugging') if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict( cluster=params, key='Steps', value=[ self._build_enable_debugging(parsed_args, parsed_globals)]) if parsed_args.applications is not None: if parsed_args.release_label is None: app_list, ba_list, step_list = \ applicationutils.build_applications( region=self.region, parsed_applications=parsed_args.applications, ami_version=params['AmiVersion']) self._update_cluster_dict( params, 'NewSupportedProducts', app_list) self._update_cluster_dict( params, 'BootstrapActions', ba_list) self._update_cluster_dict( params, 'Steps', step_list) else: params["Applications"] = [] for application in parsed_args.applications: params["Applications"].append(application) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get('Dir'), backup_version=hbase_restore_config.get('BackupVersion')) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args) self._update_cluster_dict( params, 'Steps', [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions( cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.emrfs is not None: self._handle_emrfs_parameters( cluster=params, emrfs_args=parsed_args.emrfs, release_label=parsed_args.release_label) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=self.region, release_label=parsed_args.release_label) self._update_cluster_dict( cluster=params, key='Steps', value=steps_list) if parsed_args.security_configuration is not None: emrutils.apply_dict( params, 'SecurityConfiguration', parsed_args.security_configuration) if parsed_args.custom_ami_id is not None: emrutils.apply_dict( params, 'CustomAmiId', parsed_args.custom_ami_id ) if parsed_args.ebs_root_volume_size is not None: emrutils.apply_dict( params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size) ) if parsed_args.repo_upgrade_on_boot is not None: emrutils.apply_dict( params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot ) if parsed_args.kerberos_attributes is not None: emrutils.apply_dict( params, 'KerberosAttributes', parsed_args.kerberos_attributes) if parsed_args.step_concurrency_level is not None: params['StepConcurrencyLevel'] = parsed_args.step_concurrency_level if parsed_args.managed_scaling_policy is not None: emrutils.apply_dict( params, 'ManagedScalingPolicy', parsed_args.managed_scaling_policy) if parsed_args.placement_group_configs is not None: emrutils.apply_dict( params, 'PlacementGroupConfigs', parsed_args.placement_group_configs) self._validate_required_applications(parsed_args) run_job_flow_response = emrutils.call( self._session, 'run_job_flow', params, self.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result(run_job_flow_response) emrutils.display_response(self._session, 'run_job_flow', constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service('emr') params = {} bootstrap_actions = [] params['Name'] = parsed_args.name instances_config = {} instances_config['InstanceGroups'] = \ instancegroupsutils.validate_and_build_instance_groups( instance_groups=parsed_args.instance_groups, instance_type=parsed_args.instance_type, instance_count=parsed_args.instance_count) is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version) if is_valid_ami_version is None: raise exceptions.InvalidAmiVersionError( ami_version=parsed_args.ami_version) params['AmiVersion'] = parsed_args.ami_version emrutils.apply_dict(params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) if (parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False): parsed_args.no_auto_terminate = True instances_config['KeepJobFlowAliveWhenNoSteps'] = \ emrutils.apply_boolean_options( parsed_args.no_auto_terminate, '--no-auto-terminate', parsed_args.auto_terminate, '--auto-terminate') instances_config['TerminationProtected'] = \ emrutils.apply_boolean_options( parsed_args.termination_protected, '--termination-protected', parsed_args.no_termination_protected, '--no-termination-protected') if (parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False): parsed_args.visible_to_all_users = True params['VisibleToAllUsers'] = \ emrutils.apply_boolean_options( parsed_args.visible_to_all_users, '--visible-to-all-users', parsed_args.no_visible_to_all_users, '--no-visible-to-all-users') params['Tags'] = emrutils.parse_tags(parsed_args.tags) params['Instances'] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes(cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, '--enable-debugging', parsed_args.no_enable_debugging, '--no-enable-debugging') if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict( cluster=params, key='Steps', value=[self._build_enable_debugging(parsed_globals)]) if parsed_args.applications is not None: app_list, ba_list, step_list = applicationutils.build_applications( session=self._session, parsed_applications=parsed_args.applications, parsed_globals=parsed_globals, ami_version=params['AmiVersion']) self._update_cluster_dict(params, 'NewSupportedProducts', app_list) self._update_cluster_dict(params, 'BootstrapActions', ba_list) self._update_cluster_dict(params, 'Steps', step_list) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get('Dir'), backup_version=hbase_restore_config.get('BackupVersion')) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args) self._update_cluster_dict(params, 'Steps', [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions( cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.emrfs is not None: emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs) emr_fs_ba_config = \ emrutils.build_bootstrap_action( path=emrutils.build_s3_link( relative_path=constants.CONFIG_HADOOP_PATH, region=parsed_globals.region), name=constants.EMR_FS_BA_NAME, args=emr_fs_ba_args) self._update_cluster_dict(cluster=params, key='BootstrapActions', value=[emr_fs_ba_config]) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=parsed_globals.region) self._update_cluster_dict(cluster=params, key='Steps', value=steps_list) self._validate_required_applications(parsed_args) run_job_flow = emr.get_operation('RunJobFlow') run_job_flow_response = emrutils.call(self._session, run_job_flow, params, parsed_globals.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result(run_job_flow_response[1]) emrutils.display_response(self._session, run_job_flow, constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): params = {} bootstrap_actions = [] params['Name'] = parsed_args.name service_role_validation_message = ( " Either choose --use-default-roles or use both --service-role " "<roleName> and --ec2-attributes InstanceProfile=<profileName>.") if parsed_args.use_default_roles is True and \ parsed_args.service_role is not None: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--service-role", message=service_role_validation_message) if parsed_args.use_default_roles is True and \ parsed_args.ec2_attributes is not None and \ 'InstanceProfile' in parsed_args.ec2_attributes: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--ec2-attributes InstanceProfile", message=service_role_validation_message) instances_config = {} instances_config['InstanceGroups'] = \ instancegroupsutils.validate_and_build_instance_groups( instance_groups=parsed_args.instance_groups, instance_type=parsed_args.instance_type, instance_count=parsed_args.instance_count) is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version) if is_valid_ami_version is None: raise exceptions.InvalidAmiVersionError( ami_version=parsed_args.ami_version) params['AmiVersion'] = parsed_args.ami_version emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) if ( parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False): parsed_args.no_auto_terminate = True instances_config['KeepJobFlowAliveWhenNoSteps'] = \ emrutils.apply_boolean_options( parsed_args.no_auto_terminate, '--no-auto-terminate', parsed_args.auto_terminate, '--auto-terminate') instances_config['TerminationProtected'] = \ emrutils.apply_boolean_options( parsed_args.termination_protected, '--termination-protected', parsed_args.no_termination_protected, '--no-termination-protected') if ( parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False): parsed_args.visible_to_all_users = True params['VisibleToAllUsers'] = \ emrutils.apply_boolean_options( parsed_args.visible_to_all_users, '--visible-to-all-users', parsed_args.no_visible_to_all_users, '--no-visible-to-all-users') params['Tags'] = emrutils.parse_tags(parsed_args.tags) params['Instances'] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes( cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, '--enable-debugging', parsed_args.no_enable_debugging, '--no-enable-debugging') if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict( cluster=params, key='Steps', value=[self._build_enable_debugging(parsed_globals)]) if parsed_args.applications is not None: app_list, ba_list, step_list = applicationutils.build_applications( region=self.region, parsed_applications=parsed_args.applications, ami_version=params['AmiVersion']) self._update_cluster_dict( params, 'NewSupportedProducts', app_list) self._update_cluster_dict( params, 'BootstrapActions', ba_list) self._update_cluster_dict( params, 'Steps', step_list) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get('Dir'), backup_version=hbase_restore_config.get('BackupVersion')) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args) self._update_cluster_dict( params, 'Steps', [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions( cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.emrfs is not None: emr_fs_ba_config_list = emrfsutils.build_bootstrap_action_configs( self.region, parsed_args.emrfs) self._update_cluster_dict( cluster=params, key='BootstrapActions', value=emr_fs_ba_config_list) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=self.region) self._update_cluster_dict( cluster=params, key='Steps', value=steps_list) self._validate_required_applications(parsed_args) run_job_flow_response = emrutils.call( self._session, 'run_job_flow', params, self.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result(run_job_flow_response) emrutils.display_response(self._session, 'run_job_flow', constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service('emr') params = {} bootstrap_actions = [] params['Name'] = parsed_args.name is_valid_ami = re.match('\d?\..*', parsed_args.ami_version) if is_valid_ami is None: raise exceptions.\ InvalidAmiVersionError(ami_version=parsed_args.ami_version) params['AmiVersion'] = parsed_args.ami_version emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) instances_config = {} instances_config['InstanceGroups'] = \ instancegroupsutils.build_instance_groups( parsed_args.instance_groups) if ( parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False): raise exceptions.\ MissingBooleanOptionsError( true_option='--auto-terminate', false_option='--no-auto-terminate') instances_config['KeepJobFlowAliveWhenNoSteps'] = \ emrutils.apply_boolean_options( parsed_args.no_auto_terminate, '--no-auto-terminate', parsed_args.auto_terminate, '--auto-terminate') instances_config['TerminationProtected'] = \ emrutils.apply_boolean_options( parsed_args.termination_protected, '--termination-protected', parsed_args.no_termination_protected, '--no-termination-protected') params['VisibleToAllUsers'] = \ emrutils.apply_boolean_options( parsed_args.visible_to_all_users, '--visible-to-all-users', parsed_args.no_visible_to_all_users, '--no-visible-to-all-users') params['Tags'] = emrutils.parse_tags(parsed_args.tags) params['Instances'] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes( cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, '--enable-debugging', parsed_args.no_enable_debugging, '--no-enable-debugging') if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict( cluster=params, key='Steps', value=[self._build_enable_debugging(parsed_globals)]) if parsed_args.applications is not None: app_list, ba_list, step_list = applicationutils.build_applications( parsed_applications=parsed_args.applications, parsed_globals=parsed_globals, ami_version=params['AmiVersion']) self._update_cluster_dict( params, 'NewSupportedProducts', app_list) self._update_cluster_dict( params, 'BootstrapActions', ba_list) self._update_cluster_dict( params, 'Steps', step_list) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get('Dir'), backup_version=hbase_restore_config.get('BackupVersion')) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args) self._update_cluster_dict( params, 'Steps', [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions( cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=parsed_globals.region) self._update_cluster_dict( cluster=params, key='Steps', value=steps_list) self._validate_required_applications(parsed_args) run_job_flow = emr.get_operation('RunJobFlow') run_job_flow_response = emrutils.call( self._session, run_job_flow, params, parsed_globals.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result(run_job_flow_response[1]) emrutils.display_response(self._session, run_job_flow, constructed_result, parsed_globals) return 0
def _run_main(self, parsed_args, parsed_globals): emr = self._session.get_service("emr") params = {} bootstrap_actions = [] params["Name"] = parsed_args.name service_role_validation_message = ( " Either choose --use-default-roles or use both --service-role " "<roleName> and --ec2-attributes InstanceProfile=<profileName>." ) if parsed_args.use_default_roles is True and parsed_args.service_role is not None: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--service-role", message=service_role_validation_message ) if ( parsed_args.use_default_roles is True and parsed_args.ec2_attributes is not None and "InstanceProfile" in parsed_args.ec2_attributes ): raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--ec2-attributes InstanceProfile", message=service_role_validation_message, ) instances_config = {} instances_config["InstanceGroups"] = instancegroupsutils.validate_and_build_instance_groups( instance_groups=parsed_args.instance_groups, instance_type=parsed_args.instance_type, instance_count=parsed_args.instance_count, ) is_valid_ami_version = re.match("\d?\..*", parsed_args.ami_version) if is_valid_ami_version is None: raise exceptions.InvalidAmiVersionError(ami_version=parsed_args.ami_version) params["AmiVersion"] = parsed_args.ami_version emrutils.apply_dict(params, "AdditionalInfo", parsed_args.additional_info) emrutils.apply_dict(params, "LogUri", parsed_args.log_uri) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes["InstanceProfile"] = EC2_ROLE_NAME emrutils.apply_dict(params, "ServiceRole", parsed_args.service_role) if parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False: parsed_args.no_auto_terminate = True instances_config["KeepJobFlowAliveWhenNoSteps"] = emrutils.apply_boolean_options( parsed_args.no_auto_terminate, "--no-auto-terminate", parsed_args.auto_terminate, "--auto-terminate" ) instances_config["TerminationProtected"] = emrutils.apply_boolean_options( parsed_args.termination_protected, "--termination-protected", parsed_args.no_termination_protected, "--no-termination-protected", ) if parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False: parsed_args.visible_to_all_users = True params["VisibleToAllUsers"] = emrutils.apply_boolean_options( parsed_args.visible_to_all_users, "--visible-to-all-users", parsed_args.no_visible_to_all_users, "--no-visible-to-all-users", ) params["Tags"] = emrutils.parse_tags(parsed_args.tags) params["Instances"] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes(cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, "--enable-debugging", parsed_args.no_enable_debugging, "--no-enable-debugging" ) if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict(cluster=params, key="Steps", value=[self._build_enable_debugging(parsed_globals)]) if parsed_args.applications is not None: app_list, ba_list, step_list = applicationutils.build_applications( session=self._session, parsed_applications=parsed_args.applications, parsed_globals=parsed_globals, ami_version=params["AmiVersion"], ) self._update_cluster_dict(params, "NewSupportedProducts", app_list) self._update_cluster_dict(params, "BootstrapActions", ba_list) self._update_cluster_dict(params, "Steps", step_list) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get("Dir"), backup_version=hbase_restore_config.get("BackupVersion") ) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args, ) self._update_cluster_dict(params, "Steps", [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions(cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.emrfs is not None: emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs) emr_fs_ba_config = emrutils.build_bootstrap_action( path=emrutils.build_s3_link(relative_path=constants.CONFIG_HADOOP_PATH, region=parsed_globals.region), name=constants.EMR_FS_BA_NAME, args=emr_fs_ba_args, ) self._update_cluster_dict(cluster=params, key="BootstrapActions", value=[emr_fs_ba_config]) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=parsed_globals.region ) self._update_cluster_dict(cluster=params, key="Steps", value=steps_list) self._validate_required_applications(parsed_args) run_job_flow = emr.get_operation("RunJobFlow") run_job_flow_response = emrutils.call( self._session, run_job_flow, params, parsed_globals.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl, ) constructed_result = self._construct_result(run_job_flow_response[1]) emrutils.display_response(self._session, run_job_flow, constructed_result, parsed_globals) return 0
def _run_main_command(self, parsed_args, parsed_globals): params = {} params['Name'] = parsed_args.name self._validate_release_label_ami_version(parsed_args) service_role_validation_message = ( " Either choose --use-default-roles or use both --service-role " "<roleName> and --ec2-attributes InstanceProfile=<profileName>.") if parsed_args.use_default_roles is True and \ parsed_args.service_role is not None: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--service-role", message=service_role_validation_message) if parsed_args.use_default_roles is True and \ parsed_args.ec2_attributes is not None and \ 'InstanceProfile' in parsed_args.ec2_attributes: raise exceptions.MutualExclusiveOptionError( option1="--use-default-roles", option2="--ec2-attributes InstanceProfile", message=service_role_validation_message) if parsed_args.instance_groups is not None and \ parsed_args.instance_fleets is not None: raise exceptions.MutualExclusiveOptionError( option1="--instance-groups", option2="--instance-fleets") instances_config = {} if parsed_args.instance_fleets is not None: instances_config['InstanceFleets'] = \ instancefleetsutils.validate_and_build_instance_fleets( parsed_args.instance_fleets) else: instances_config['InstanceGroups'] = \ instancegroupsutils.validate_and_build_instance_groups( instance_groups=parsed_args.instance_groups, instance_type=parsed_args.instance_type, instance_count=parsed_args.instance_count) if parsed_args.release_label is not None: params["ReleaseLabel"] = parsed_args.release_label if parsed_args.configurations is not None: try: params["Configurations"] = json.loads( parsed_args.configurations) except ValueError: raise ValueError('aws: error: invalid json argument for ' 'option --configurations') if (parsed_args.release_label is None and parsed_args.ami_version is not None): is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version) if is_valid_ami_version is None: raise exceptions.InvalidAmiVersionError( ami_version=parsed_args.ami_version) params['AmiVersion'] = parsed_args.ami_version emrutils.apply_dict( params, 'AdditionalInfo', parsed_args.additional_info) emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri) if parsed_args.use_default_roles is True: parsed_args.service_role = EMR_ROLE_NAME if parsed_args.ec2_attributes is None: parsed_args.ec2_attributes = {} parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role) if parsed_args.instance_groups is not None: for instance_group in instances_config['InstanceGroups']: if 'AutoScalingPolicy' in instance_group.keys(): if parsed_args.auto_scaling_role is None: raise exceptions.MissingAutoScalingRoleError() emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role) if parsed_args.scale_down_behavior is not None: emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior) if ( parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False): parsed_args.no_auto_terminate = True instances_config['KeepJobFlowAliveWhenNoSteps'] = \ emrutils.apply_boolean_options( parsed_args.no_auto_terminate, '--no-auto-terminate', parsed_args.auto_terminate, '--auto-terminate') instances_config['TerminationProtected'] = \ emrutils.apply_boolean_options( parsed_args.termination_protected, '--termination-protected', parsed_args.no_termination_protected, '--no-termination-protected') if (parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False): parsed_args.visible_to_all_users = True params['VisibleToAllUsers'] = \ emrutils.apply_boolean_options( parsed_args.visible_to_all_users, '--visible-to-all-users', parsed_args.no_visible_to_all_users, '--no-visible-to-all-users') params['Tags'] = emrutils.parse_tags(parsed_args.tags) params['Instances'] = instances_config if parsed_args.ec2_attributes is not None: self._build_ec2_attributes( cluster=params, parsed_attrs=parsed_args.ec2_attributes) debugging_enabled = emrutils.apply_boolean_options( parsed_args.enable_debugging, '--enable-debugging', parsed_args.no_enable_debugging, '--no-enable-debugging') if parsed_args.log_uri is None and debugging_enabled is True: raise exceptions.LogUriError if debugging_enabled is True: self._update_cluster_dict( cluster=params, key='Steps', value=[ self._build_enable_debugging(parsed_args, parsed_globals)]) if parsed_args.applications is not None: if parsed_args.release_label is None: app_list, ba_list, step_list = \ applicationutils.build_applications( region=self.region, parsed_applications=parsed_args.applications, ami_version=params['AmiVersion']) self._update_cluster_dict( params, 'NewSupportedProducts', app_list) self._update_cluster_dict( params, 'BootstrapActions', ba_list) self._update_cluster_dict( params, 'Steps', step_list) else: params["Applications"] = [] for application in parsed_args.applications: params["Applications"].append(application) hbase_restore_config = parsed_args.restore_from_hbase_backup if hbase_restore_config is not None: args = hbaseutils.build_hbase_restore_from_backup_args( dir=hbase_restore_config.get('Dir'), backup_version=hbase_restore_config.get('BackupVersion')) step_config = emrutils.build_step( jar=constants.HBASE_JAR_PATH, name=constants.HBASE_RESTORE_STEP_NAME, action_on_failure=constants.CANCEL_AND_WAIT, args=args) self._update_cluster_dict( params, 'Steps', [step_config]) if parsed_args.bootstrap_actions is not None: self._build_bootstrap_actions( cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions) if parsed_args.emrfs is not None: self._handle_emrfs_parameters( cluster=params, emrfs_args=parsed_args.emrfs, release_label=parsed_args.release_label) if parsed_args.steps is not None: steps_list = steputils.build_step_config_list( parsed_step_list=parsed_args.steps, region=self.region, release_label=parsed_args.release_label) self._update_cluster_dict( cluster=params, key='Steps', value=steps_list) if parsed_args.security_configuration is not None: emrutils.apply_dict( params, 'SecurityConfiguration', parsed_args.security_configuration) if parsed_args.custom_ami_id is not None: emrutils.apply_dict( params, 'CustomAmiId', parsed_args.custom_ami_id ) if parsed_args.ebs_root_volume_size is not None: emrutils.apply_dict( params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size) ) if parsed_args.repo_upgrade_on_boot is not None: emrutils.apply_dict( params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot ) self._validate_required_applications(parsed_args) run_job_flow_response = emrutils.call( self._session, 'run_job_flow', params, self.region, parsed_globals.endpoint_url, parsed_globals.verify_ssl) constructed_result = self._construct_result(run_job_flow_response) emrutils.display_response(self._session, 'run_job_flow', constructed_result, parsed_globals) return 0