def _run_main(self, parsed_args, parsed_globals):
parameters = {'ClusterId': parsed_args.cluster_id}
describe_cluster_result = self._call(self._session, 'describe_cluster',
parameters, parsed_globals)
list_instance_groups_result = self._call(self._session,
'list_instance_groups',
parameters, parsed_globals)
list_bootstrap_actions_result = self._call(self._session,
'list_bootstrap_actions',
parameters, parsed_globals)
master_public_dns = self._find_master_public_dns(
cluster_id=parsed_args.cluster_id, parsed_globals=parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result, list_instance_groups_result,
list_bootstrap_actions_result, master_public_dns)
emrutils.display_response(self._session, 'describe_cluster',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service('emr')
describe_cluster = emr.get_operation('DescribeCluster')
parameters = {'ClusterId': parsed_args.cluster_id}
describe_cluster_result = self._call(describe_cluster, parameters,
parsed_globals)
list_instance_groups_result = self._call(
emr.get_operation('ListInstanceGroups'), parameters,
parsed_globals)
list_bootstrap_actions_result = self._call(
emr.get_operation('ListBootstrapActions'), parameters,
parsed_globals)
master_public_dns = self._find_master_public_dns(
cluster_id=parsed_args.cluster_id, parsed_globals=parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result, list_instance_groups_result,
list_bootstrap_actions_result, master_public_dns)
emrutils.display_response(self._session, describe_cluster,
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
parameters = {'ClusterId': parsed_args.cluster_id}
describe_cluster_result = self._call(
self._session, 'describe_cluster', parameters, parsed_globals)
list_instance_groups_result = self._call(
self._session, 'list_instance_groups', parameters, parsed_globals)
list_bootstrap_actions_result = self._call(
self._session, 'list_bootstrap_actions',
parameters, parsed_globals)
master_public_dns = self._find_master_public_dns(
cluster_id=parsed_args.cluster_id,
parsed_globals=parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result,
list_instance_groups_result,
list_bootstrap_actions_result,
master_public_dns)
emrutils.display_response(self._session, 'describe_cluster',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service('emr')
describe_cluster = emr.get_operation('DescribeCluster')
parameters = {'ClusterId': parsed_args.cluster_id}
describe_cluster_result = self._call(describe_cluster, parameters,
parsed_globals)
list_instance_groups_result = self._call(
emr.get_operation('ListInstanceGroups'), parameters,
parsed_globals)
list_bootstrap_actions_result = self._call(
emr.get_operation('ListBootstrapActions'),
parameters, parsed_globals)
master_public_dns = self._find_master_public_dns(
cluster_id=parsed_args.cluster_id,
parsed_globals=parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result,
list_instance_groups_result,
list_bootstrap_actions_result,
master_public_dns)
emrutils.display_response(self._session, describe_cluster,
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
ec2_result = None
emr_result = None
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.create_client(
'emr',
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).meta.endpoint_url
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Check if the default EC2 Role for EMR exists.
role_name = EC2_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role for EC2: ' + role_name)
ec2_result = self._create_role_with_role_policy(
role_name, role_name, EC2,
emrutils.dict_to_string(EC2_ROLE_POLICY), parsed_globals)
# Check if the default EC2 Instance Profile for EMR exists.
instance_profile_name = EC2_ROLE_NAME
if self._check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
# Check if the default EMR Role exists.
role_name = EMR_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role for EMR: ' + role_name)
emr_result = self._create_role_with_role_policy(
role_name, role_name, EMR,
emrutils.dict_to_string(EMR_ROLE_POLICY), parsed_globals)
configutils.update_roles(self._session)
emrutils.display_response(
self._session, 'create_role',
self._construct_result(ec2_result, emr_result), parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
ec2_result = None
ec2_policy = None
emr_result = None
emr_policy = None
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = self._session.create_client(
"emr", region_name=self.region, endpoint_url=parsed_globals.endpoint_url, verify=parsed_globals.verify_ssl
).meta.endpoint_url
LOG.debug("elasticmapreduce endpoint used for resolving" " service principal: " + self.emr_endpoint_url)
# Check if the default EC2 Role for EMR exists.
role_name = EC2_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug("Role " + role_name + " exists.")
else:
LOG.debug("Role " + role_name + " does not exist." " Creating default role for EC2: " + role_name)
role_arn = get_ec2_role_policy_arn(self.region)
ec2_result = self._create_role_with_role_policy(role_name, EC2, role_arn, parsed_globals)
ec2_policy = self._get_role_policy(role_arn, parsed_globals)
# Check if the default EC2 Instance Profile for EMR exists.
instance_profile_name = EC2_ROLE_NAME
if self._check_if_instance_profile_exists(instance_profile_name, parsed_globals):
LOG.debug("Instance Profile " + instance_profile_name + " exists.")
else:
LOG.debug(
"Instance Profile "
+ instance_profile_name
+ "does not exist. Creating default Instance Profile "
+ instance_profile_name
)
self._create_instance_profile_with_role(instance_profile_name, instance_profile_name, parsed_globals)
# Check if the default EMR Role exists.
role_name = EMR_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug("Role " + role_name + " exists.")
else:
LOG.debug("Role " + role_name + " does not exist." " Creating default role for EMR: " + role_name)
role_arn = get_service_role_policy_arn(self.region)
emr_result = self._create_role_with_role_policy(role_name, EMR, role_arn, parsed_globals)
emr_policy = self._get_role_policy(role_arn, parsed_globals)
configutils.update_roles(self._session)
emrutils.display_response(
self._session,
"create_role",
self._construct_result(ec2_result, ec2_policy, emr_result, emr_policy),
parsed_globals,
)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.create_client(
'emr',
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).meta.endpoint_url
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Create default EC2 Role for EMR if it does not exist.
ec2_result, ec2_policy = self._create_role_if_not_exists(parsed_globals, EC2_ROLE_NAME,
EC2_ROLE_POLICY_NAME, [EC2])
# Create default EC2 Instance Profile for EMR if it does not exist.
instance_profile_name = EC2_ROLE_NAME
if self.check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
# Create default EMR Role if it does not exist.
emr_result, emr_policy = self._create_role_if_not_exists(parsed_globals, EMR_ROLE_NAME,
EMR_ROLE_POLICY_NAME, [EMR])
# Create default EMR AutoScaling Role if it does not exist.
emr_autoscaling_result, emr_autoscaling_policy = \
self._create_role_if_not_exists(parsed_globals, EMR_AUTOSCALING_ROLE_NAME,
EMR_AUTOSCALING_ROLE_POLICY_NAME, [EMR, APPLICATION_AUTOSCALING])
configutils.update_roles(self._session)
emrutils.display_response(
self._session,
'create_role',
self._construct_result(ec2_result, ec2_policy,
emr_result, emr_policy,
emr_autoscaling_result, emr_autoscaling_policy),
parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.create_client(
'emr',
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).meta.endpoint_url
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Create default EC2 Role for EMR if it does not exist.
ec2_result, ec2_policy = self._create_role_if_not_exists(parsed_globals, EC2_ROLE_NAME,
EC2_ROLE_POLICY_NAME, [EC2])
# Create default EC2 Instance Profile for EMR if it does not exist.
instance_profile_name = EC2_ROLE_NAME
if self.check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
# Create default EMR Role if it does not exist.
emr_result, emr_policy = self._create_role_if_not_exists(parsed_globals, EMR_ROLE_NAME,
EMR_ROLE_POLICY_NAME, [EMR])
# Create default EMR AutoScaling Role if it does not exist.
emr_autoscaling_result, emr_autoscaling_policy = \
self._create_role_if_not_exists(parsed_globals, EMR_AUTOSCALING_ROLE_NAME,
EMR_AUTOSCALING_ROLE_POLICY_NAME, [EMR, APPLICATION_AUTOSCALING])
configutils.update_roles(self._session)
emrutils.display_response(
self._session,
'create_role',
self._construct_result(ec2_result, ec2_policy,
emr_result, emr_policy,
emr_autoscaling_result, emr_autoscaling_policy),
parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
ec2_result = None
emr_result = None
self.iam = self._session.get_service('iam')
self.iam_endpoint_url = parsed_args.iam_endpoint
region = self._get_region(parsed_globals)
self._check_for_iam_endpoint(region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.get_service('emr').get_endpoint(
region_name=parsed_globals.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).host
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Check if the default EC2 Role for EMR exists.
role_name = EC2_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role ' + role_name)
ec2_result = self._create_role_with_role_policy(
role_name, role_name, constants.EC2,
emrutils.dict_to_string(EC2_ROLE_POLICY),
parsed_globals)
# Check if the default EC2 Instance Profile for EMR exists.
instance_profile_name = EC2_ROLE_NAME
if self._check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
emrutils.display_response(
self._session,
self._session.get_service('iam').get_operation('CreateRole'),
self._construct_result(ec2_result),
parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
parameters = {'JobFlowId': parsed_args.cluster_id}
parameters['InstanceGroups'] = \
instancegroupsutils.build_instance_groups(
parsed_args.instance_groups)
add_instance_groups_response = emrutils.call(
self._session, 'add_instance_groups', parameters, self.region,
parsed_globals.endpoint_url, parsed_globals.verify_ssl)
constructed_result = self._construct_result(
add_instance_groups_response)
emrutils.display_response(self._session, 'add_instance_groups',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
ec2_result = None
emr_result = None
self.iam = self._session.get_service('iam')
self.iam_endpoint_url = parsed_args.iam_endpoint
region = self._get_region(parsed_globals)
self._check_for_iam_endpoint(region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.get_service('emr').get_endpoint(
region_name=parsed_globals.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).host
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Check if the default EC2 Role for EMR exists.
role_name = EC2_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role ' + role_name)
ec2_result = self._create_role_with_role_policy(
role_name, role_name, constants.EC2,
emrutils.dict_to_string(EC2_ROLE_POLICY), parsed_globals)
# Check if the default EC2 Instance Profile for EMR exists.
instance_profile_name = EC2_ROLE_NAME
if self._check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
emrutils.display_response(
self._session,
self._session.get_service('iam').get_operation('CreateRole'),
self._construct_result(ec2_result), parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
parameters = {'ClusterId': parsed_args.cluster_id}
list_instance_fleets_result = None
list_instance_groups_result = None
is_fleet_based_cluster = False
describe_cluster_result = self._call(self._session, 'describe_cluster',
parameters, parsed_globals)
if 'Cluster' in describe_cluster_result:
describe_cluster = describe_cluster_result['Cluster']
if describe_cluster.get(
'InstanceCollectionType') == constants.INSTANCE_FLEET_TYPE:
is_fleet_based_cluster = True
if 'Ec2InstanceAttributes' in describe_cluster:
ec2_instance_attr_keys = \
describe_cluster['Ec2InstanceAttributes'].keys()
ec2_instance_attr = \
describe_cluster['Ec2InstanceAttributes']
else:
ec2_instance_attr_keys = {}
if is_fleet_based_cluster:
list_instance_fleets_result = self._call(self._session,
'list_instance_fleets',
parameters,
parsed_globals)
else:
list_instance_groups_result = self._call(self._session,
'list_instance_groups',
parameters,
parsed_globals)
list_bootstrap_actions_result = self._call(self._session,
'list_bootstrap_actions',
parameters, parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result, list_instance_fleets_result,
list_instance_groups_result, list_bootstrap_actions_result)
emrutils.display_response(self._session, 'describe_cluster',
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
parameters = {'JobFlowId': parsed_args.cluster_id}
parameters['InstanceGroups'] = \
instancegroupsutils.build_instance_groups(
parsed_args.instance_groups)
add_instance_groups_response = emrutils.call(
self._session, 'add_instance_groups', parameters,
self.region, parsed_globals.endpoint_url,
parsed_globals.verify_ssl)
constructed_result = self._construct_result(
add_instance_groups_response)
emrutils.display_response(self._session, 'add_instance_groups',
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
parameters = {'ClusterId': parsed_args.cluster_id}
list_instance_fleets_result = None
list_instance_groups_result = None
is_fleet_based_cluster = False
describe_cluster_result = self._call(
self._session, 'describe_cluster', parameters, parsed_globals)
if 'Cluster' in describe_cluster_result:
describe_cluster = describe_cluster_result['Cluster']
if describe_cluster.get('InstanceCollectionType') == constants.INSTANCE_FLEET_TYPE:
is_fleet_based_cluster = True
if 'Ec2InstanceAttributes' in describe_cluster:
ec2_instance_attr_keys = \
describe_cluster['Ec2InstanceAttributes'].keys()
ec2_instance_attr = \
describe_cluster['Ec2InstanceAttributes']
else:
ec2_instance_attr_keys = {}
if is_fleet_based_cluster:
list_instance_fleets_result = self._call(
self._session, 'list_instance_fleets', parameters,
parsed_globals)
else:
list_instance_groups_result = self._call(
self._session, 'list_instance_groups', parameters,
parsed_globals)
list_bootstrap_actions_result = self._call(
self._session, 'list_bootstrap_actions',
parameters, parsed_globals)
constructed_result = self._construct_result(
describe_cluster_result,
list_instance_fleets_result,
list_instance_groups_result,
list_bootstrap_actions_result)
emrutils.display_response(self._session, 'describe_cluster',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service('emr')
add_instance_groups = emr.get_operation('AddInstanceGroups')
parameters = {'JobFlowId': parsed_args.cluster_id}
parameters['InstanceGroups'] = \
instancegroupsutils.build_instance_groups(
parsed_args.instance_groups)
add_instance_groups_response = emrutils.call(
self._session, add_instance_groups, parameters,
parsed_globals.region, parsed_globals.endpoint_url,
parsed_globals.verify_ssl)
constructed_result = self._construct_result(
add_instance_groups_response[1])
emrutils.display_response(self._session, add_instance_groups,
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service("emr")
add_instance_groups = emr.get_operation("AddInstanceGroups")
parameters = {"JobFlowId": parsed_args.cluster_id}
parameters["InstanceGroups"] = instancegroupsutils.build_instance_groups(parsed_args.instance_groups)
add_instance_groups_response = emrutils.call(
self._session,
add_instance_groups,
parameters,
parsed_globals.region,
parsed_globals.endpoint_url,
parsed_globals.verify_ssl,
)
constructed_result = self._construct_result(add_instance_groups_response[1])
emrutils.display_response(self._session, add_instance_groups, constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
params = {}
params['Name'] = parsed_args.name
self._validate_release_label_ami_version(parsed_args)
service_role_validation_message = (
" Either choose --use-default-roles or use both --service-role "
"<roleName> and --ec2-attributes InstanceProfile=<profileName>.")
if parsed_args.use_default_roles is True and \
parsed_args.service_role is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--service-role",
message=service_role_validation_message)
if parsed_args.use_default_roles is True and \
parsed_args.ec2_attributes is not None and \
'InstanceProfile' in parsed_args.ec2_attributes:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--ec2-attributes InstanceProfile",
message=service_role_validation_message)
if parsed_args.instance_groups is not None and \
parsed_args.instance_fleets is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--instance-groups",
option2="--instance-fleets")
instances_config = {}
if parsed_args.instance_fleets is not None:
instances_config['InstanceFleets'] = \
instancefleetsutils.validate_and_build_instance_fleets(
parsed_args.instance_fleets)
else:
instances_config['InstanceGroups'] = \
instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count)
if parsed_args.release_label is not None:
params["ReleaseLabel"] = parsed_args.release_label
if parsed_args.configurations is not None:
try:
params["Configurations"] = json.loads(
parsed_args.configurations)
except ValueError:
raise ValueError('aws: error: invalid json argument for '
'option --configurations')
if (parsed_args.release_label is None and
parsed_args.ami_version is not None):
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(
ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(
params, 'AdditionalInfo', parsed_args.additional_info)
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
if parsed_args.log_encryption_kms_key_id is not None:
emrutils.apply_dict(params, 'LogEncryptionKmsKeyId',
parsed_args.log_encryption_kms_key_id)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
if parsed_args.instance_groups is not None:
for instance_group in instances_config['InstanceGroups']:
if 'AutoScalingPolicy' in instance_group.keys():
if parsed_args.auto_scaling_role is None:
raise exceptions.MissingAutoScalingRoleError()
emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role)
if parsed_args.scale_down_behavior is not None:
emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior)
if (
parsed_args.no_auto_terminate is False and
parsed_args.auto_terminate is False):
parsed_args.no_auto_terminate = True
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
parsed_args.no_auto_terminate,
'--no-auto-terminate',
parsed_args.auto_terminate,
'--auto-terminate')
instances_config['TerminationProtected'] = \
emrutils.apply_boolean_options(
parsed_args.termination_protected,
'--termination-protected',
parsed_args.no_termination_protected,
'--no-termination-protected')
if (parsed_args.visible_to_all_users is False and
parsed_args.no_visible_to_all_users is False):
parsed_args.visible_to_all_users = True
params['VisibleToAllUsers'] = \
emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
'--visible-to-all-users',
parsed_args.no_visible_to_all_users,
'--no-visible-to-all-users')
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
params['Instances'] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(
cluster=params, parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging,
'--enable-debugging',
parsed_args.no_enable_debugging,
'--no-enable-debugging')
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(
cluster=params,
key='Steps',
value=[
self._build_enable_debugging(parsed_args, parsed_globals)])
if parsed_args.applications is not None:
if parsed_args.release_label is None:
app_list, ba_list, step_list = \
applicationutils.build_applications(
region=self.region,
parsed_applications=parsed_args.applications,
ami_version=params['AmiVersion'])
self._update_cluster_dict(
params, 'NewSupportedProducts', app_list)
self._update_cluster_dict(
params, 'BootstrapActions', ba_list)
self._update_cluster_dict(
params, 'Steps', step_list)
else:
params["Applications"] = []
for application in parsed_args.applications:
params["Applications"].append(application)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get('Dir'),
backup_version=hbase_restore_config.get('BackupVersion'))
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
self._update_cluster_dict(
params, 'Steps', [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(
cluster=params,
parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.emrfs is not None:
self._handle_emrfs_parameters(
cluster=params,
emrfs_args=parsed_args.emrfs,
release_label=parsed_args.release_label)
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps,
region=self.region,
release_label=parsed_args.release_label)
self._update_cluster_dict(
cluster=params, key='Steps', value=steps_list)
if parsed_args.security_configuration is not None:
emrutils.apply_dict(
params, 'SecurityConfiguration', parsed_args.security_configuration)
if parsed_args.custom_ami_id is not None:
emrutils.apply_dict(
params, 'CustomAmiId', parsed_args.custom_ami_id
)
if parsed_args.ebs_root_volume_size is not None:
emrutils.apply_dict(
params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size)
)
if parsed_args.repo_upgrade_on_boot is not None:
emrutils.apply_dict(
params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot
)
if parsed_args.kerberos_attributes is not None:
emrutils.apply_dict(
params, 'KerberosAttributes', parsed_args.kerberos_attributes)
if parsed_args.step_concurrency_level is not None:
params['StepConcurrencyLevel'] = parsed_args.step_concurrency_level
if parsed_args.managed_scaling_policy is not None:
emrutils.apply_dict(
params, 'ManagedScalingPolicy', parsed_args.managed_scaling_policy)
if parsed_args.placement_group_configs is not None:
emrutils.apply_dict(
params, 'PlacementGroupConfigs',
parsed_args.placement_group_configs)
self._validate_required_applications(parsed_args)
run_job_flow_response = emrutils.call(
self._session, 'run_job_flow', params, self.region,
parsed_globals.endpoint_url, parsed_globals.verify_ssl)
constructed_result = self._construct_result(run_job_flow_response)
emrutils.display_response(self._session, 'run_job_flow',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service('emr')
params = {}
bootstrap_actions = []
params['Name'] = parsed_args.name
instances_config = {}
instances_config['InstanceGroups'] = \
instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count)
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(
ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(params, 'AdditionalInfo',
parsed_args.additional_info)
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
if (parsed_args.no_auto_terminate is False
and parsed_args.auto_terminate is False):
parsed_args.no_auto_terminate = True
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
parsed_args.no_auto_terminate,
'--no-auto-terminate',
parsed_args.auto_terminate,
'--auto-terminate')
instances_config['TerminationProtected'] = \
emrutils.apply_boolean_options(
parsed_args.termination_protected,
'--termination-protected',
parsed_args.no_termination_protected,
'--no-termination-protected')
if (parsed_args.visible_to_all_users is False
and parsed_args.no_visible_to_all_users is False):
parsed_args.visible_to_all_users = True
params['VisibleToAllUsers'] = \
emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
'--visible-to-all-users',
parsed_args.no_visible_to_all_users,
'--no-visible-to-all-users')
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
params['Instances'] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(cluster=params,
parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging, '--enable-debugging',
parsed_args.no_enable_debugging, '--no-enable-debugging')
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(
cluster=params,
key='Steps',
value=[self._build_enable_debugging(parsed_globals)])
if parsed_args.applications is not None:
app_list, ba_list, step_list = applicationutils.build_applications(
session=self._session,
parsed_applications=parsed_args.applications,
parsed_globals=parsed_globals,
ami_version=params['AmiVersion'])
self._update_cluster_dict(params, 'NewSupportedProducts', app_list)
self._update_cluster_dict(params, 'BootstrapActions', ba_list)
self._update_cluster_dict(params, 'Steps', step_list)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get('Dir'),
backup_version=hbase_restore_config.get('BackupVersion'))
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
self._update_cluster_dict(params, 'Steps', [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(
cluster=params,
parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.emrfs is not None:
emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs)
emr_fs_ba_config = \
emrutils.build_bootstrap_action(
path=emrutils.build_s3_link(
relative_path=constants.CONFIG_HADOOP_PATH,
region=parsed_globals.region),
name=constants.EMR_FS_BA_NAME,
args=emr_fs_ba_args)
self._update_cluster_dict(cluster=params,
key='BootstrapActions',
value=[emr_fs_ba_config])
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps,
region=parsed_globals.region)
self._update_cluster_dict(cluster=params,
key='Steps',
value=steps_list)
self._validate_required_applications(parsed_args)
run_job_flow = emr.get_operation('RunJobFlow')
run_job_flow_response = emrutils.call(self._session, run_job_flow,
params, parsed_globals.region,
parsed_globals.endpoint_url,
parsed_globals.verify_ssl)
constructed_result = self._construct_result(run_job_flow_response[1])
emrutils.display_response(self._session, run_job_flow,
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
params = {}
bootstrap_actions = []
params['Name'] = parsed_args.name
service_role_validation_message = (
" Either choose --use-default-roles or use both --service-role "
"<roleName> and --ec2-attributes InstanceProfile=<profileName>.")
if parsed_args.use_default_roles is True and \
parsed_args.service_role is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--service-role",
message=service_role_validation_message)
if parsed_args.use_default_roles is True and \
parsed_args.ec2_attributes is not None and \
'InstanceProfile' in parsed_args.ec2_attributes:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--ec2-attributes InstanceProfile",
message=service_role_validation_message)
instances_config = {}
instances_config['InstanceGroups'] = \
instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count)
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(
ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(
params, 'AdditionalInfo', parsed_args.additional_info)
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
if (
parsed_args.no_auto_terminate is False and
parsed_args.auto_terminate is False):
parsed_args.no_auto_terminate = True
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
parsed_args.no_auto_terminate,
'--no-auto-terminate',
parsed_args.auto_terminate,
'--auto-terminate')
instances_config['TerminationProtected'] = \
emrutils.apply_boolean_options(
parsed_args.termination_protected,
'--termination-protected',
parsed_args.no_termination_protected,
'--no-termination-protected')
if (
parsed_args.visible_to_all_users is False and
parsed_args.no_visible_to_all_users is False):
parsed_args.visible_to_all_users = True
params['VisibleToAllUsers'] = \
emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
'--visible-to-all-users',
parsed_args.no_visible_to_all_users,
'--no-visible-to-all-users')
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
params['Instances'] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(
cluster=params, parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging,
'--enable-debugging',
parsed_args.no_enable_debugging,
'--no-enable-debugging')
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(
cluster=params,
key='Steps',
value=[self._build_enable_debugging(parsed_globals)])
if parsed_args.applications is not None:
app_list, ba_list, step_list = applicationutils.build_applications(
region=self.region,
parsed_applications=parsed_args.applications,
ami_version=params['AmiVersion'])
self._update_cluster_dict(
params, 'NewSupportedProducts', app_list)
self._update_cluster_dict(
params, 'BootstrapActions', ba_list)
self._update_cluster_dict(
params, 'Steps', step_list)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get('Dir'),
backup_version=hbase_restore_config.get('BackupVersion'))
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
self._update_cluster_dict(
params, 'Steps', [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(
cluster=params,
parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.emrfs is not None:
emr_fs_ba_config_list = emrfsutils.build_bootstrap_action_configs(
self.region, parsed_args.emrfs)
self._update_cluster_dict(
cluster=params, key='BootstrapActions',
value=emr_fs_ba_config_list)
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps,
region=self.region)
self._update_cluster_dict(
cluster=params, key='Steps', value=steps_list)
self._validate_required_applications(parsed_args)
run_job_flow_response = emrutils.call(
self._session, 'run_job_flow', params, self.region,
parsed_globals.endpoint_url, parsed_globals.verify_ssl)
constructed_result = self._construct_result(run_job_flow_response)
emrutils.display_response(self._session, 'run_job_flow',
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service('emr')
params = {}
bootstrap_actions = []
params['Name'] = parsed_args.name
is_valid_ami = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami is None:
raise exceptions.\
InvalidAmiVersionError(ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(
params, 'AdditionalInfo', parsed_args.additional_info)
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
instances_config = {}
instances_config['InstanceGroups'] = \
instancegroupsutils.build_instance_groups(
parsed_args.instance_groups)
if (
parsed_args.no_auto_terminate is False and
parsed_args.auto_terminate is False):
raise exceptions.\
MissingBooleanOptionsError(
true_option='--auto-terminate',
false_option='--no-auto-terminate')
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
parsed_args.no_auto_terminate,
'--no-auto-terminate',
parsed_args.auto_terminate,
'--auto-terminate')
instances_config['TerminationProtected'] = \
emrutils.apply_boolean_options(
parsed_args.termination_protected,
'--termination-protected',
parsed_args.no_termination_protected,
'--no-termination-protected')
params['VisibleToAllUsers'] = \
emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
'--visible-to-all-users',
parsed_args.no_visible_to_all_users,
'--no-visible-to-all-users')
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
params['Instances'] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(
cluster=params, parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging,
'--enable-debugging',
parsed_args.no_enable_debugging,
'--no-enable-debugging')
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(
cluster=params,
key='Steps',
value=[self._build_enable_debugging(parsed_globals)])
if parsed_args.applications is not None:
app_list, ba_list, step_list = applicationutils.build_applications(
parsed_applications=parsed_args.applications,
parsed_globals=parsed_globals,
ami_version=params['AmiVersion'])
self._update_cluster_dict(
params, 'NewSupportedProducts', app_list)
self._update_cluster_dict(
params, 'BootstrapActions', ba_list)
self._update_cluster_dict(
params, 'Steps', step_list)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get('Dir'),
backup_version=hbase_restore_config.get('BackupVersion'))
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
self._update_cluster_dict(
params, 'Steps', [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(
cluster=params,
parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps,
region=parsed_globals.region)
self._update_cluster_dict(
cluster=params, key='Steps', value=steps_list)
self._validate_required_applications(parsed_args)
run_job_flow = emr.get_operation('RunJobFlow')
run_job_flow_response = emrutils.call(
self._session, run_job_flow, params,
parsed_globals.region, parsed_globals.endpoint_url,
parsed_globals.verify_ssl)
constructed_result = self._construct_result(run_job_flow_response[1])
emrutils.display_response(self._session, run_job_flow,
constructed_result, parsed_globals)
return 0
def _run_main(self, parsed_args, parsed_globals):
emr = self._session.get_service("emr")
params = {}
bootstrap_actions = []
params["Name"] = parsed_args.name
service_role_validation_message = (
" Either choose --use-default-roles or use both --service-role "
"<roleName> and --ec2-attributes InstanceProfile=<profileName>."
)
if parsed_args.use_default_roles is True and parsed_args.service_role is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles", option2="--service-role", message=service_role_validation_message
)
if (
parsed_args.use_default_roles is True
and parsed_args.ec2_attributes is not None
and "InstanceProfile" in parsed_args.ec2_attributes
):
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--ec2-attributes InstanceProfile",
message=service_role_validation_message,
)
instances_config = {}
instances_config["InstanceGroups"] = instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count,
)
is_valid_ami_version = re.match("\d?\..*", parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(ami_version=parsed_args.ami_version)
params["AmiVersion"] = parsed_args.ami_version
emrutils.apply_dict(params, "AdditionalInfo", parsed_args.additional_info)
emrutils.apply_dict(params, "LogUri", parsed_args.log_uri)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes["InstanceProfile"] = EC2_ROLE_NAME
emrutils.apply_dict(params, "ServiceRole", parsed_args.service_role)
if parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False:
parsed_args.no_auto_terminate = True
instances_config["KeepJobFlowAliveWhenNoSteps"] = emrutils.apply_boolean_options(
parsed_args.no_auto_terminate, "--no-auto-terminate", parsed_args.auto_terminate, "--auto-terminate"
)
instances_config["TerminationProtected"] = emrutils.apply_boolean_options(
parsed_args.termination_protected,
"--termination-protected",
parsed_args.no_termination_protected,
"--no-termination-protected",
)
if parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False:
parsed_args.visible_to_all_users = True
params["VisibleToAllUsers"] = emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
"--visible-to-all-users",
parsed_args.no_visible_to_all_users,
"--no-visible-to-all-users",
)
params["Tags"] = emrutils.parse_tags(parsed_args.tags)
params["Instances"] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(cluster=params, parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging, "--enable-debugging", parsed_args.no_enable_debugging, "--no-enable-debugging"
)
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(cluster=params, key="Steps", value=[self._build_enable_debugging(parsed_globals)])
if parsed_args.applications is not None:
app_list, ba_list, step_list = applicationutils.build_applications(
session=self._session,
parsed_applications=parsed_args.applications,
parsed_globals=parsed_globals,
ami_version=params["AmiVersion"],
)
self._update_cluster_dict(params, "NewSupportedProducts", app_list)
self._update_cluster_dict(params, "BootstrapActions", ba_list)
self._update_cluster_dict(params, "Steps", step_list)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get("Dir"), backup_version=hbase_restore_config.get("BackupVersion")
)
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args,
)
self._update_cluster_dict(params, "Steps", [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.emrfs is not None:
emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs)
emr_fs_ba_config = emrutils.build_bootstrap_action(
path=emrutils.build_s3_link(relative_path=constants.CONFIG_HADOOP_PATH, region=parsed_globals.region),
name=constants.EMR_FS_BA_NAME,
args=emr_fs_ba_args,
)
self._update_cluster_dict(cluster=params, key="BootstrapActions", value=[emr_fs_ba_config])
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps, region=parsed_globals.region
)
self._update_cluster_dict(cluster=params, key="Steps", value=steps_list)
self._validate_required_applications(parsed_args)
run_job_flow = emr.get_operation("RunJobFlow")
run_job_flow_response = emrutils.call(
self._session,
run_job_flow,
params,
parsed_globals.region,
parsed_globals.endpoint_url,
parsed_globals.verify_ssl,
)
constructed_result = self._construct_result(run_job_flow_response[1])
emrutils.display_response(self._session, run_job_flow, constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
params = {}
params['Name'] = parsed_args.name
self._validate_release_label_ami_version(parsed_args)
service_role_validation_message = (
" Either choose --use-default-roles or use both --service-role "
"<roleName> and --ec2-attributes InstanceProfile=<profileName>.")
if parsed_args.use_default_roles is True and \
parsed_args.service_role is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--service-role",
message=service_role_validation_message)
if parsed_args.use_default_roles is True and \
parsed_args.ec2_attributes is not None and \
'InstanceProfile' in parsed_args.ec2_attributes:
raise exceptions.MutualExclusiveOptionError(
option1="--use-default-roles",
option2="--ec2-attributes InstanceProfile",
message=service_role_validation_message)
if parsed_args.instance_groups is not None and \
parsed_args.instance_fleets is not None:
raise exceptions.MutualExclusiveOptionError(
option1="--instance-groups",
option2="--instance-fleets")
instances_config = {}
if parsed_args.instance_fleets is not None:
instances_config['InstanceFleets'] = \
instancefleetsutils.validate_and_build_instance_fleets(
parsed_args.instance_fleets)
else:
instances_config['InstanceGroups'] = \
instancegroupsutils.validate_and_build_instance_groups(
instance_groups=parsed_args.instance_groups,
instance_type=parsed_args.instance_type,
instance_count=parsed_args.instance_count)
if parsed_args.release_label is not None:
params["ReleaseLabel"] = parsed_args.release_label
if parsed_args.configurations is not None:
try:
params["Configurations"] = json.loads(
parsed_args.configurations)
except ValueError:
raise ValueError('aws: error: invalid json argument for '
'option --configurations')
if (parsed_args.release_label is None and
parsed_args.ami_version is not None):
is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
if is_valid_ami_version is None:
raise exceptions.InvalidAmiVersionError(
ami_version=parsed_args.ami_version)
params['AmiVersion'] = parsed_args.ami_version
emrutils.apply_dict(
params, 'AdditionalInfo', parsed_args.additional_info)
emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
if parsed_args.use_default_roles is True:
parsed_args.service_role = EMR_ROLE_NAME
if parsed_args.ec2_attributes is None:
parsed_args.ec2_attributes = {}
parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME
emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
if parsed_args.instance_groups is not None:
for instance_group in instances_config['InstanceGroups']:
if 'AutoScalingPolicy' in instance_group.keys():
if parsed_args.auto_scaling_role is None:
raise exceptions.MissingAutoScalingRoleError()
emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role)
if parsed_args.scale_down_behavior is not None:
emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior)
if (
parsed_args.no_auto_terminate is False and
parsed_args.auto_terminate is False):
parsed_args.no_auto_terminate = True
instances_config['KeepJobFlowAliveWhenNoSteps'] = \
emrutils.apply_boolean_options(
parsed_args.no_auto_terminate,
'--no-auto-terminate',
parsed_args.auto_terminate,
'--auto-terminate')
instances_config['TerminationProtected'] = \
emrutils.apply_boolean_options(
parsed_args.termination_protected,
'--termination-protected',
parsed_args.no_termination_protected,
'--no-termination-protected')
if (parsed_args.visible_to_all_users is False and
parsed_args.no_visible_to_all_users is False):
parsed_args.visible_to_all_users = True
params['VisibleToAllUsers'] = \
emrutils.apply_boolean_options(
parsed_args.visible_to_all_users,
'--visible-to-all-users',
parsed_args.no_visible_to_all_users,
'--no-visible-to-all-users')
params['Tags'] = emrutils.parse_tags(parsed_args.tags)
params['Instances'] = instances_config
if parsed_args.ec2_attributes is not None:
self._build_ec2_attributes(
cluster=params, parsed_attrs=parsed_args.ec2_attributes)
debugging_enabled = emrutils.apply_boolean_options(
parsed_args.enable_debugging,
'--enable-debugging',
parsed_args.no_enable_debugging,
'--no-enable-debugging')
if parsed_args.log_uri is None and debugging_enabled is True:
raise exceptions.LogUriError
if debugging_enabled is True:
self._update_cluster_dict(
cluster=params,
key='Steps',
value=[
self._build_enable_debugging(parsed_args, parsed_globals)])
if parsed_args.applications is not None:
if parsed_args.release_label is None:
app_list, ba_list, step_list = \
applicationutils.build_applications(
region=self.region,
parsed_applications=parsed_args.applications,
ami_version=params['AmiVersion'])
self._update_cluster_dict(
params, 'NewSupportedProducts', app_list)
self._update_cluster_dict(
params, 'BootstrapActions', ba_list)
self._update_cluster_dict(
params, 'Steps', step_list)
else:
params["Applications"] = []
for application in parsed_args.applications:
params["Applications"].append(application)
hbase_restore_config = parsed_args.restore_from_hbase_backup
if hbase_restore_config is not None:
args = hbaseutils.build_hbase_restore_from_backup_args(
dir=hbase_restore_config.get('Dir'),
backup_version=hbase_restore_config.get('BackupVersion'))
step_config = emrutils.build_step(
jar=constants.HBASE_JAR_PATH,
name=constants.HBASE_RESTORE_STEP_NAME,
action_on_failure=constants.CANCEL_AND_WAIT,
args=args)
self._update_cluster_dict(
params, 'Steps', [step_config])
if parsed_args.bootstrap_actions is not None:
self._build_bootstrap_actions(
cluster=params,
parsed_boostrap_actions=parsed_args.bootstrap_actions)
if parsed_args.emrfs is not None:
self._handle_emrfs_parameters(
cluster=params,
emrfs_args=parsed_args.emrfs,
release_label=parsed_args.release_label)
if parsed_args.steps is not None:
steps_list = steputils.build_step_config_list(
parsed_step_list=parsed_args.steps,
region=self.region,
release_label=parsed_args.release_label)
self._update_cluster_dict(
cluster=params, key='Steps', value=steps_list)
if parsed_args.security_configuration is not None:
emrutils.apply_dict(
params, 'SecurityConfiguration', parsed_args.security_configuration)
if parsed_args.custom_ami_id is not None:
emrutils.apply_dict(
params, 'CustomAmiId', parsed_args.custom_ami_id
)
if parsed_args.ebs_root_volume_size is not None:
emrutils.apply_dict(
params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size)
)
if parsed_args.repo_upgrade_on_boot is not None:
emrutils.apply_dict(
params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot
)
self._validate_required_applications(parsed_args)
run_job_flow_response = emrutils.call(
self._session, 'run_job_flow', params, self.region,
parsed_globals.endpoint_url, parsed_globals.verify_ssl)
constructed_result = self._construct_result(run_job_flow_response)
emrutils.display_response(self._session, 'run_job_flow',
constructed_result, parsed_globals)
return 0
def _run_main_command(self, parsed_args, parsed_globals):
ec2_result = None
emr_result = None
self.iam_endpoint_url = parsed_args.iam_endpoint
self._check_for_iam_endpoint(self.region, self.iam_endpoint_url)
self.emr_endpoint_url = \
self._session.create_client(
'emr',
region_name=self.region,
endpoint_url=parsed_globals.endpoint_url,
verify=parsed_globals.verify_ssl).meta.endpoint_url
LOG.debug('elasticmapreduce endpoint used for resolving'
' service principal: ' + self.emr_endpoint_url)
# Check if the default EC2 Role for EMR exists.
role_name = EC2_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role for EC2: ' + role_name)
ec2_result = self._create_role_with_role_policy(
role_name, role_name, EC2,
emrutils.dict_to_string(EC2_ROLE_POLICY),
parsed_globals)
# Check if the default EC2 Instance Profile for EMR exists.
instance_profile_name = EC2_ROLE_NAME
if self._check_if_instance_profile_exists(instance_profile_name,
parsed_globals):
LOG.debug('Instance Profile ' + instance_profile_name + ' exists.')
else:
LOG.debug('Instance Profile ' + instance_profile_name +
'does not exist. Creating default Instance Profile ' +
instance_profile_name)
self._create_instance_profile_with_role(instance_profile_name,
instance_profile_name,
parsed_globals)
# Check if the default EMR Role exists.
role_name = EMR_ROLE_NAME
if self._check_if_role_exists(role_name, parsed_globals):
LOG.debug('Role ' + role_name + ' exists.')
else:
LOG.debug('Role ' + role_name + ' does not exist.'
' Creating default role for EMR: ' + role_name)
emr_result = self._create_role_with_role_policy(
role_name, role_name, EMR,
emrutils.dict_to_string(EMR_ROLE_POLICY),
parsed_globals)
configutils.update_roles(self._session)
emrutils.display_response(
self._session,
'create_role',
self._construct_result(ec2_result, emr_result),
parsed_globals)
return 0
