Example #1
0
def build_hive_step(parsed_step, region=None):
    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.HIVE_STEP_CONFIG, name='Args', value=args)
    emrutils.check_empty_string_list(name='Args', value=args)
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_HIVE_STEP_NAME)
    action_on_failure = \
        _apply_default_value(
            arg=parsed_step.get('ActionOnFailure'),
            value=constants.DEFAULT_FAILURE_ACTION)
    args_list = [
        emrutils.build_s3_link(
            relative_path=constants.HIVE_SCRIPT_PATH, region=region),
        constants.RUN_HIVE_SCRIPT]
    args_list.append(constants.HIVE_VERSIONS)
    args_list.append(constants.LATEST)
    args_list.append(constants.ARGS)
    args_list += args

    return emrutils.build_step(
        jar=emrutils.get_script_runner(region),
        args=args_list,
        name=name,
        action_on_failure=action_on_failure)
Example #2
0
def build_pig_step(parsed_step, region=None):
    version = parsed_step.get('Version')
    args = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.PIG_STEP_CONFIG,
                                  name='Args',
                                  value=args)

    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_PIG_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args_list = [
        emrutils.build_s3_link(relative_path=constants.PIG_SCRIPT_PATH,
                               region=region), constants.RUN_PIG_SCRIPT
    ]
    args_list.append(constants.PIG_VERSIONS)
    if version is not None:
        args_list.append(version)
    else:
        args_list.append(constants.LATEST)
    args_list.append(constants.ARGS)
    args_list += args

    return emrutils.build_step(jar=emrutils.get_script_runner(region),
                               args=args_list,
                               name=name,
                               action_on_failure=action_on_failure)
Example #3
0
def build_streaming_step(parsed_step, release_label):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.STREAMING_STEP_CONFIG,
        name='Args',
        value=args)
    emrutils.check_empty_string_list(name='Args', value=args)
    args_list = []

    if release_label:
        jar = constants.COMMAND_RUNNER
        args_list.append(constants.HADOOP_STREAMING_COMMAND)
    else:
        jar = constants.HADOOP_STREAMING_PATH

    args_list += args

    return emrutils.build_step(
        jar=jar,
        args=args_list,
        name=name,
        action_on_failure=action_on_failure)
Example #4
0
 def _build_enable_debugging(self, parsed_globals):
     return emrutils.build_step(
         name=constants.DEBUGGING_NAME,
         action_on_failure=constants.TERMINATE_CLUSTER,
         jar=emrutils.get_script_runner(parsed_globals.region),
         args=[emrutils.build_s3_link(relative_path=constants.DEBUGGING_PATH, region=parsed_globals.region)],
     )
def build_streaming_step(parsed_step, release_label):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG,
                                  name='Args',
                                  value=args)
    emrutils.check_empty_string_list(name='Args', value=args)
    args_list = []

    if release_label:
        jar = constants.COMMAND_RUNNER
        args_list.append(constants.HADOOP_STREAMING_COMMAND)
    else:
        jar = constants.HADOOP_STREAMING_PATH

    args_list += args

    return emrutils.build_step(jar=jar,
                               args=args_list,
                               name=name,
                               action_on_failure=action_on_failure)
 def _build_enable_debugging(self, parsed_globals):
     return emrutils.build_step(
         name=constants.DEBUGGING_NAME,
         action_on_failure=constants.TERMINATE_CLUSTER,
         jar=emrutils.get_script_runner(),
         args=[emrutils.build_s3_link(
             relative_path=constants.DEBUGGING_PATH,
             region=parsed_globals.region)])
Example #7
0
def build_streaming_step(parsed_step):
    name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION
    )

    args_list = parsed_step.get("Args")
    emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG, name="Args", value=args_list)
    return emrutils.build_step(
        jar=constants.STREAMING_JAR_PATH, args=args_list, name=name, action_on_failure=action_on_failure
    )
Example #8
0
def _build_pig_install_step(region,
                            action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.PIG_SCRIPT_PATH, region),
        constants.INSTALL_PIG_ARG, constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.PIG_BASE_PATH, region),
        constants.PIG_VERSIONS, constants.LATEST
    ]
    step = emrutils.build_step(name=constants.INSTALL_PIG_NAME,
                               action_on_failure=action_on_failure,
                               jar=emrutils.build_s3_link(
                                   constants.SCRIPT_RUNNER_PATH, region),
                               args=step_args)
    return step
Example #9
0
def build_custom_jar_step(parsed_step):
    name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION
    )
    emrutils.check_required_field(structure=constants.CUSTOM_JAR_STEP_CONFIG, name="Jar", value=parsed_step.get("Jar"))
    return emrutils.build_step(
        jar=parsed_step.get("Jar"),
        args=parsed_step.get("Args"),
        name=name,
        action_on_failure=action_on_failure,
        main_class=parsed_step.get("MainClass"),
        properties=emrutils.parse_key_value_string(parsed_step.get("Properties")),
    )
Example #10
0
def _build_install_hive_step(region, version,
                             action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.INSTALL_HIVE_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH),
        constants.HIVE_VERSIONS,
        version]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step
Example #11
0
def build_streaming_step(parsed_step):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args_list = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG,
                                  name='Args',
                                  value=args_list)
    return emrutils.build_step(jar=constants.STREAMING_JAR_PATH,
                               args=args_list,
                               name=name,
                               action_on_failure=action_on_failure)
Example #12
0
    def _run_main_command(self, parsed_args, parsed_globals):
        steps = []

        args = self._build_hbase_disable_backups_args(parsed_args)

        step_config = emrutils.build_step(
            constants.HBASE_JAR_PATH,
            constants.HBASE_SCHEDULE_BACKUP_STEP_NAME,
            constants.CANCEL_AND_WAIT, args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id, 'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
Example #13
0
    def _build_enable_debugging(self, parsed_args, parsed_globals):
        if parsed_args.release_label:
            jar = constants.COMMAND_RUNNER
            args = [constants.DEBUGGING_COMMAND]
        else:
            jar = emrutils.get_script_runner(self.region)
            args = [emrutils.build_s3_link(
                relative_path=constants.DEBUGGING_PATH,
                region=self.region)]

        return emrutils.build_step(
            name=constants.DEBUGGING_NAME,
            action_on_failure=constants.TERMINATE_CLUSTER,
            jar=jar,
            args=args)
Example #14
0
    def _build_enable_debugging(self, parsed_args, parsed_globals):
        if parsed_args.release_label:
            jar = constants.COMMAND_RUNNER
            args = [constants.DEBUGGING_COMMAND]
        else:
            jar = emrutils.get_script_runner(self.region)
            args = [emrutils.build_s3_link(
                relative_path=constants.DEBUGGING_PATH,
                region=self.region)]

        return emrutils.build_step(
            name=constants.DEBUGGING_NAME,
            action_on_failure=constants.TERMINATE_CLUSTER,
            jar=jar,
            args=args)
Example #15
0
def _build_pig_install_step(region,
                            action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.PIG_SCRIPT_PATH, region),
        constants.INSTALL_PIG_ARG,
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.PIG_BASE_PATH, region),
        constants.PIG_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_PIG_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step
Example #16
0
def _build_install_hive_step(region,
                             version,
                             action_on_failure=constants.TERMINATE_CLUSTER):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.INSTALL_HIVE_ARG, constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH),
        constants.HIVE_VERSIONS, version
    ]
    step = emrutils.build_step(name=constants.INSTALL_HIVE_NAME,
                               action_on_failure=action_on_failure,
                               jar=emrutils.build_s3_link(
                                   constants.SCRIPT_RUNNER_PATH, region),
                               args=step_args)
    return step
Example #17
0
    def _run_main_command(self, parsed_args, parsed_globals):
        steps = []
        args = hbaseutils.build_hbase_restore_from_backup_args(
            parsed_args.dir, parsed_args.backup_version)

        step_config = emrutils.build_step(
            jar=constants.HBASE_JAR_PATH,
            name=constants.HBASE_RESTORE_STEP_NAME,
            action_on_failure=constants.CANCEL_AND_WAIT,
            args=args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id, 'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
Example #18
0
def build_spark_step(parsed_step, region):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_SPARK_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args_list = [constants.SPARK_SUBMIT_COMMAND]
    args = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.SPARK_STEP_CONFIG,
                                  name='Args',
                                  value=args)
    args_list += args
    return emrutils.build_step(jar=emrutils.get_script_runner(region),
                               args=args_list,
                               name=name,
                               action_on_failure=action_on_failure)
Example #19
0
    def _run_main_command(self, parsed_args, parsed_globals):
        steps = []
        args = self._build_hbase_backup_args(parsed_args)

        step_config = emrutils.build_step(
            jar=constants.HBASE_JAR_PATH,
            name=constants.HBASE_BACKUP_STEP_NAME,
            action_on_failure=constants.CANCEL_AND_WAIT,
            args=args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id,
                      'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
Example #20
0
def build_custom_jar_step(parsed_step):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_CUSTOM_JAR_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    emrutils.check_required_field(structure=constants.CUSTOM_JAR_STEP_CONFIG,
                                  name='Jar',
                                  value=parsed_step.get('Jar'))
    return emrutils.build_step(jar=parsed_step.get('Jar'),
                               args=parsed_step.get('Args'),
                               name=name,
                               action_on_failure=action_on_failure,
                               main_class=parsed_step.get('MainClass'),
                               properties=emrutils.parse_key_value_string(
                                   parsed_step.get('Properties')))
Example #21
0
def _build_install_hive_site_step(region, hive_site_path,
                                  action_on_failure=constants.CANCEL_AND_WAIT):
    step_args = [
        emrutils.build_s3_link(constants.HIVE_SCRIPT_PATH, region),
        constants.BASE_PATH_ARG,
        emrutils.build_s3_link(constants.HIVE_BASE_PATH),
        constants.INSTALL_HIVE_SITE_ARG,
        hive_site_path,
        constants.HIVE_VERSIONS,
        constants.LATEST]
    step = emrutils.build_step(
        name=constants.INSTALL_HIVE_SITE_NAME,
        action_on_failure=action_on_failure,
        jar=emrutils.build_s3_link(constants.SCRIPT_RUNNER_PATH, region),
        args=step_args)
    return step
def build_spark_step(parsed_step, release_label, region=None):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_SPARK_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.SPARK_STEP_CONFIG,
                                  name='Args',
                                  value=args)

    return emrutils.build_step(jar=_get_runner_jar(release_label, region),
                               args=_build_spark_args(args, release_label,
                                                      region),
                               name=name,
                               action_on_failure=action_on_failure)
Example #23
0
def build_impala_step(parsed_step, region):
    name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_IMPALA_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION
    )
    args_list = [
        emrutils.build_s3_link(relative_path=constants.IMPALA_INSTALL_PATH, region=region),
        constants.RUN_IMPALA_SCRIPT,
    ]
    args = parsed_step.get("Args")
    emrutils.check_required_field(structure=constants.IMPALA_STEP_CONFIG, name="Args", value=args)
    args_list += args

    return emrutils.build_step(
        jar=emrutils.get_script_runner(region), args=args_list, name=name, action_on_failure=action_on_failure
    )
Example #24
0
def build_spark_step(parsed_step, release_label, region=None):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_SPARK_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.SPARK_STEP_CONFIG, name='Args', value=args)

    return emrutils.build_step(
        jar=_get_runner_jar(release_label, region),
        args=_build_spark_args(args, release_label, region),
        name=name,
        action_on_failure=action_on_failure)
Example #25
0
    def _run_main_command(self, parsed_args, parsed_globals):
        steps = []
        self._check_type(parsed_args.type)
        self._check_unit(parsed_args.unit)
        args = self._build_hbase_schedule_backup_args(parsed_args)

        step_config = emrutils.build_step(
            jar=constants.HBASE_JAR_PATH,
            name=constants.HBASE_SCHEDULE_BACKUP_STEP_NAME,
            action_on_failure=constants.CANCEL_AND_WAIT,
            args=args)

        steps.append(step_config)
        parameters = {'JobFlowId': parsed_args.cluster_id, 'Steps': steps}
        emrutils.call_and_display_response(self._session, 'AddJobFlowSteps',
                                           parameters, parsed_globals)
        return 0
Example #26
0
def build_spark_step(parsed_step, region):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_SPARK_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args_list=[constants.SPARK_SUBMIT_COMMAND]
    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.SPARK_STEP_CONFIG, name='Args', value=args)
    args_list += args
    return emrutils.build_step(
        jar=emrutils.get_script_runner(region),
        args=args_list,
        name=name,
        action_on_failure=action_on_failure)
Example #27
0
def build_streaming_step(parsed_step):
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_STREAMING_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)

    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.STREAMING_STEP_CONFIG,
        name='Args',
        value=args)
    emrutils.check_empty_string_list(name='Args', value=args)
    return emrutils.build_step(
        jar=constants.STREAMING_JAR_PATH,
        args=args,
        name=name,
        action_on_failure=action_on_failure)
Example #28
0
def build_impala_step(parsed_step, region):
    name = _apply_default_value(arg=parsed_step.get('Name'),
                                value=constants.DEFAULT_IMPALA_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args_list = [
        emrutils.build_s3_link(relative_path=constants.IMPALA_INSTALL_PATH,
                               region=region), constants.RUN_IMPALA_SCRIPT
    ]
    args = parsed_step.get('Args')
    emrutils.check_required_field(structure=constants.IMPALA_STEP_CONFIG,
                                  name='Args',
                                  value=args)
    args_list += args

    return emrutils.build_step(jar=emrutils.get_script_runner(region),
                               args=args_list,
                               name=name,
                               action_on_failure=action_on_failure)
Example #29
0
def build_pig_step(parsed_step, region=None):
    args = parsed_step.get("Args")
    emrutils.check_required_field(structure=constants.PIG_STEP_CONFIG, name="Args", value=args)

    name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_PIG_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION
    )

    args_list = [
        emrutils.build_s3_link(relative_path=constants.PIG_SCRIPT_PATH, region=region),
        constants.RUN_PIG_SCRIPT,
    ]
    args_list.append(constants.PIG_VERSIONS)
    args_list.append(constants.LATEST)
    args_list.append(constants.ARGS)
    args_list += args

    return emrutils.build_step(
        jar=emrutils.get_script_runner(region), args=args_list, name=name, action_on_failure=action_on_failure
    )
Example #30
0
def build_impala_step(parsed_step, release_label, region=None):
    if release_label:
        raise exceptions.UnknownStepTypeError(step_type=constants.IMPALA)
    name = _apply_default_value(
        arg=parsed_step.get('Name'),
        value=constants.DEFAULT_IMPALA_STEP_NAME)
    action_on_failure = _apply_default_value(
        arg=parsed_step.get('ActionOnFailure'),
        value=constants.DEFAULT_FAILURE_ACTION)
    args_list = [
        emrutils.build_s3_link(
            relative_path=constants.IMPALA_INSTALL_PATH, region=region),
        constants.RUN_IMPALA_SCRIPT]
    args = parsed_step.get('Args')
    emrutils.check_required_field(
        structure=constants.IMPALA_STEP_CONFIG, name='Args', value=args)
    args_list += args

    return emrutils.build_step(
        jar=emrutils.get_script_runner(region),
        args=args_list,
        name=name,
        action_on_failure=action_on_failure)
Example #31
0
    def _run_main_command(self, parsed_args, parsed_globals):
        params = {}
        bootstrap_actions = []
        params['Name'] = parsed_args.name

        service_role_validation_message = (
            " Either choose --use-default-roles or use both --service-role "
            "<roleName> and --ec2-attributes InstanceProfile=<profileName>.")

        if parsed_args.use_default_roles is True and \
                parsed_args.service_role is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--service-role",
                message=service_role_validation_message)

        if parsed_args.use_default_roles is True and \
                parsed_args.ec2_attributes is not None and \
                'InstanceProfile' in parsed_args.ec2_attributes:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--ec2-attributes InstanceProfile",
                message=service_role_validation_message)

        instances_config = {}
        instances_config['InstanceGroups'] = \
            instancegroupsutils.validate_and_build_instance_groups(
                instance_groups=parsed_args.instance_groups,
                instance_type=parsed_args.instance_type,
                instance_count=parsed_args.instance_count)

        is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
        if is_valid_ami_version is None:
            raise exceptions.InvalidAmiVersionError(
                ami_version=parsed_args.ami_version)
        params['AmiVersion'] = parsed_args.ami_version
        emrutils.apply_dict(
            params, 'AdditionalInfo', parsed_args.additional_info)
        emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)

        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)

        if (
                parsed_args.no_auto_terminate is False and
                parsed_args.auto_terminate is False):
            parsed_args.no_auto_terminate = True

        instances_config['KeepJobFlowAliveWhenNoSteps'] = \
            emrutils.apply_boolean_options(
                parsed_args.no_auto_terminate,
                '--no-auto-terminate',
                parsed_args.auto_terminate,
                '--auto-terminate')

        instances_config['TerminationProtected'] = \
            emrutils.apply_boolean_options(
                parsed_args.termination_protected,
                '--termination-protected',
                parsed_args.no_termination_protected,
                '--no-termination-protected')

        if (
                parsed_args.visible_to_all_users is False and
                parsed_args.no_visible_to_all_users is False):
            parsed_args.visible_to_all_users = True

        params['VisibleToAllUsers'] = \
            emrutils.apply_boolean_options(
                parsed_args.visible_to_all_users,
                '--visible-to-all-users',
                parsed_args.no_visible_to_all_users,
                '--no-visible-to-all-users')

        params['Tags'] = emrutils.parse_tags(parsed_args.tags)
        params['Instances'] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(
                cluster=params, parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging,
            '--enable-debugging',
            parsed_args.no_enable_debugging,
            '--no-enable-debugging')

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(
                cluster=params,
                key='Steps',
                value=[self._build_enable_debugging(parsed_globals)])

        if parsed_args.applications is not None:
            app_list, ba_list, step_list = applicationutils.build_applications(
                region=self.region,
                parsed_applications=parsed_args.applications,
                ami_version=params['AmiVersion'])
            self._update_cluster_dict(
                params, 'NewSupportedProducts', app_list)
            self._update_cluster_dict(
                params, 'BootstrapActions', ba_list)
            self._update_cluster_dict(
                params, 'Steps', step_list)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get('Dir'),
                backup_version=hbase_restore_config.get('BackupVersion'))
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args)
            self._update_cluster_dict(
                params, 'Steps', [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(
                cluster=params,
                parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.emrfs is not None:
            emr_fs_ba_config_list = emrfsutils.build_bootstrap_action_configs(
                self.region, parsed_args.emrfs)

            self._update_cluster_dict(
                cluster=params, key='BootstrapActions',
                value=emr_fs_ba_config_list)

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps,
                region=self.region)
            self._update_cluster_dict(
                cluster=params, key='Steps', value=steps_list)

        self._validate_required_applications(parsed_args)

        run_job_flow_response = emrutils.call(
            self._session, 'run_job_flow', params, self.region,
            parsed_globals.endpoint_url, parsed_globals.verify_ssl)

        constructed_result = self._construct_result(run_job_flow_response)
        emrutils.display_response(self._session, 'run_job_flow',
                                  constructed_result, parsed_globals)

        return 0
Example #32
0
    def _run_main_command(self, parsed_args, parsed_globals):
        params = {}
        params['Name'] = parsed_args.name

        self._validate_release_label_ami_version(parsed_args)

        service_role_validation_message = (
            " Either choose --use-default-roles or use both --service-role "
            "<roleName> and --ec2-attributes InstanceProfile=<profileName>.")

        if parsed_args.use_default_roles is True and \
                parsed_args.service_role is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--service-role",
                message=service_role_validation_message)

        if parsed_args.use_default_roles is True and \
                parsed_args.ec2_attributes is not None and \
                'InstanceProfile' in parsed_args.ec2_attributes:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--ec2-attributes InstanceProfile",
                message=service_role_validation_message)

        if parsed_args.instance_groups is not None and \
                parsed_args.instance_fleets is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--instance-groups",
                option2="--instance-fleets")

        instances_config = {}
        if parsed_args.instance_fleets is not None:
            instances_config['InstanceFleets'] = \
                instancefleetsutils.validate_and_build_instance_fleets(
                    parsed_args.instance_fleets)
        else:
            instances_config['InstanceGroups'] = \
                instancegroupsutils.validate_and_build_instance_groups(
                    instance_groups=parsed_args.instance_groups,
                    instance_type=parsed_args.instance_type,
                    instance_count=parsed_args.instance_count)

        if parsed_args.release_label is not None:
            params["ReleaseLabel"] = parsed_args.release_label
            if parsed_args.configurations is not None:
                try:
                    params["Configurations"] = json.loads(
                        parsed_args.configurations)
                except ValueError:
                    raise ValueError('aws: error: invalid json argument for '
                                     'option --configurations')

        if (parsed_args.release_label is None and
                parsed_args.ami_version is not None):
            is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
            if is_valid_ami_version is None:
                raise exceptions.InvalidAmiVersionError(
                    ami_version=parsed_args.ami_version)
            params['AmiVersion'] = parsed_args.ami_version
        emrutils.apply_dict(
            params, 'AdditionalInfo', parsed_args.additional_info)
        emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)

        if parsed_args.log_encryption_kms_key_id is not None:
            emrutils.apply_dict(params, 'LogEncryptionKmsKeyId',
                parsed_args.log_encryption_kms_key_id)

        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)

        if parsed_args.instance_groups is not None:
            for instance_group in instances_config['InstanceGroups']:
                if 'AutoScalingPolicy' in instance_group.keys():
                    if parsed_args.auto_scaling_role is None:
                        raise exceptions.MissingAutoScalingRoleError()

        emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role)

        if parsed_args.scale_down_behavior is not None:
            emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior)

        if (
                parsed_args.no_auto_terminate is False and
                parsed_args.auto_terminate is False):
            parsed_args.no_auto_terminate = True

        instances_config['KeepJobFlowAliveWhenNoSteps'] = \
            emrutils.apply_boolean_options(
                parsed_args.no_auto_terminate,
                '--no-auto-terminate',
                parsed_args.auto_terminate,
                '--auto-terminate')

        instances_config['TerminationProtected'] = \
            emrutils.apply_boolean_options(
                parsed_args.termination_protected,
                '--termination-protected',
                parsed_args.no_termination_protected,
                '--no-termination-protected')

        if (parsed_args.visible_to_all_users is False and
                parsed_args.no_visible_to_all_users is False):
            parsed_args.visible_to_all_users = True

        params['VisibleToAllUsers'] = \
            emrutils.apply_boolean_options(
                parsed_args.visible_to_all_users,
                '--visible-to-all-users',
                parsed_args.no_visible_to_all_users,
                '--no-visible-to-all-users')

        params['Tags'] = emrutils.parse_tags(parsed_args.tags)
        params['Instances'] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(
                cluster=params, parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging,
            '--enable-debugging',
            parsed_args.no_enable_debugging,
            '--no-enable-debugging')

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(
                cluster=params,
                key='Steps',
                value=[
                    self._build_enable_debugging(parsed_args, parsed_globals)])

        if parsed_args.applications is not None:
            if parsed_args.release_label is None:
                app_list, ba_list, step_list = \
                    applicationutils.build_applications(
                        region=self.region,
                        parsed_applications=parsed_args.applications,
                        ami_version=params['AmiVersion'])
                self._update_cluster_dict(
                    params, 'NewSupportedProducts', app_list)
                self._update_cluster_dict(
                    params, 'BootstrapActions', ba_list)
                self._update_cluster_dict(
                    params, 'Steps', step_list)
            else:
                params["Applications"] = []
                for application in parsed_args.applications:
                    params["Applications"].append(application)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get('Dir'),
                backup_version=hbase_restore_config.get('BackupVersion'))
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args)
            self._update_cluster_dict(
                params, 'Steps', [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(
                cluster=params,
                parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.emrfs is not None:
            self._handle_emrfs_parameters(
                cluster=params,
                emrfs_args=parsed_args.emrfs,
                release_label=parsed_args.release_label)

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps,
                region=self.region,
                release_label=parsed_args.release_label)
            self._update_cluster_dict(
                cluster=params, key='Steps', value=steps_list)

        if parsed_args.security_configuration is not None:
            emrutils.apply_dict(
                params, 'SecurityConfiguration', parsed_args.security_configuration)

        if parsed_args.custom_ami_id is not None:
            emrutils.apply_dict(
                params, 'CustomAmiId', parsed_args.custom_ami_id
            )
        if parsed_args.ebs_root_volume_size is not None:
            emrutils.apply_dict(
                params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size)
            )

        if parsed_args.repo_upgrade_on_boot is not None:
            emrutils.apply_dict(
                params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot
            )

        if parsed_args.kerberos_attributes is not None:
            emrutils.apply_dict(
                params, 'KerberosAttributes', parsed_args.kerberos_attributes)

        if parsed_args.step_concurrency_level is not None:
            params['StepConcurrencyLevel'] = parsed_args.step_concurrency_level

        if parsed_args.managed_scaling_policy is not None:
            emrutils.apply_dict(
                params, 'ManagedScalingPolicy', parsed_args.managed_scaling_policy)

        if parsed_args.placement_group_configs is not None:
            emrutils.apply_dict(
                params, 'PlacementGroupConfigs',
                parsed_args.placement_group_configs)

        self._validate_required_applications(parsed_args)

        run_job_flow_response = emrutils.call(
            self._session, 'run_job_flow', params, self.region,
            parsed_globals.endpoint_url, parsed_globals.verify_ssl)

        constructed_result = self._construct_result(run_job_flow_response)
        emrutils.display_response(self._session, 'run_job_flow',
                                  constructed_result, parsed_globals)

        return 0
    def _run_main(self, parsed_args, parsed_globals):
        emr = self._session.get_service('emr')
        params = {}
        bootstrap_actions = []
        params['Name'] = parsed_args.name

        instances_config = {}
        instances_config['InstanceGroups'] = \
            instancegroupsutils.validate_and_build_instance_groups(
                instance_groups=parsed_args.instance_groups,
                instance_type=parsed_args.instance_type,
                instance_count=parsed_args.instance_count)

        is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
        if is_valid_ami_version is None:
            raise exceptions.InvalidAmiVersionError(
                ami_version=parsed_args.ami_version)
        params['AmiVersion'] = parsed_args.ami_version
        emrutils.apply_dict(params, 'AdditionalInfo',
                            parsed_args.additional_info)
        emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)

        if (parsed_args.no_auto_terminate is False
                and parsed_args.auto_terminate is False):
            parsed_args.no_auto_terminate = True

        instances_config['KeepJobFlowAliveWhenNoSteps'] = \
            emrutils.apply_boolean_options(
                parsed_args.no_auto_terminate,
                '--no-auto-terminate',
                parsed_args.auto_terminate,
                '--auto-terminate')

        instances_config['TerminationProtected'] = \
            emrutils.apply_boolean_options(
                parsed_args.termination_protected,
                '--termination-protected',
                parsed_args.no_termination_protected,
                '--no-termination-protected')

        if (parsed_args.visible_to_all_users is False
                and parsed_args.no_visible_to_all_users is False):
            parsed_args.visible_to_all_users = True

        params['VisibleToAllUsers'] = \
            emrutils.apply_boolean_options(
                parsed_args.visible_to_all_users,
                '--visible-to-all-users',
                parsed_args.no_visible_to_all_users,
                '--no-visible-to-all-users')

        params['Tags'] = emrutils.parse_tags(parsed_args.tags)
        params['Instances'] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(cluster=params,
                                       parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging, '--enable-debugging',
            parsed_args.no_enable_debugging, '--no-enable-debugging')

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(
                cluster=params,
                key='Steps',
                value=[self._build_enable_debugging(parsed_globals)])

        if parsed_args.applications is not None:
            app_list, ba_list, step_list = applicationutils.build_applications(
                session=self._session,
                parsed_applications=parsed_args.applications,
                parsed_globals=parsed_globals,
                ami_version=params['AmiVersion'])
            self._update_cluster_dict(params, 'NewSupportedProducts', app_list)
            self._update_cluster_dict(params, 'BootstrapActions', ba_list)
            self._update_cluster_dict(params, 'Steps', step_list)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get('Dir'),
                backup_version=hbase_restore_config.get('BackupVersion'))
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args)
            self._update_cluster_dict(params, 'Steps', [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(
                cluster=params,
                parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.emrfs is not None:
            emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs)
            emr_fs_ba_config = \
                emrutils.build_bootstrap_action(
                    path=emrutils.build_s3_link(
                        relative_path=constants.CONFIG_HADOOP_PATH,
                        region=parsed_globals.region),
                    name=constants.EMR_FS_BA_NAME,
                    args=emr_fs_ba_args)
            self._update_cluster_dict(cluster=params,
                                      key='BootstrapActions',
                                      value=[emr_fs_ba_config])

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps,
                region=parsed_globals.region)
            self._update_cluster_dict(cluster=params,
                                      key='Steps',
                                      value=steps_list)

        self._validate_required_applications(parsed_args)

        run_job_flow = emr.get_operation('RunJobFlow')
        run_job_flow_response = emrutils.call(self._session, run_job_flow,
                                              params, parsed_globals.region,
                                              parsed_globals.endpoint_url,
                                              parsed_globals.verify_ssl)

        constructed_result = self._construct_result(run_job_flow_response[1])
        emrutils.display_response(self._session, run_job_flow,
                                  constructed_result, parsed_globals)

        return 0
Example #34
0
    def _run_main_command(self, parsed_args, parsed_globals):
        params = {}
        params['Name'] = parsed_args.name

        self._validate_release_label_ami_version(parsed_args)

        service_role_validation_message = (
            " Either choose --use-default-roles or use both --service-role "
            "<roleName> and --ec2-attributes InstanceProfile=<profileName>.")

        if parsed_args.use_default_roles is True and \
                parsed_args.service_role is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--service-role",
                message=service_role_validation_message)

        if parsed_args.use_default_roles is True and \
                parsed_args.ec2_attributes is not None and \
                'InstanceProfile' in parsed_args.ec2_attributes:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--ec2-attributes InstanceProfile",
                message=service_role_validation_message)

        if parsed_args.instance_groups is not None and \
                parsed_args.instance_fleets is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--instance-groups",
                option2="--instance-fleets")

        instances_config = {}
        if parsed_args.instance_fleets is not None:
            instances_config['InstanceFleets'] = \
                instancefleetsutils.validate_and_build_instance_fleets(
                    parsed_args.instance_fleets)
        else:
            instances_config['InstanceGroups'] = \
                instancegroupsutils.validate_and_build_instance_groups(
                    instance_groups=parsed_args.instance_groups,
                    instance_type=parsed_args.instance_type,
                    instance_count=parsed_args.instance_count)

        if parsed_args.release_label is not None:
            params["ReleaseLabel"] = parsed_args.release_label
            if parsed_args.configurations is not None:
                try:
                    params["Configurations"] = json.loads(
                        parsed_args.configurations)
                except ValueError:
                    raise ValueError('aws: error: invalid json argument for '
                                     'option --configurations')

        if (parsed_args.release_label is None and
                parsed_args.ami_version is not None):
            is_valid_ami_version = re.match('\d?\..*', parsed_args.ami_version)
            if is_valid_ami_version is None:
                raise exceptions.InvalidAmiVersionError(
                    ami_version=parsed_args.ami_version)
            params['AmiVersion'] = parsed_args.ami_version
        emrutils.apply_dict(
            params, 'AdditionalInfo', parsed_args.additional_info)
        emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)

        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)

        if parsed_args.instance_groups is not None:
            for instance_group in instances_config['InstanceGroups']:
                if 'AutoScalingPolicy' in instance_group.keys():
                    if parsed_args.auto_scaling_role is None:
                        raise exceptions.MissingAutoScalingRoleError()

        emrutils.apply_dict(params, 'AutoScalingRole', parsed_args.auto_scaling_role)

        if parsed_args.scale_down_behavior is not None:
            emrutils.apply_dict(params, 'ScaleDownBehavior', parsed_args.scale_down_behavior)

        if (
                parsed_args.no_auto_terminate is False and
                parsed_args.auto_terminate is False):
            parsed_args.no_auto_terminate = True

        instances_config['KeepJobFlowAliveWhenNoSteps'] = \
            emrutils.apply_boolean_options(
                parsed_args.no_auto_terminate,
                '--no-auto-terminate',
                parsed_args.auto_terminate,
                '--auto-terminate')

        instances_config['TerminationProtected'] = \
            emrutils.apply_boolean_options(
                parsed_args.termination_protected,
                '--termination-protected',
                parsed_args.no_termination_protected,
                '--no-termination-protected')

        if (parsed_args.visible_to_all_users is False and
                parsed_args.no_visible_to_all_users is False):
            parsed_args.visible_to_all_users = True

        params['VisibleToAllUsers'] = \
            emrutils.apply_boolean_options(
                parsed_args.visible_to_all_users,
                '--visible-to-all-users',
                parsed_args.no_visible_to_all_users,
                '--no-visible-to-all-users')

        params['Tags'] = emrutils.parse_tags(parsed_args.tags)
        params['Instances'] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(
                cluster=params, parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging,
            '--enable-debugging',
            parsed_args.no_enable_debugging,
            '--no-enable-debugging')

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(
                cluster=params,
                key='Steps',
                value=[
                    self._build_enable_debugging(parsed_args, parsed_globals)])

        if parsed_args.applications is not None:
            if parsed_args.release_label is None:
                app_list, ba_list, step_list = \
                    applicationutils.build_applications(
                        region=self.region,
                        parsed_applications=parsed_args.applications,
                        ami_version=params['AmiVersion'])
                self._update_cluster_dict(
                    params, 'NewSupportedProducts', app_list)
                self._update_cluster_dict(
                    params, 'BootstrapActions', ba_list)
                self._update_cluster_dict(
                    params, 'Steps', step_list)
            else:
                params["Applications"] = []
                for application in parsed_args.applications:
                    params["Applications"].append(application)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get('Dir'),
                backup_version=hbase_restore_config.get('BackupVersion'))
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args)
            self._update_cluster_dict(
                params, 'Steps', [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(
                cluster=params,
                parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.emrfs is not None:
            self._handle_emrfs_parameters(
                cluster=params,
                emrfs_args=parsed_args.emrfs,
                release_label=parsed_args.release_label)

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps,
                region=self.region,
                release_label=parsed_args.release_label)
            self._update_cluster_dict(
                cluster=params, key='Steps', value=steps_list)

        if parsed_args.security_configuration is not None:
            emrutils.apply_dict(
                params, 'SecurityConfiguration', parsed_args.security_configuration)

        if parsed_args.custom_ami_id is not None:
            emrutils.apply_dict(
                params, 'CustomAmiId', parsed_args.custom_ami_id
            )
        if parsed_args.ebs_root_volume_size is not None:
            emrutils.apply_dict(
                params, 'EbsRootVolumeSize', int(parsed_args.ebs_root_volume_size)
            )

        if parsed_args.repo_upgrade_on_boot is not None:
            emrutils.apply_dict(
                params, 'RepoUpgradeOnBoot', parsed_args.repo_upgrade_on_boot
            )


        self._validate_required_applications(parsed_args)

        run_job_flow_response = emrutils.call(
            self._session, 'run_job_flow', params, self.region,
            parsed_globals.endpoint_url, parsed_globals.verify_ssl)

        constructed_result = self._construct_result(run_job_flow_response)
        emrutils.display_response(self._session, 'run_job_flow',
                                  constructed_result, parsed_globals)

        return 0
Example #35
0
def _build_hbase_install_step(jar):
    return emrutils.build_step(
        jar=jar,
        name=constants.START_HBASE_NAME,
        action_on_failure=constants.TERMINATE_CLUSTER,
        args=constants.HBASE_INSTALL_ARG)
Example #36
0
    def _run_main(self, parsed_args, parsed_globals):
        emr = self._session.get_service("emr")
        params = {}
        bootstrap_actions = []
        params["Name"] = parsed_args.name

        service_role_validation_message = (
            " Either choose --use-default-roles or use both --service-role "
            "<roleName> and --ec2-attributes InstanceProfile=<profileName>."
        )

        if parsed_args.use_default_roles is True and parsed_args.service_role is not None:
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles", option2="--service-role", message=service_role_validation_message
            )

        if (
            parsed_args.use_default_roles is True
            and parsed_args.ec2_attributes is not None
            and "InstanceProfile" in parsed_args.ec2_attributes
        ):
            raise exceptions.MutualExclusiveOptionError(
                option1="--use-default-roles",
                option2="--ec2-attributes InstanceProfile",
                message=service_role_validation_message,
            )

        instances_config = {}
        instances_config["InstanceGroups"] = instancegroupsutils.validate_and_build_instance_groups(
            instance_groups=parsed_args.instance_groups,
            instance_type=parsed_args.instance_type,
            instance_count=parsed_args.instance_count,
        )

        is_valid_ami_version = re.match("\d?\..*", parsed_args.ami_version)
        if is_valid_ami_version is None:
            raise exceptions.InvalidAmiVersionError(ami_version=parsed_args.ami_version)
        params["AmiVersion"] = parsed_args.ami_version
        emrutils.apply_dict(params, "AdditionalInfo", parsed_args.additional_info)
        emrutils.apply_dict(params, "LogUri", parsed_args.log_uri)

        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes["InstanceProfile"] = EC2_ROLE_NAME

        emrutils.apply_dict(params, "ServiceRole", parsed_args.service_role)

        if parsed_args.no_auto_terminate is False and parsed_args.auto_terminate is False:
            parsed_args.no_auto_terminate = True

        instances_config["KeepJobFlowAliveWhenNoSteps"] = emrutils.apply_boolean_options(
            parsed_args.no_auto_terminate, "--no-auto-terminate", parsed_args.auto_terminate, "--auto-terminate"
        )

        instances_config["TerminationProtected"] = emrutils.apply_boolean_options(
            parsed_args.termination_protected,
            "--termination-protected",
            parsed_args.no_termination_protected,
            "--no-termination-protected",
        )

        if parsed_args.visible_to_all_users is False and parsed_args.no_visible_to_all_users is False:
            parsed_args.visible_to_all_users = True

        params["VisibleToAllUsers"] = emrutils.apply_boolean_options(
            parsed_args.visible_to_all_users,
            "--visible-to-all-users",
            parsed_args.no_visible_to_all_users,
            "--no-visible-to-all-users",
        )

        params["Tags"] = emrutils.parse_tags(parsed_args.tags)
        params["Instances"] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(cluster=params, parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging, "--enable-debugging", parsed_args.no_enable_debugging, "--no-enable-debugging"
        )

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(cluster=params, key="Steps", value=[self._build_enable_debugging(parsed_globals)])

        if parsed_args.applications is not None:
            app_list, ba_list, step_list = applicationutils.build_applications(
                session=self._session,
                parsed_applications=parsed_args.applications,
                parsed_globals=parsed_globals,
                ami_version=params["AmiVersion"],
            )
            self._update_cluster_dict(params, "NewSupportedProducts", app_list)
            self._update_cluster_dict(params, "BootstrapActions", ba_list)
            self._update_cluster_dict(params, "Steps", step_list)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get("Dir"), backup_version=hbase_restore_config.get("BackupVersion")
            )
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args,
            )
            self._update_cluster_dict(params, "Steps", [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(cluster=params, parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.emrfs is not None:
            emr_fs_ba_args = self._build_emr_fs_args(parsed_args.emrfs)
            emr_fs_ba_config = emrutils.build_bootstrap_action(
                path=emrutils.build_s3_link(relative_path=constants.CONFIG_HADOOP_PATH, region=parsed_globals.region),
                name=constants.EMR_FS_BA_NAME,
                args=emr_fs_ba_args,
            )
            self._update_cluster_dict(cluster=params, key="BootstrapActions", value=[emr_fs_ba_config])

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps, region=parsed_globals.region
            )
            self._update_cluster_dict(cluster=params, key="Steps", value=steps_list)

        self._validate_required_applications(parsed_args)

        run_job_flow = emr.get_operation("RunJobFlow")
        run_job_flow_response = emrutils.call(
            self._session,
            run_job_flow,
            params,
            parsed_globals.region,
            parsed_globals.endpoint_url,
            parsed_globals.verify_ssl,
        )

        constructed_result = self._construct_result(run_job_flow_response[1])
        emrutils.display_response(self._session, run_job_flow, constructed_result, parsed_globals)

        return 0
Example #37
0
    def _run_main(self, parsed_args, parsed_globals):
        emr = self._session.get_service('emr')
        params = {}
        bootstrap_actions = []
        params['Name'] = parsed_args.name

        is_valid_ami = re.match('\d?\..*', parsed_args.ami_version)
        if is_valid_ami is None:
            raise exceptions.\
                InvalidAmiVersionError(ami_version=parsed_args.ami_version)
        params['AmiVersion'] = parsed_args.ami_version
        emrutils.apply_dict(
            params, 'AdditionalInfo', parsed_args.additional_info)
        emrutils.apply_dict(params, 'LogUri', parsed_args.log_uri)
        if parsed_args.use_default_roles is True:
            parsed_args.service_role = EMR_ROLE_NAME
            if parsed_args.ec2_attributes is None:
                parsed_args.ec2_attributes = {}
            parsed_args.ec2_attributes['InstanceProfile'] = EC2_ROLE_NAME

        emrutils.apply_dict(params, 'ServiceRole', parsed_args.service_role)
        instances_config = {}
        instances_config['InstanceGroups'] = \
            instancegroupsutils.build_instance_groups(
                parsed_args.instance_groups)

        if (
                parsed_args.no_auto_terminate is False and
                parsed_args.auto_terminate is False):
            raise exceptions.\
                MissingBooleanOptionsError(
                    true_option='--auto-terminate',
                    false_option='--no-auto-terminate')

        instances_config['KeepJobFlowAliveWhenNoSteps'] = \
            emrutils.apply_boolean_options(
                parsed_args.no_auto_terminate,
                '--no-auto-terminate',
                parsed_args.auto_terminate,
                '--auto-terminate')

        instances_config['TerminationProtected'] = \
            emrutils.apply_boolean_options(
                parsed_args.termination_protected,
                '--termination-protected',
                parsed_args.no_termination_protected,
                '--no-termination-protected')

        params['VisibleToAllUsers'] = \
            emrutils.apply_boolean_options(
                parsed_args.visible_to_all_users,
                '--visible-to-all-users',
                parsed_args.no_visible_to_all_users,
                '--no-visible-to-all-users')

        params['Tags'] = emrutils.parse_tags(parsed_args.tags)
        params['Instances'] = instances_config

        if parsed_args.ec2_attributes is not None:
            self._build_ec2_attributes(
                cluster=params, parsed_attrs=parsed_args.ec2_attributes)

        debugging_enabled = emrutils.apply_boolean_options(
            parsed_args.enable_debugging,
            '--enable-debugging',
            parsed_args.no_enable_debugging,
            '--no-enable-debugging')

        if parsed_args.log_uri is None and debugging_enabled is True:
            raise exceptions.LogUriError

        if debugging_enabled is True:
            self._update_cluster_dict(
                cluster=params,
                key='Steps',
                value=[self._build_enable_debugging(parsed_globals)])

        if parsed_args.applications is not None:
            app_list, ba_list, step_list = applicationutils.build_applications(
                parsed_applications=parsed_args.applications,
                parsed_globals=parsed_globals,
                ami_version=params['AmiVersion'])
            self._update_cluster_dict(
                params, 'NewSupportedProducts', app_list)
            self._update_cluster_dict(
                params, 'BootstrapActions', ba_list)
            self._update_cluster_dict(
                params, 'Steps', step_list)

        hbase_restore_config = parsed_args.restore_from_hbase_backup
        if hbase_restore_config is not None:
            args = hbaseutils.build_hbase_restore_from_backup_args(
                dir=hbase_restore_config.get('Dir'),
                backup_version=hbase_restore_config.get('BackupVersion'))
            step_config = emrutils.build_step(
                jar=constants.HBASE_JAR_PATH,
                name=constants.HBASE_RESTORE_STEP_NAME,
                action_on_failure=constants.CANCEL_AND_WAIT,
                args=args)
            self._update_cluster_dict(
                params, 'Steps', [step_config])

        if parsed_args.bootstrap_actions is not None:
            self._build_bootstrap_actions(
                cluster=params,
                parsed_boostrap_actions=parsed_args.bootstrap_actions)

        if parsed_args.steps is not None:
            steps_list = steputils.build_step_config_list(
                parsed_step_list=parsed_args.steps,
                region=parsed_globals.region)
            self._update_cluster_dict(
                cluster=params, key='Steps', value=steps_list)

        self._validate_required_applications(parsed_args)

        run_job_flow = emr.get_operation('RunJobFlow')
        run_job_flow_response = emrutils.call(
            self._session, run_job_flow, params,
            parsed_globals.region, parsed_globals.endpoint_url,
            parsed_globals.verify_ssl)

        constructed_result = self._construct_result(run_job_flow_response[1])
        emrutils.display_response(self._session, run_job_flow,
                                  constructed_result, parsed_globals)

        return 0