def build_streaming_step(parsed_step, release_label): name = _apply_default_value( arg=parsed_step.get('Name'), value=constants.DEFAULT_STREAMING_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) args = parsed_step.get('Args') emrutils.check_required_field( structure=constants.STREAMING_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) args_list = [] if release_label: jar = constants.COMMAND_RUNNER args_list.append(constants.HADOOP_STREAMING_COMMAND) else: jar = constants.HADOOP_STREAMING_PATH args_list += args return emrutils.build_step( jar=jar, args=args_list, name=name, action_on_failure=action_on_failure)
def build_hive_step(parsed_step, region=None): args = parsed_step.get('Args') emrutils.check_required_field( structure=constants.HIVE_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) name = _apply_default_value( arg=parsed_step.get('Name'), value=constants.DEFAULT_HIVE_STEP_NAME) action_on_failure = \ _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) args_list = [ emrutils.build_s3_link( relative_path=constants.HIVE_SCRIPT_PATH, region=region), constants.RUN_HIVE_SCRIPT] args_list.append(constants.HIVE_VERSIONS) args_list.append(constants.LATEST) args_list.append(constants.ARGS) args_list += args return emrutils.build_step( jar=emrutils.get_script_runner(region), args=args_list, name=name, action_on_failure=action_on_failure)
def build_hive_step(parsed_step, region=None): args = parsed_step.get('Args') emrutils.check_required_field(structure=constants.HIVE_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) name = _apply_default_value(arg=parsed_step.get('Name'), value=constants.DEFAULT_HIVE_STEP_NAME) action_on_failure = \ _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) args_list = [ emrutils.build_s3_link(relative_path=constants.HIVE_SCRIPT_PATH, region=region), constants.RUN_HIVE_SCRIPT ] args_list.append(constants.HIVE_VERSIONS) args_list.append(constants.LATEST) args_list.append(constants.ARGS) args_list += args return emrutils.build_step(jar=emrutils.get_script_runner(region), args=args_list, name=name, action_on_failure=action_on_failure)
def build_streaming_step(parsed_step, release_label): name = _apply_default_value(arg=parsed_step.get('Name'), value=constants.DEFAULT_STREAMING_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) args = parsed_step.get('Args') emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) args_list = [] if release_label: jar = constants.COMMAND_RUNNER args_list.append(constants.HADOOP_STREAMING_COMMAND) else: jar = constants.HADOOP_STREAMING_PATH args_list += args return emrutils.build_step(jar=jar, args=args_list, name=name, action_on_failure=action_on_failure)
def build_streaming_step(parsed_step): name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_STREAMING_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION ) args = parsed_step.get("Args") emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG, name="Args", value=args) emrutils.check_empty_string_list(name="Args", value=args) return emrutils.build_step( jar=constants.STREAMING_JAR_PATH, args=args, name=name, action_on_failure=action_on_failure )
def build_streaming_step(parsed_step): name = _apply_default_value(arg=parsed_step.get('Name'), value=constants.DEFAULT_STREAMING_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) args = parsed_step.get('Args') emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) return emrutils.build_step(jar=constants.STREAMING_JAR_PATH, args=args, name=name, action_on_failure=action_on_failure)
def build_pig_step(parsed_step, release_label, region=None): args = parsed_step.get('Args') emrutils.check_required_field( structure=constants.PIG_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) name = _apply_default_value( arg=parsed_step.get('Name'), value=constants.DEFAULT_PIG_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) return emrutils.build_step( jar=_get_runner_jar(release_label, region), args=_build_pig_args(args, release_label, region), name=name, action_on_failure=action_on_failure)
def build_pig_step(parsed_step, release_label, region=None): args = parsed_step.get('Args') emrutils.check_required_field(structure=constants.PIG_STEP_CONFIG, name='Args', value=args) emrutils.check_empty_string_list(name='Args', value=args) name = _apply_default_value(arg=parsed_step.get('Name'), value=constants.DEFAULT_PIG_STEP_NAME) action_on_failure = _apply_default_value( arg=parsed_step.get('ActionOnFailure'), value=constants.DEFAULT_FAILURE_ACTION) return emrutils.build_step(jar=_get_runner_jar(release_label, region), args=_build_pig_args(args, release_label, region), name=name, action_on_failure=action_on_failure)