def build_streaming_step(parsed_step, release_label):
name = _apply_default_value(
arg=parsed_step.get('Name'),
value=constants.DEFAULT_STREAMING_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args = parsed_step.get('Args')
emrutils.check_required_field(
structure=constants.STREAMING_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
args_list = []
if release_label:
jar = constants.COMMAND_RUNNER
args_list.append(constants.HADOOP_STREAMING_COMMAND)
else:
jar = constants.HADOOP_STREAMING_PATH
args_list += args
return emrutils.build_step(
jar=jar,
args=args_list,
name=name,
action_on_failure=action_on_failure)
def build_hive_step(parsed_step, region=None):
args = parsed_step.get('Args')
emrutils.check_required_field(
structure=constants.HIVE_STEP_CONFIG, name='Args', value=args)
emrutils.check_empty_string_list(name='Args', value=args)
name = _apply_default_value(
arg=parsed_step.get('Name'),
value=constants.DEFAULT_HIVE_STEP_NAME)
action_on_failure = \
_apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args_list = [
emrutils.build_s3_link(
relative_path=constants.HIVE_SCRIPT_PATH, region=region),
constants.RUN_HIVE_SCRIPT]
args_list.append(constants.HIVE_VERSIONS)
args_list.append(constants.LATEST)
args_list.append(constants.ARGS)
args_list += args
return emrutils.build_step(
jar=emrutils.get_script_runner(region),
args=args_list,
name=name,
action_on_failure=action_on_failure)
def build_hive_step(parsed_step, region=None):
args = parsed_step.get('Args')
emrutils.check_required_field(structure=constants.HIVE_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
name = _apply_default_value(arg=parsed_step.get('Name'),
value=constants.DEFAULT_HIVE_STEP_NAME)
action_on_failure = \
_apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args_list = [
emrutils.build_s3_link(relative_path=constants.HIVE_SCRIPT_PATH,
region=region), constants.RUN_HIVE_SCRIPT
]
args_list.append(constants.HIVE_VERSIONS)
args_list.append(constants.LATEST)
args_list.append(constants.ARGS)
args_list += args
return emrutils.build_step(jar=emrutils.get_script_runner(region),
args=args_list,
name=name,
action_on_failure=action_on_failure)
def build_streaming_step(parsed_step, release_label):
name = _apply_default_value(arg=parsed_step.get('Name'),
value=constants.DEFAULT_STREAMING_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args = parsed_step.get('Args')
emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
args_list = []
if release_label:
jar = constants.COMMAND_RUNNER
args_list.append(constants.HADOOP_STREAMING_COMMAND)
else:
jar = constants.HADOOP_STREAMING_PATH
args_list += args
return emrutils.build_step(jar=jar,
args=args_list,
name=name,
action_on_failure=action_on_failure)
def build_streaming_step(parsed_step):
name = _apply_default_value(arg=parsed_step.get("Name"), value=constants.DEFAULT_STREAMING_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get("ActionOnFailure"), value=constants.DEFAULT_FAILURE_ACTION
)
args = parsed_step.get("Args")
emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG, name="Args", value=args)
emrutils.check_empty_string_list(name="Args", value=args)
return emrutils.build_step(
jar=constants.STREAMING_JAR_PATH, args=args, name=name, action_on_failure=action_on_failure
)
def build_streaming_step(parsed_step):
name = _apply_default_value(arg=parsed_step.get('Name'),
value=constants.DEFAULT_STREAMING_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
args = parsed_step.get('Args')
emrutils.check_required_field(structure=constants.STREAMING_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
return emrutils.build_step(jar=constants.STREAMING_JAR_PATH,
args=args,
name=name,
action_on_failure=action_on_failure)
def build_pig_step(parsed_step, release_label, region=None):
args = parsed_step.get('Args')
emrutils.check_required_field(
structure=constants.PIG_STEP_CONFIG, name='Args', value=args)
emrutils.check_empty_string_list(name='Args', value=args)
name = _apply_default_value(
arg=parsed_step.get('Name'),
value=constants.DEFAULT_PIG_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
return emrutils.build_step(
jar=_get_runner_jar(release_label, region),
args=_build_pig_args(args, release_label, region),
name=name,
action_on_failure=action_on_failure)
def build_pig_step(parsed_step, release_label, region=None):
args = parsed_step.get('Args')
emrutils.check_required_field(structure=constants.PIG_STEP_CONFIG,
name='Args',
value=args)
emrutils.check_empty_string_list(name='Args', value=args)
name = _apply_default_value(arg=parsed_step.get('Name'),
value=constants.DEFAULT_PIG_STEP_NAME)
action_on_failure = _apply_default_value(
arg=parsed_step.get('ActionOnFailure'),
value=constants.DEFAULT_FAILURE_ACTION)
return emrutils.build_step(jar=_get_runner_jar(release_label, region),
args=_build_pig_args(args, release_label,
region),
name=name,
action_on_failure=action_on_failure)
