def _add_step(emr_connection, step, jobflow_name, **jobflow_kw):
"""Add step to a running jobflow.
Append the step onto a compatible jobflow with the specified name if one
exists, otherwise create a new jobflow and run it. Returns the jobflowid.
NOTE: jobflow_kw will be used to configure the jobflow ONLY if a new
jobflow is created.
"""
running = get_compatible_jobflows(
emr_connection,
bootstrap_actions=TrafficBase._bootstrap_actions(),
setup_steps=TrafficBase._setup_steps())
for jf in running:
if jf.name == jobflow_name:
jobflowid = jf.jobflowid
emr_connection.add_jobflow_steps(jobflowid, step)
print 'Added %s to jobflow %s' % (step.name, jobflowid)
break
else:
base = TrafficBase(emr_connection, jobflow_name, steps=[step],
**jobflow_kw)
base.run()
jobflowid = base.jobflowid
print 'Added %s to new jobflow %s' % (step.name, jobflowid)
return jobflowid
def _add_step(emr_connection, step, jobflow_name, **jobflow_kw):
"""Add step to a running jobflow.
Append the step onto a compatible jobflow with the specified name if one
exists, otherwise create a new jobflow and run it. Returns the jobflowid.
NOTE: jobflow_kw will be used to configure the jobflow ONLY if a new
jobflow is created.
"""
running = get_compatible_jobflows(
emr_connection,
bootstrap_actions=TrafficBase._bootstrap_actions(),
setup_steps=TrafficBase._setup_steps())
for jf in running:
if jf.name == jobflow_name:
jobflowid = jf.jobflowid
emr_connection.add_jobflow_steps(jobflowid, step)
print 'Added %s to jobflow %s' % (step.name, jobflowid)
break
else:
base = TrafficBase(emr_connection,
jobflow_name,
steps=[step],
**jobflow_kw)
base.run()
jobflowid = base.jobflowid
print 'Added %s to new jobflow %s' % (step.name, jobflowid)
return jobflowid