def run_jobflow(emr_conn, resource, context):
now = datetime.now().strftime("%Y-%m-%d %H%M%S")
target_steps = list(ifilterfalse(attrgetter("step_run"), context.children))
job_steps = make_jobsteps(now, resource, context, target_steps)
jobflow_name = _JOBFLOW_NAME.format(context.jobtype.title(), now)
if context.jobflow_id:
emr_conn.add_jobflow_steps(context.jobflow_id, job_steps)
else:
bootstraps = get_bootstrap_actions(context)
params = filter_object_attributes(resource, _JOBFLOW_PARAMS)
context.jobflow_id = emr_conn.run_jobflow(jobflow_name, steps=job_steps, bootstrap_actions=bootstraps, **params)
_ = map(lambda step: setattr(step, "step_run", True), target_steps)
if resource.termination_protection:
emr_conn.set_termination_protection(context.jobflow_id, resource.termination_protection)
def get_s3_connection(resource):
settings = filter_object_attributes(resource, _S3_CONNECTION_SETTINGS)
return S3Connection(**settings)
def get_emr_connection(resource):
settings = filter_object_attributes(resource, _EMR_CONNECTION_SETTINGS)
if resource.region:
region = RegionInfo(name=resource.region, endpoint=_EMR_REGION_TO_ENDPOINT[resource.region])
settings.update(region=region)
return EmrConnection(**settings)
