def terminate_and_notify(runner, to_terminate, dry_run=False,
max_mins_locked=None, quiet=False):
if not to_terminate:
return
for jf, pending, time_idle, time_to_end_of_hour in to_terminate:
fmt = ('Terminated job flow %s (%s); was %s for %s, %s to end of hour')
msg = fmt % (
jf.jobflowid, jf.name,
'pending' if pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
did_terminate = False
if not dry_run:
status = attempt_to_acquire_lock(
runner.make_s3_conn(),
runner._lock_uri(jf),
runner._opts['s3_sync_wait_time'],
'%s (%s)' % (msg,
runner._make_unique_job_name(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(jf.jobflowid)
did_terminate = True
elif not quiet:
log.info('%s was locked between getting job flow info and'
' trying to terminate it; skipping' % jf.jobflowid)
if did_terminate and not quiet:
print msg
def terminate_and_notify(runner,
to_terminate,
dry_run=False,
max_mins_locked=None,
quiet=False):
if not to_terminate:
return
for jf, pending, time_idle, time_to_end_of_hour in to_terminate:
fmt = ('Terminated job flow %s (%s); was %s for %s, %s to end of hour')
msg = fmt % (jf.jobflowid, jf.name, 'pending' if pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
did_terminate = False
if not dry_run:
status = attempt_to_acquire_lock(
runner.make_s3_conn(),
runner._lock_uri(jf),
runner._opts['s3_sync_wait_time'],
'%s (%s)' %
(msg, runner._make_unique_job_key(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(jf.jobflowid)
did_terminate = True
elif not quiet:
log.info('%s was locked between getting job flow info and'
' trying to terminate it; skipping' % jf.jobflowid)
if did_terminate and not quiet:
print(msg)
def terminate_and_notify(runner, to_terminate, dry_run=False,
max_mins_locked=None, quiet=False):
if not to_terminate:
return
for jf, pending, time_idle, time_to_end_of_hour in to_terminate:
did_terminate = False
if not dry_run:
status = attempt_to_acquire_lock(
runner.make_s3_conn(),
runner._lock_uri(jf),
runner._opts['s3_sync_wait_time'],
runner._make_unique_job_name(label='terminate'),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(jf.jobflowid)
did_terminate = True
if did_terminate and not quiet:
fmt = ('Terminated job flow %s (%s); was %s for %s, %s to end of'
' hour')
print fmt % (
jf.jobflowid, jf.name,
'pending' if pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
def terminate_and_notify(runner, cluster_id, cluster_name, num_steps,
is_pending, time_idle, time_to_end_of_hour,
dry_run=False, max_mins_locked=None, quiet=False):
fmt = ('Terminated cluster %s (%s); was %s for %s, %s to end of hour')
msg = fmt % (
cluster_id, cluster_name,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
did_terminate = False
if dry_run:
did_terminate = True
else:
status = attempt_to_acquire_lock(
runner.fs.make_s3_conn(),
runner._lock_uri(cluster_id, num_steps),
runner._opts['s3_sync_wait_time'],
'%s (%s)' % (msg,
runner._make_unique_job_key(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(cluster_id)
did_terminate = True
elif not quiet:
log.info('%s was locked between getting cluster info and'
' trying to terminate it; skipping' % cluster_id)
if did_terminate and not quiet:
print(msg)
def terminate_and_notify(runner,
cluster_id,
cluster_name,
num_steps,
is_pending,
time_idle,
time_to_end_of_hour,
dry_run=False,
max_mins_locked=None,
quiet=False):
fmt = ('Terminated cluster %s (%s); was %s for %s, %s to end of hour')
msg = fmt % (cluster_id, cluster_name, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
did_terminate = False
if dry_run:
did_terminate = True
else:
status = attempt_to_acquire_lock(
runner.fs.make_s3_conn(),
runner._lock_uri(cluster_id, num_steps),
runner._opts['s3_sync_wait_time'],
'%s (%s)' % (msg, runner._make_unique_job_key(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(cluster_id)
did_terminate = True
elif not quiet:
log.info('%s was locked between getting cluster info and'
' trying to terminate it; skipping' % cluster_id)
if did_terminate and not quiet:
print(msg)
