def _terminate_and_notify(runner,
cluster_id,
cluster_name,
num_steps,
is_pending,
time_idle,
dry_run=False,
max_mins_locked=None,
quiet=False):
fmt = ('Terminated cluster %s (%s); was %s for %s')
msg = fmt % (cluster_id, cluster_name, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle))
did_terminate = False
if dry_run:
did_terminate = True
else:
status = _attempt_to_acquire_lock(
runner.fs,
runner._lock_uri(cluster_id, num_steps),
runner._opts['cloud_fs_sync_secs'],
'%s (%s)' % (msg, runner._make_unique_job_key(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_client().terminate_job_flows(
JobFlowIds=[cluster_id])
did_terminate = True
elif not quiet:
log.info('%s was locked between getting cluster info and'
' trying to terminate it; skipping' % cluster_id)
if did_terminate and not quiet:
print(msg)
def _terminate_and_notify(runner, cluster_id, cluster_name, num_steps,
is_pending, time_idle, time_to_end_of_hour,
dry_run=False, max_mins_locked=None, quiet=False):
fmt = ('Terminated cluster %s (%s); was %s for %s, %s to end of hour')
msg = fmt % (
cluster_id, cluster_name,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour))
did_terminate = False
if dry_run:
did_terminate = True
else:
status = _attempt_to_acquire_lock(
runner.fs.make_s3_conn(),
runner._lock_uri(cluster_id, num_steps),
runner._opts['cloud_fs_sync_secs'],
'%s (%s)' % (msg,
runner._make_unique_job_key(label='terminate')),
mins_to_expiration=max_mins_locked,
)
if status:
runner.make_emr_conn().terminate_jobflow(cluster_id)
did_terminate = True
elif not quiet:
log.info('%s was locked between getting cluster info and'
' trying to terminate it; skipping' % cluster_id)
if did_terminate and not quiet:
print(msg)