def assert_mock_cluster_is(
self, mock_cluster,
starting=False,
bootstrapping=False,
done=False,
from_end_of_hour=timedelta(hours=1),
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False,
non_streaming=False):
self.assertEqual(starting,
is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done,
is_cluster_done(mock_cluster))
self.assertEqual(from_end_of_hour,
_est_time_to_hour(mock_cluster, self.now))
self.assertEqual(has_pending_steps,
cluster_has_pending_steps(mock_cluster._steps))
self.assertEqual(idle_for,
self.time_mock_cluster_idle(mock_cluster))
self.assertEqual((pool_hash, pool_name),
_pool_hash_and_name(mock_cluster._bootstrapactions))
self.assertEqual(running,
is_cluster_running(mock_cluster._steps))
self.assertEqual(non_streaming,
is_cluster_non_streaming(mock_cluster._steps))
def assert_mock_cluster_is(self,
mock_cluster,
starting=False,
bootstrapping=False,
done=False,
from_end_of_hour=timedelta(hours=1),
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False,
non_streaming=False):
self.assertEqual(starting, _is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
_is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done, _is_cluster_done(mock_cluster))
self.assertEqual(from_end_of_hour,
_est_time_to_hour(mock_cluster, self.now))
self.assertEqual(has_pending_steps,
_cluster_has_pending_steps(mock_cluster._steps))
self.assertEqual(idle_for, self.time_mock_cluster_idle(mock_cluster))
self.assertEqual((pool_hash, pool_name),
_pool_hash_and_name(mock_cluster._bootstrapactions))
self.assertEqual(running, _is_cluster_running(mock_cluster._steps))
self.assertEqual(non_streaming,
_is_cluster_non_streaming(mock_cluster._steps))
def test_started(self):
cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)),
readydatetime=to_iso8601(datetime(2010, 6, 6, 4, 30)))))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 35)),
timedelta(minutes=51))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 5, 20)),
timedelta(minutes=6))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 6, 26)),
timedelta(minutes=60))
def test_now_is_automatically_set(self):
cs = dict(Status=dict(Timeline=dict(CreationDateTime=_boto3_now())))
t = _est_time_to_hour(cs)
self.assertLessEqual(t, timedelta(minutes=60))
self.assertGreater(t, timedelta(minutes=59))
def test_not_yet_started(self):
cs = dict(Status=dict(Timeline=dict(
CreationDateTime=datetime(2010, 6, 6, 4, tzinfo=tzutc()))))
self.assertEqual(
_est_time_to_hour(cs,
now=datetime(2010, 6, 6, 4, 35, tzinfo=tzutc())),
timedelta(minutes=25))
self.assertEqual(
_est_time_to_hour(cs,
now=datetime(2010, 6, 6, 5, 20, tzinfo=tzutc())),
timedelta(minutes=40))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, tzinfo=tzutc())),
timedelta(minutes=60))
def test_not_yet_started(self):
cs = MockEmrObject(
status=MockEmrObject(
timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime(2010, 6, 6, 4)))))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 35)),
timedelta(minutes=25))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 5, 20)),
timedelta(minutes=40))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4)),
timedelta(minutes=60))
def test_now_is_automatically_set(self):
cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime.utcnow()))))
t = _est_time_to_hour(cs)
self.assertLessEqual(t, timedelta(minutes=60))
self.assertGreater(t, timedelta(minutes=59))
def test_clock_skew(self):
# make sure something reasonable happens if now is before
# the start time
cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)))))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 25, 59)),
timedelta(seconds=1))
def test_now_is_automatically_set(self):
cs = MockEmrObject(
status=MockEmrObject(
timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime.utcnow()))))
t = _est_time_to_hour(cs)
self.assertLessEqual(t, timedelta(minutes=60))
self.assertGreater(t, timedelta(minutes=59))
def test_clock_skew(self):
# make sure something reasonable happens if now is before
# the start time
cs = MockEmrObject(
status=MockEmrObject(
timeline=MockEmrObject(
creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)))))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 25, 59)),
timedelta(seconds=1))
def test_clock_skew(self):
# make sure something reasonable happens if now is before
# the start time
cs = dict(
Status=dict(
Timeline=dict(
CreationDateTime=datetime(
2010, 6, 6, 4, 26, tzinfo=tzutc()))))
self.assertEqual(
_est_time_to_hour(cs, now=datetime(
2010, 6, 6, 4, 25, 59, tzinfo=tzutc())),
timedelta(seconds=1))
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = datetime.utcnow()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_conn = runner.make_emr_conn()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_non_streaming = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _yield_all_clusters(emr_conn):
cluster_id = cluster_summary.id
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = _list_all_steps(emr_conn, cluster_id)
# we can't really tell if non-streaming jobs are idle or not, so
# let them be (see Issue #60)
if _is_cluster_non_streaming(steps):
num_non_streaming += 1
continue
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
bootstrap_actions = list(
_yield_all_bootstrap_actions(emr_conn, cluster_id))
_, pool = _pool_hash_and_name(bootstrap_actions)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary.name))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None
and time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary.name,
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d active non-streaming, %d done' %
(num_starting, num_bootstrapping, num_running, num_pending,
num_idle, num_non_streaming, num_done))
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = datetime.utcnow()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_conn = runner.make_emr_conn()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _yield_all_clusters(emr_conn):
cluster_id = cluster_summary.id
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = _list_all_steps(emr_conn, cluster_id)
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
bootstrap_actions = list(_yield_all_bootstrap_actions(
emr_conn, cluster_id))
_, pool = _pool_hash_and_name(bootstrap_actions)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug(
'cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary.name))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None and
time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(
minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(
runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary.name,
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info(
'Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' % (
num_starting, num_bootstrapping, num_running,
num_pending, num_idle, num_done))
def test_empty(self):
cs = dict()
self.assertEqual(_est_time_to_hour(cs), timedelta(hours=1))
def test_empty(self):
cs = MockEmrObject()
self.assertEqual(_est_time_to_hour(cs), timedelta(hours=1))
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = _boto3_now()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_client = runner.make_emr_client()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _boto3_paginate('Clusters', emr_client,
'list_clusters'):
cluster_id = cluster_summary['Id']
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = list(
reversed(
list(
_boto3_paginate('Steps',
emr_client,
'list_steps',
ClusterId=cluster_id))))
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
# need to get actual cluster to see tags
cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']
_, pool = _pool_hash_and_name(cluster)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary['Name']))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None
and time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary['Name'],
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' %
(num_starting, num_bootstrapping, num_running, num_pending,
num_idle, num_done))