def test_pooled_cluster_with_other_tags(self):
cluster = dict(Tags=[
dict(Key='__mrjob_pool_hash',
Value='0123456789abcdef0123456789abcdef'),
dict(Key='__mrjob_pool_name', Value='reflecting'),
dict(Key='price', Value='$9.99'),
])
self.assertEqual(_pool_name(cluster), 'reflecting')
def assert_mock_cluster_is(
self, mock_cluster,
starting=False,
bootstrapping=False,
done=False,
has_pending_steps=False,
idle_for=timedelta(0),
pool_name=None,
running=False):
self.assertEqual(starting,
_is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
_is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done,
_is_cluster_done(mock_cluster))
self.assertEqual(has_pending_steps,
_cluster_has_pending_steps(mock_cluster['_Steps']))
self.assertEqual(idle_for,
self.time_mock_cluster_idle(mock_cluster))
self.assertEqual(pool_name,
_pool_name(mock_cluster))
self.assertEqual(running,
_is_cluster_running(mock_cluster['_Steps']))
def test_pooled_cluster(self):
cluster = dict(Tags=[
dict(Key='__mrjob_pool_name', Value='reflecting'),
])
self.assertEqual(_pool_name(cluster), 'reflecting')
def test_empty(self):
self.assertEqual(_pool_name({}), None)
def _cluster_to_basic_summary(cluster, now=None):
"""Extract fields such as creation time, owner, etc. from the cluster.
:param cluster: a :py:mod:`boto3` cluster data structure
:param now: the current UTC time, as a :py:class:`datetime.datetime`.
Defaults to the current time.
Returns a dictionary with the following keys. These will be ``None`` if the
corresponding field in the cluster is unavailable.
* *created*: UTC `datetime.datetime` that the cluster was created,
or ``None``
* *end*: UTC `datetime.datetime` that the cluster finished, or ``None``
* *id*: cluster ID, or ``None`` (this should never happen)
* *label*: The label for the cluster (usually the module name of the
:py:class:`~mrjob.job.MRJob` script that started it), or
``None`` for non-:py:mod:`mrjob` clusters.
* *name*: cluster name, or ``None`` (this should never happen)
* *nih*: number of normalized instance hours cluster *would* use if it
ran to the end of the next full hour (
* *num_steps*: Number of steps in the cluster.
* *owner*: The owner for the cluster (usually the user that started it),
or ``None`` for non-:py:mod:`mrjob` clusters.
* *pool*: pool name (e.g. ``'default'``) if the cluster is pooled,
otherwise ``None``.
* *ran*: How long the cluster ran, or has been running, as a
:py:class:`datetime.timedelta`. This will be ``timedelta(0)`` if
the cluster hasn't started.
* *ready*: UTC `datetime.datetime` that the cluster finished
bootstrapping, or ``None``
* *state*: The cluster's state as a string (e.g. ``'RUNNING'``)
"""
if now is None:
now = _boto3_now()
bcs = {} # basic cluster summary to fill in
bcs['id'] = cluster['Id']
bcs['name'] = cluster['Name']
Status = cluster['Status']
Timeline = Status.get('Timeline', {})
bcs['created'] = Timeline.get('CreationDateTime')
bcs['ready'] = Timeline.get('ReadyDateTime')
bcs['end'] = Timeline.get('EndDateTime')
if bcs['created']:
bcs['ran'] = (bcs['end'] or now) - bcs['created']
else:
bcs['ran'] = timedelta(0)
bcs['state'] = Status.get('State')
bcs['num_steps'] = len(cluster['Steps'])
bcs['pool'] = _pool_name(cluster)
m = _JOB_KEY_RE.match(bcs['name'] or '')
if m:
bcs['label'], bcs['owner'] = m.group(1), m.group(2)
else:
bcs['label'], bcs['owner'] = None, None
bcs['nih'] = float(cluster.get('NormalizedInstanceHours', 0))
return bcs
def _maybe_terminate_clusters(dry_run=False,
max_mins_idle=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
quiet=False,
**kwargs):
if now is None:
now = _boto3_now()
# old default behavior
if max_mins_idle is None:
max_mins_idle = _DEFAULT_MAX_MINS_IDLE
runner = EMRJobRunner(**kwargs)
emr_client = runner.make_emr_client()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# include RUNNING to catch clusters with PENDING jobs that
# never ran (see #365).
for cluster_summary in _boto3_paginate(
'Clusters', emr_client, 'list_clusters',
ClusterStates=['WAITING', 'RUNNING']):
cluster_id = cluster_summary['Id']
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = list(reversed(list(_boto3_paginate(
'Steps', emr_client, 'list_steps',
ClusterId=cluster_id))))
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
is_pending = _cluster_has_pending_steps(steps)
# need to get actual cluster to see tags
cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']
pool = _pool_name(cluster)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug(
'cluster %s %s for %s, %s (%s) - %s' %
(cluster_id,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary['Name'],
'protected' if cluster['TerminationProtected'] else 'unprotected',
))
# filter out clusters that don't meet our criteria
if (max_mins_idle is not None and
time_idle <= timedelta(minutes=max_mins_idle)):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
if cluster['TerminationProtected']:
continue
# terminate idle cluster
_terminate_and_notify(
runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary['Name'],
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
dry_run=dry_run,
quiet=quiet)
log.info(
'Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' % (
num_starting, num_bootstrapping, num_running,
num_pending, num_idle, num_done))