def assert_mock_cluster_is(
self, mock_cluster,
starting=False,
bootstrapping=False,
done=False,
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False):
self.assertEqual(starting,
_is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
_is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done,
_is_cluster_done(mock_cluster))
self.assertEqual(has_pending_steps,
_cluster_has_pending_steps(mock_cluster['_Steps']))
self.assertEqual(idle_for,
self.time_mock_cluster_idle(mock_cluster))
self.assertEqual((pool_hash, pool_name),
_pool_hash_and_name(mock_cluster))
self.assertEqual(running,
_is_cluster_running(mock_cluster['_Steps']))
def assert_mock_cluster_is(
self, mock_cluster,
starting=False,
bootstrapping=False,
done=False,
from_end_of_hour=timedelta(hours=1),
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False,
non_streaming=False):
self.assertEqual(starting,
is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done,
is_cluster_done(mock_cluster))
self.assertEqual(from_end_of_hour,
_est_time_to_hour(mock_cluster, self.now))
self.assertEqual(has_pending_steps,
cluster_has_pending_steps(mock_cluster._steps))
self.assertEqual(idle_for,
self.time_mock_cluster_idle(mock_cluster))
self.assertEqual((pool_hash, pool_name),
_pool_hash_and_name(mock_cluster._bootstrapactions))
self.assertEqual(running,
is_cluster_running(mock_cluster._steps))
self.assertEqual(non_streaming,
is_cluster_non_streaming(mock_cluster._steps))
def assert_mock_cluster_is(self,
mock_cluster,
starting=False,
bootstrapping=False,
done=False,
from_end_of_hour=timedelta(hours=1),
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False):
self.assertEqual(starting, _is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
_is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done, _is_cluster_done(mock_cluster))
self.assertEqual(from_end_of_hour,
_est_time_to_hour(mock_cluster, self.now))
self.assertEqual(has_pending_steps,
_cluster_has_pending_steps(mock_cluster['_Steps']))
self.assertEqual(idle_for, self.time_mock_cluster_idle(mock_cluster))
self.assertEqual(
(pool_hash, pool_name),
_pool_hash_and_name(mock_cluster['_BootstrapActions']))
self.assertEqual(running, _is_cluster_running(mock_cluster['_Steps']))
def assert_mock_cluster_is(
self, mock_cluster,
starting=False,
bootstrapping=False,
done=False,
has_pending_steps=False,
idle_for=timedelta(0),
pool_hash=None,
pool_name=None,
running=False):
self.assertEqual(starting,
_is_cluster_starting(mock_cluster))
self.assertEqual(bootstrapping,
_is_cluster_bootstrapping(mock_cluster))
self.assertEqual(done,
_is_cluster_done(mock_cluster))
self.assertEqual(has_pending_steps,
_cluster_has_pending_steps(mock_cluster['_Steps']))
self.assertEqual(idle_for,
self.time_mock_cluster_idle(mock_cluster))
self.assertEqual((pool_hash, pool_name),
_pool_hash_and_name(mock_cluster))
self.assertEqual(running,
_is_cluster_running(mock_cluster['_Steps']))
def test_too_few_args(self):
actions = [
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
], name='master'),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_first_arg_doesnt_start_with_pool(self):
actions = [
dict(
Args=['cowsay', 'mrjob'],
Name='master',
),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_too_many_args(self):
actions = [
dict(
Args=['cowsay', '-b', 'mrjob'],
Name='master',
),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_pooled_cluster(self):
cluster = dict(Tags=[
dict(Key='__mrjob_pool_hash',
Value='0123456789abcdef0123456789abcdef'),
dict(Key='__mrjob_pool_name', Value='reflecting'),
])
self.assertEqual(_pool_hash_and_name(cluster),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_too_few_args(self):
actions = [
dict(
Args=['pool-0123456789abcdef0123456789abcdef'],
Name='master',
),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_bootstrap_action_isnt_named_master(self):
actions = [
dict(
Args=['pool-0123456789abcdef0123456789abcdef', 'reflecting'],
Name='apprentice',
),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_first_arg_doesnt_start_with_pool(self):
actions = [
MockEmrObject(args=[
MockEmrObject(value='cowsay'),
MockEmrObject(value='mrjob'),
], name='master'),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_pooled_cluster(self):
cluster = dict(Tags=[
dict(Key='__mrjob_pool_hash',
Value='0123456789abcdef0123456789abcdef'),
dict(Key='__mrjob_pool_name',
Value='reflecting'),
])
self.assertEqual(_pool_hash_and_name(cluster),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_bootstrap_action_isnt_named_master(self):
actions = [
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
], name='apprentice'),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_too_many_args(self):
actions = [
MockEmrObject(args=[
MockEmrObject(value='cowsay'),
MockEmrObject(value='-b'),
MockEmrObject(value='mrjob'),
], name='master'),
]
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def test_pooled_cluster(self):
actions = [
dict(
Args=['pool-0123456789abcdef0123456789abcdef', 'reflecting'],
Name='master',
),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_cluster(self):
actions = [
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
], name='master'),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_cluster_with_other_bootstrap_actions(self):
actions = [
dict(Args=[], Name='action 0'),
dict(Args=[], Name='action 1'),
dict(
Args=['pool-0123456789abcdef0123456789abcdef', 'reflecting'],
Name='master',
),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_job_flow_with_other_bootstrap_actions(self):
actions = [
MockEmrObject(args=[], name='action 0'),
MockEmrObject(args=[], name='action 1'),
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
], name='master'),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_job_flow_with_other_bootstrap_actions(self):
actions = [
MockEmrObject(args=[], name='action 0'),
MockEmrObject(args=[], name='action 1'),
MockEmrObject(args=[
MockEmrObject(value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
],
name='master'),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_cluster_with_max_hours_idle(self):
# max hours idle is added AFTER the master bootstrap script,
# which was a problem when we just look at the last action
actions = [
dict(
Args=['pool-0123456789abcdef0123456789abcdef', 'reflecting'],
Name='master',
),
dict(
Args=['900', '300'],
Name='idle timeout',
),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def test_pooled_cluster_with_max_hours_idle(self):
# max hours idle is added AFTER the master bootstrap script,
# which was a problem when we just look at the last action
actions = [
MockEmrObject(args=[
MockEmrObject(
value='pool-0123456789abcdef0123456789abcdef'),
MockEmrObject(value='reflecting'),
], name='master'),
MockEmrObject(args=[
MockEmrObject(value='900'),
MockEmrObject(value='300'),
], name='idle timeout'),
]
self.assertEqual(_pool_hash_and_name(actions),
('0123456789abcdef0123456789abcdef', 'reflecting'))
def _maybe_terminate_clusters(dry_run=False,
max_mins_idle=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = _boto3_now()
# old default behavior
if max_mins_idle is None:
max_mins_idle = _DEFAULT_MAX_MINS_IDLE
runner = EMRJobRunner(**kwargs)
emr_client = runner.make_emr_client()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# include RUNNING to catch clusters with PENDING jobs that
# never ran (see #365).
for cluster_summary in _boto3_paginate(
'Clusters',
emr_client,
'list_clusters',
ClusterStates=['WAITING', 'RUNNING']):
cluster_id = cluster_summary['Id']
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = list(
reversed(
list(
_boto3_paginate('Steps',
emr_client,
'list_steps',
ClusterId=cluster_id))))
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
is_pending = _cluster_has_pending_steps(steps)
# need to get actual cluster to see tags
cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']
_, pool = _pool_hash_and_name(cluster)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug('cluster %s %s for %s, %s (%s) - %s' % (
cluster_id,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary['Name'],
'protected' if cluster['TerminationProtected'] else 'unprotected',
))
# filter out clusters that don't meet our criteria
if (max_mins_idle is not None
and time_idle <= timedelta(minutes=max_mins_idle)):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
if cluster['TerminationProtected']:
continue
# terminate idle cluster
_terminate_and_notify(runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary['Name'],
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' %
(num_starting, num_bootstrapping, num_running, num_pending,
num_idle, num_done))
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = datetime.utcnow()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_conn = runner.make_emr_conn()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_non_streaming = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _yield_all_clusters(emr_conn):
cluster_id = cluster_summary.id
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = _list_all_steps(emr_conn, cluster_id)
# we can't really tell if non-streaming jobs are idle or not, so
# let them be (see Issue #60)
if _is_cluster_non_streaming(steps):
num_non_streaming += 1
continue
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
bootstrap_actions = list(
_yield_all_bootstrap_actions(emr_conn, cluster_id))
_, pool = _pool_hash_and_name(bootstrap_actions)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary.name))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None
and time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary.name,
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d active non-streaming, %d done' %
(num_starting, num_bootstrapping, num_running, num_pending,
num_idle, num_non_streaming, num_done))
def _cluster_to_basic_summary(cluster, now=None):
"""Extract fields such as creation time, owner, etc. from the cluster.
:param cluster: a :py:mod:`boto3` cluster data structure
:param now: the current UTC time, as a :py:class:`datetime.datetime`.
Defaults to the current time.
Returns a dictionary with the following keys. These will be ``None`` if the
corresponding field in the cluster is unavailable.
* *created*: UTC `datetime.datetime` that the cluster was created,
or ``None``
* *end*: UTC `datetime.datetime` that the cluster finished, or ``None``
* *id*: cluster ID, or ``None`` (this should never happen)
* *label*: The label for the cluster (usually the module name of the
:py:class:`~mrjob.job.MRJob` script that started it), or
``None`` for non-:py:mod:`mrjob` clusters.
* *name*: cluster name, or ``None`` (this should never happen)
* *nih*: number of normalized instance hours used by the cluster.
* *num_steps*: Number of steps in the cluster.
* *owner*: The owner for the cluster (usually the user that started it),
or ``None`` for non-:py:mod:`mrjob` clusters.
* *pool*: pool name (e.g. ``'default'``) if the cluster is pooled,
otherwise ``None``.
* *ran*: How long the cluster ran, or has been running, as a
:py:class:`datetime.timedelta`. This will be ``timedelta(0)`` if
the cluster hasn't started.
* *ready*: UTC `datetime.datetime` that the cluster finished
bootstrapping, or ``None``
* *state*: The cluster's state as a string (e.g. ``'RUNNING'``)
"""
if now is None:
now = _boto3_now()
bcs = {} # basic cluster summary to fill in
bcs['id'] = cluster['Id']
bcs['name'] = cluster['Name']
Status = cluster['Status']
Timeline = Status.get('Timeline', {})
bcs['created'] = Timeline.get('CreationDateTime')
bcs['ready'] = Timeline.get('ReadyDateTime')
bcs['end'] = Timeline.get('EndDateTime')
if bcs['created']:
bcs['ran'] = (bcs['end'] or now) - bcs['created']
else:
bcs['ran'] = timedelta(0)
bcs['state'] = Status.get('State')
bcs['num_steps'] = len(cluster['Steps'])
_, bcs['pool'] = _pool_hash_and_name(cluster)
if not bcs['pool']:
_, bcs['pool'] = _legacy_pool_hash_and_name(
cluster['BootstrapActions'])
m = _JOB_KEY_RE.match(bcs['name'] or '')
if m:
bcs['label'], bcs['owner'] = m.group(1), m.group(2)
else:
bcs['label'], bcs['owner'] = None, None
bcs['nih'] = cluster.get('NormalizedInstanceHours', 0)
return bcs
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = _boto3_now()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_client = runner.make_emr_client()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _boto3_paginate('Clusters', emr_client,
'list_clusters'):
cluster_id = cluster_summary['Id']
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = list(
reversed(
list(
_boto3_paginate('Steps',
emr_client,
'list_steps',
ClusterId=cluster_id))))
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
# need to get actual cluster to see tags
cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']
_, pool = _pool_hash_and_name(cluster)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id, 'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary['Name']))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None
and time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary['Name'],
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' %
(num_starting, num_bootstrapping, num_running, num_pending,
num_idle, num_done))
def test_empty(self):
actions = []
self.assertEqual(_pool_hash_and_name(actions), (None, None))
def _maybe_terminate_clusters(dry_run=False,
max_hours_idle=None,
mins_to_end_of_hour=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = datetime.utcnow()
# old default behavior
if max_hours_idle is None and mins_to_end_of_hour is None:
max_hours_idle = _DEFAULT_MAX_HOURS_IDLE
runner = EMRJobRunner(**kwargs)
emr_conn = runner.make_emr_conn()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _yield_all_clusters(emr_conn):
cluster_id = cluster_summary.id
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = _list_all_steps(emr_conn, cluster_id)
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
is_pending = _cluster_has_pending_steps(steps)
bootstrap_actions = list(_yield_all_bootstrap_actions(
emr_conn, cluster_id))
_, pool = _pool_hash_and_name(bootstrap_actions)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug(
'cluster %s %s for %s, %s to end of hour, %s (%s)' %
(cluster_id,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
strip_microseconds(time_to_end_of_hour),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary.name))
# filter out clusters that don't meet our criteria
if (max_hours_idle is not None and
time_idle <= timedelta(hours=max_hours_idle)):
continue
# mins_to_end_of_hour doesn't apply to jobs with pending steps
if (mins_to_end_of_hour is not None and
(is_pending or
time_to_end_of_hour >= timedelta(
minutes=mins_to_end_of_hour))):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(
runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary.name,
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
time_to_end_of_hour=time_to_end_of_hour,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info(
'Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' % (
num_starting, num_bootstrapping, num_running,
num_pending, num_idle, num_done))
def _cluster_to_basic_summary(cluster, now=None):
"""Extract fields such as creation time, owner, etc. from the cluster.
:param cluster: a :py:mod:`boto3` cluster data structure
:param now: the current UTC time, as a :py:class:`datetime.datetime`.
Defaults to the current time.
Returns a dictionary with the following keys. These will be ``None`` if the
corresponding field in the cluster is unavailable.
* *created*: UTC `datetime.datetime` that the cluster was created,
or ``None``
* *end*: UTC `datetime.datetime` that the cluster finished, or ``None``
* *id*: cluster ID, or ``None`` (this should never happen)
* *label*: The label for the cluster (usually the module name of the
:py:class:`~mrjob.job.MRJob` script that started it), or
``None`` for non-:py:mod:`mrjob` clusters.
* *name*: cluster name, or ``None`` (this should never happen)
* *nih*: number of normalized instance hours cluster *would* use if it
ran to the end of the next full hour (
* *num_steps*: Number of steps in the cluster.
* *owner*: The owner for the cluster (usually the user that started it),
or ``None`` for non-:py:mod:`mrjob` clusters.
* *pool*: pool name (e.g. ``'default'``) if the cluster is pooled,
otherwise ``None``.
* *ran*: How long the cluster ran, or has been running, as a
:py:class:`datetime.timedelta`. This will be ``timedelta(0)`` if
the cluster hasn't started.
* *ready*: UTC `datetime.datetime` that the cluster finished
bootstrapping, or ``None``
* *state*: The cluster's state as a string (e.g. ``'RUNNING'``)
"""
if now is None:
now = _boto3_now()
bcs = {} # basic cluster summary to fill in
bcs['id'] = cluster['Id']
bcs['name'] = cluster['Name']
Status = cluster['Status']
Timeline = Status.get('Timeline', {})
bcs['created'] = Timeline.get('CreationDateTime')
bcs['ready'] = Timeline.get('ReadyDateTime')
bcs['end'] = Timeline.get('EndDateTime')
if bcs['created']:
bcs['ran'] = (bcs['end'] or now) - bcs['created']
else:
bcs['ran'] = timedelta(0)
bcs['state'] = Status.get('State')
bcs['num_steps'] = len(cluster['Steps'])
_, bcs['pool'] = _pool_hash_and_name(cluster)
if not bcs['pool']:
_, bcs['pool'] = _legacy_pool_hash_and_name(
cluster['BootstrapActions'])
m = _JOB_KEY_RE.match(bcs['name'] or '')
if m:
bcs['label'], bcs['owner'] = m.group(1), m.group(2)
else:
bcs['label'], bcs['owner'] = None, None
bcs['nih'] = float(cluster.get('NormalizedInstanceHours', 0))
return bcs
def test_empty(self):
self.assertEqual(_pool_hash_and_name({}), (None, None))
def test_empty(self):
self.assertEqual(_pool_hash_and_name({}), (None, None))
def _maybe_terminate_clusters(dry_run=False,
max_mins_idle=None,
now=None,
pool_name=None,
pooled_only=False,
unpooled_only=False,
max_mins_locked=None,
quiet=False,
**kwargs):
if now is None:
now = _boto3_now()
# old default behavior
if max_mins_idle is None:
max_mins_idle = _DEFAULT_MAX_MINS_IDLE
runner = EMRJobRunner(**kwargs)
emr_client = runner.make_emr_client()
num_starting = 0
num_bootstrapping = 0
num_done = 0
num_idle = 0
num_pending = 0
num_running = 0
# We don't filter by cluster state because we want this to work even
# if Amazon adds another kind of idle state.
for cluster_summary in _boto3_paginate(
'Clusters', emr_client, 'list_clusters'):
cluster_id = cluster_summary['Id']
# check if cluster is done
if _is_cluster_done(cluster_summary):
num_done += 1
continue
# check if cluster is starting
if _is_cluster_starting(cluster_summary):
num_starting += 1
continue
# check if cluster is bootstrapping
if _is_cluster_bootstrapping(cluster_summary):
num_bootstrapping += 1
continue
# need steps to learn more about cluster
steps = list(reversed(list(_boto3_paginate(
'Steps', emr_client, 'list_steps',
ClusterId=cluster_id))))
if any(_is_step_running(step) for step in steps):
num_running += 1
continue
# cluster is idle
time_idle = now - _time_last_active(cluster_summary, steps)
is_pending = _cluster_has_pending_steps(steps)
# need to get actual cluster to see tags
cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']
_, pool = _pool_hash_and_name(cluster)
if is_pending:
num_pending += 1
else:
num_idle += 1
log.debug(
'cluster %s %s for %s, %s (%s)' %
(cluster_id,
'pending' if is_pending else 'idle',
strip_microseconds(time_idle),
('unpooled' if pool is None else 'in %s pool' % pool),
cluster_summary['Name']))
# filter out clusters that don't meet our criteria
if (max_mins_idle is not None and
time_idle <= timedelta(minutes=max_mins_idle)):
continue
if (pooled_only and pool is None):
continue
if (unpooled_only and pool is not None):
continue
if (pool_name is not None and pool != pool_name):
continue
# terminate idle cluster
_terminate_and_notify(
runner=runner,
cluster_id=cluster_id,
cluster_name=cluster_summary['Name'],
num_steps=len(steps),
is_pending=is_pending,
time_idle=time_idle,
dry_run=dry_run,
max_mins_locked=max_mins_locked,
quiet=quiet)
log.info(
'Cluster statuses: %d starting, %d bootstrapping, %d running,'
' %d pending, %d idle, %d done' % (
num_starting, num_bootstrapping, num_running,
num_pending, num_idle, num_done))
def test_empty(self):
actions = []
self.assertEqual(_pool_hash_and_name(actions), (None, None))
