def assert_mock_cluster_is(
        self, mock_cluster,
        starting=False,
        bootstrapping=False,
        done=False,
        from_end_of_hour=timedelta(hours=1),
        has_pending_steps=False,
        idle_for=timedelta(0),
        pool_hash=None,
        pool_name=None,
        running=False,
        non_streaming=False):

        self.assertEqual(starting,
                         is_cluster_starting(mock_cluster))
        self.assertEqual(bootstrapping,
                         is_cluster_bootstrapping(mock_cluster))
        self.assertEqual(done,
                         is_cluster_done(mock_cluster))
        self.assertEqual(from_end_of_hour,
                         _est_time_to_hour(mock_cluster, self.now))
        self.assertEqual(has_pending_steps,
                         cluster_has_pending_steps(mock_cluster._steps))
        self.assertEqual(idle_for,
                         self.time_mock_cluster_idle(mock_cluster))
        self.assertEqual((pool_hash, pool_name),
                         _pool_hash_and_name(mock_cluster._bootstrapactions))
        self.assertEqual(running,
                         is_cluster_running(mock_cluster._steps))
        self.assertEqual(non_streaming,
                         is_cluster_non_streaming(mock_cluster._steps))
    def assert_mock_cluster_is(self,
                               mock_cluster,
                               starting=False,
                               bootstrapping=False,
                               done=False,
                               from_end_of_hour=timedelta(hours=1),
                               has_pending_steps=False,
                               idle_for=timedelta(0),
                               pool_hash=None,
                               pool_name=None,
                               running=False,
                               non_streaming=False):

        self.assertEqual(starting, _is_cluster_starting(mock_cluster))
        self.assertEqual(bootstrapping,
                         _is_cluster_bootstrapping(mock_cluster))
        self.assertEqual(done, _is_cluster_done(mock_cluster))
        self.assertEqual(from_end_of_hour,
                         _est_time_to_hour(mock_cluster, self.now))
        self.assertEqual(has_pending_steps,
                         _cluster_has_pending_steps(mock_cluster._steps))
        self.assertEqual(idle_for, self.time_mock_cluster_idle(mock_cluster))
        self.assertEqual((pool_hash, pool_name),
                         _pool_hash_and_name(mock_cluster._bootstrapactions))
        self.assertEqual(running, _is_cluster_running(mock_cluster._steps))
        self.assertEqual(non_streaming,
                         _is_cluster_non_streaming(mock_cluster._steps))
Example #3
0
    def test_started(self):
        cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
            creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)),
            readydatetime=to_iso8601(datetime(2010, 6, 6, 4, 30)))))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 35)),
            timedelta(minutes=51))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 5, 20)),
            timedelta(minutes=6))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 6, 26)),
            timedelta(minutes=60))
Example #4
0
    def test_now_is_automatically_set(self):
        cs = dict(Status=dict(Timeline=dict(CreationDateTime=_boto3_now())))

        t = _est_time_to_hour(cs)

        self.assertLessEqual(t, timedelta(minutes=60))
        self.assertGreater(t, timedelta(minutes=59))
Example #5
0
    def test_not_yet_started(self):
        cs = dict(Status=dict(Timeline=dict(
            CreationDateTime=datetime(2010, 6, 6, 4, tzinfo=tzutc()))))

        self.assertEqual(
            _est_time_to_hour(cs,
                              now=datetime(2010, 6, 6, 4, 35, tzinfo=tzutc())),
            timedelta(minutes=25))

        self.assertEqual(
            _est_time_to_hour(cs,
                              now=datetime(2010, 6, 6, 5, 20, tzinfo=tzutc())),
            timedelta(minutes=40))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, tzinfo=tzutc())),
            timedelta(minutes=60))
Example #6
0
    def test_not_yet_started(self):
        cs = MockEmrObject(
            status=MockEmrObject(
                timeline=MockEmrObject(
                    creationdatetime=to_iso8601(datetime(2010, 6, 6, 4)))))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 35)),
            timedelta(minutes=25))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 5, 20)),
            timedelta(minutes=40))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4)),
            timedelta(minutes=60))
Example #7
0
    def test_now_is_automatically_set(self):
        cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
            creationdatetime=to_iso8601(datetime.utcnow()))))

        t = _est_time_to_hour(cs)

        self.assertLessEqual(t, timedelta(minutes=60))
        self.assertGreater(t, timedelta(minutes=59))
Example #8
0
    def test_clock_skew(self):
        # make sure something reasonable happens if now is before
        # the start time
        cs = MockEmrObject(status=MockEmrObject(timeline=MockEmrObject(
            creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)))))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 25, 59)),
            timedelta(seconds=1))
Example #9
0
    def test_now_is_automatically_set(self):
        cs = MockEmrObject(
            status=MockEmrObject(
                timeline=MockEmrObject(
                    creationdatetime=to_iso8601(datetime.utcnow()))))

        t = _est_time_to_hour(cs)

        self.assertLessEqual(t, timedelta(minutes=60))
        self.assertGreater(t, timedelta(minutes=59))
Example #10
0
    def test_clock_skew(self):
        # make sure something reasonable happens if now is before
        # the start time
        cs = MockEmrObject(
            status=MockEmrObject(
                timeline=MockEmrObject(
                    creationdatetime=to_iso8601(datetime(2010, 6, 6, 4, 26)))))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(2010, 6, 6, 4, 25, 59)),
            timedelta(seconds=1))
Example #11
0
    def test_clock_skew(self):
        # make sure something reasonable happens if now is before
        # the start time
        cs = dict(
            Status=dict(
                Timeline=dict(
                    CreationDateTime=datetime(
                        2010, 6, 6, 4, 26, tzinfo=tzutc()))))

        self.assertEqual(
            _est_time_to_hour(cs, now=datetime(
                2010, 6, 6, 4, 25, 59, tzinfo=tzutc())),
            timedelta(seconds=1))
Example #12
0
def _maybe_terminate_clusters(dry_run=False,
                              max_hours_idle=None,
                              mins_to_end_of_hour=None,
                              now=None,
                              pool_name=None,
                              pooled_only=False,
                              unpooled_only=False,
                              max_mins_locked=None,
                              quiet=False,
                              **kwargs):
    if now is None:
        now = datetime.utcnow()

    # old default behavior
    if max_hours_idle is None and mins_to_end_of_hour is None:
        max_hours_idle = _DEFAULT_MAX_HOURS_IDLE

    runner = EMRJobRunner(**kwargs)
    emr_conn = runner.make_emr_conn()

    num_starting = 0
    num_bootstrapping = 0
    num_done = 0
    num_idle = 0
    num_non_streaming = 0
    num_pending = 0
    num_running = 0

    # We don't filter by cluster state because we want this to work even
    # if Amazon adds another kind of idle state.
    for cluster_summary in _yield_all_clusters(emr_conn):
        cluster_id = cluster_summary.id

        # check if cluster is done
        if _is_cluster_done(cluster_summary):
            num_done += 1
            continue

        # check if cluster is starting
        if _is_cluster_starting(cluster_summary):
            num_starting += 1
            continue

        # check if cluster is bootstrapping
        if _is_cluster_bootstrapping(cluster_summary):
            num_bootstrapping += 1
            continue

        # need steps to learn more about cluster
        steps = _list_all_steps(emr_conn, cluster_id)

        # we can't really tell if non-streaming jobs are idle or not, so
        # let them be (see Issue #60)
        if _is_cluster_non_streaming(steps):
            num_non_streaming += 1
            continue

        if any(_is_step_running(step) for step in steps):
            num_running += 1
            continue

        # cluster is idle
        time_idle = now - _time_last_active(cluster_summary, steps)
        time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
        is_pending = _cluster_has_pending_steps(steps)

        bootstrap_actions = list(
            _yield_all_bootstrap_actions(emr_conn, cluster_id))
        _, pool = _pool_hash_and_name(bootstrap_actions)

        if is_pending:
            num_pending += 1
        else:
            num_idle += 1

        log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
                  (cluster_id, 'pending' if is_pending else 'idle',
                   strip_microseconds(time_idle),
                   strip_microseconds(time_to_end_of_hour),
                   ('unpooled' if pool is None else 'in %s pool' % pool),
                   cluster_summary.name))

        # filter out clusters that don't meet our criteria
        if (max_hours_idle is not None
                and time_idle <= timedelta(hours=max_hours_idle)):
            continue

        # mins_to_end_of_hour doesn't apply to jobs with pending steps
        if (mins_to_end_of_hour is not None and
            (is_pending or
             time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
            continue

        if (pooled_only and pool is None):
            continue

        if (unpooled_only and pool is not None):
            continue

        if (pool_name is not None and pool != pool_name):
            continue

        # terminate idle cluster
        _terminate_and_notify(runner=runner,
                              cluster_id=cluster_id,
                              cluster_name=cluster_summary.name,
                              num_steps=len(steps),
                              is_pending=is_pending,
                              time_idle=time_idle,
                              time_to_end_of_hour=time_to_end_of_hour,
                              dry_run=dry_run,
                              max_mins_locked=max_mins_locked,
                              quiet=quiet)

    log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
             ' %d pending, %d idle, %d active non-streaming, %d done' %
             (num_starting, num_bootstrapping, num_running, num_pending,
              num_idle, num_non_streaming, num_done))
def _maybe_terminate_clusters(dry_run=False,
                              max_hours_idle=None,
                              mins_to_end_of_hour=None,
                              now=None,
                              pool_name=None,
                              pooled_only=False,
                              unpooled_only=False,
                              max_mins_locked=None,
                              quiet=False,
                              **kwargs):
    if now is None:
        now = datetime.utcnow()

    # old default behavior
    if max_hours_idle is None and mins_to_end_of_hour is None:
        max_hours_idle = _DEFAULT_MAX_HOURS_IDLE

    runner = EMRJobRunner(**kwargs)
    emr_conn = runner.make_emr_conn()

    num_starting = 0
    num_bootstrapping = 0
    num_done = 0
    num_idle = 0
    num_pending = 0
    num_running = 0

    # We don't filter by cluster state because we want this to work even
    # if Amazon adds another kind of idle state.
    for cluster_summary in _yield_all_clusters(emr_conn):
        cluster_id = cluster_summary.id

        # check if cluster is done
        if _is_cluster_done(cluster_summary):
            num_done += 1
            continue

        # check if cluster is starting
        if _is_cluster_starting(cluster_summary):
            num_starting += 1
            continue

        # check if cluster is bootstrapping
        if _is_cluster_bootstrapping(cluster_summary):
            num_bootstrapping += 1
            continue

        # need steps to learn more about cluster
        steps = _list_all_steps(emr_conn, cluster_id)

        if any(_is_step_running(step) for step in steps):
            num_running += 1
            continue

        # cluster is idle
        time_idle = now - _time_last_active(cluster_summary, steps)
        time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
        is_pending = _cluster_has_pending_steps(steps)

        bootstrap_actions = list(_yield_all_bootstrap_actions(
            emr_conn, cluster_id))
        _, pool = _pool_hash_and_name(bootstrap_actions)

        if is_pending:
            num_pending += 1
        else:
            num_idle += 1

        log.debug(
            'cluster %s %s for %s, %s to end of hour, %s (%s)' %
            (cluster_id,
             'pending' if is_pending else 'idle',
             strip_microseconds(time_idle),
             strip_microseconds(time_to_end_of_hour),
             ('unpooled' if pool is None else 'in %s pool' % pool),
             cluster_summary.name))

        # filter out clusters that don't meet our criteria
        if (max_hours_idle is not None and
                time_idle <= timedelta(hours=max_hours_idle)):
            continue

        # mins_to_end_of_hour doesn't apply to jobs with pending steps
        if (mins_to_end_of_hour is not None and
            (is_pending or
             time_to_end_of_hour >= timedelta(
                minutes=mins_to_end_of_hour))):
            continue

        if (pooled_only and pool is None):
            continue

        if (unpooled_only and pool is not None):
            continue

        if (pool_name is not None and pool != pool_name):
            continue

        # terminate idle cluster
        _terminate_and_notify(
            runner=runner,
            cluster_id=cluster_id,
            cluster_name=cluster_summary.name,
            num_steps=len(steps),
            is_pending=is_pending,
            time_idle=time_idle,
            time_to_end_of_hour=time_to_end_of_hour,
            dry_run=dry_run,
            max_mins_locked=max_mins_locked,
            quiet=quiet)

    log.info(
        'Cluster statuses: %d starting, %d bootstrapping, %d running,'
        ' %d pending, %d idle, %d done' % (
            num_starting, num_bootstrapping, num_running,
            num_pending, num_idle, num_done))
Example #14
0
 def test_empty(self):
     cs = dict()
     self.assertEqual(_est_time_to_hour(cs), timedelta(hours=1))
Example #15
0
 def test_empty(self):
     cs = MockEmrObject()
     self.assertEqual(_est_time_to_hour(cs), timedelta(hours=1))
def _maybe_terminate_clusters(dry_run=False,
                              max_hours_idle=None,
                              mins_to_end_of_hour=None,
                              now=None,
                              pool_name=None,
                              pooled_only=False,
                              unpooled_only=False,
                              max_mins_locked=None,
                              quiet=False,
                              **kwargs):
    if now is None:
        now = _boto3_now()

    # old default behavior
    if max_hours_idle is None and mins_to_end_of_hour is None:
        max_hours_idle = _DEFAULT_MAX_HOURS_IDLE

    runner = EMRJobRunner(**kwargs)
    emr_client = runner.make_emr_client()

    num_starting = 0
    num_bootstrapping = 0
    num_done = 0
    num_idle = 0
    num_pending = 0
    num_running = 0

    # We don't filter by cluster state because we want this to work even
    # if Amazon adds another kind of idle state.
    for cluster_summary in _boto3_paginate('Clusters', emr_client,
                                           'list_clusters'):

        cluster_id = cluster_summary['Id']

        # check if cluster is done
        if _is_cluster_done(cluster_summary):
            num_done += 1
            continue

        # check if cluster is starting
        if _is_cluster_starting(cluster_summary):
            num_starting += 1
            continue

        # check if cluster is bootstrapping
        if _is_cluster_bootstrapping(cluster_summary):
            num_bootstrapping += 1
            continue

        # need steps to learn more about cluster
        steps = list(
            reversed(
                list(
                    _boto3_paginate('Steps',
                                    emr_client,
                                    'list_steps',
                                    ClusterId=cluster_id))))

        if any(_is_step_running(step) for step in steps):
            num_running += 1
            continue

        # cluster is idle
        time_idle = now - _time_last_active(cluster_summary, steps)
        time_to_end_of_hour = _est_time_to_hour(cluster_summary, now=now)
        is_pending = _cluster_has_pending_steps(steps)

        # need to get actual cluster to see tags
        cluster = emr_client.describe_cluster(ClusterId=cluster_id)['Cluster']

        _, pool = _pool_hash_and_name(cluster)

        if is_pending:
            num_pending += 1
        else:
            num_idle += 1

        log.debug('cluster %s %s for %s, %s to end of hour, %s (%s)' %
                  (cluster_id, 'pending' if is_pending else 'idle',
                   strip_microseconds(time_idle),
                   strip_microseconds(time_to_end_of_hour),
                   ('unpooled' if pool is None else 'in %s pool' % pool),
                   cluster_summary['Name']))

        # filter out clusters that don't meet our criteria
        if (max_hours_idle is not None
                and time_idle <= timedelta(hours=max_hours_idle)):
            continue

        # mins_to_end_of_hour doesn't apply to jobs with pending steps
        if (mins_to_end_of_hour is not None and
            (is_pending or
             time_to_end_of_hour >= timedelta(minutes=mins_to_end_of_hour))):
            continue

        if (pooled_only and pool is None):
            continue

        if (unpooled_only and pool is not None):
            continue

        if (pool_name is not None and pool != pool_name):
            continue

        # terminate idle cluster
        _terminate_and_notify(runner=runner,
                              cluster_id=cluster_id,
                              cluster_name=cluster_summary['Name'],
                              num_steps=len(steps),
                              is_pending=is_pending,
                              time_idle=time_idle,
                              time_to_end_of_hour=time_to_end_of_hour,
                              dry_run=dry_run,
                              max_mins_locked=max_mins_locked,
                              quiet=quiet)

    log.info('Cluster statuses: %d starting, %d bootstrapping, %d running,'
             ' %d pending, %d idle, %d done' %
             (num_starting, num_bootstrapping, num_running, num_pending,
              num_idle, num_done))