예제 #1
0
def _yield_clusters(max_days_ago=None, now=None, **runner_kwargs):
    """Get relevant cluster information from EMR.

    :param float max_days_ago: If set, don't fetch clusters created longer
                               than this many days ago.
    :param now: the current UTC time, as a :py:class:`datetime.datetime`.
                Defaults to the current time.
    :param runner_kwargs: keyword args to pass through to
                          :py:class:`~mrjob.emr.EMRJobRunner`
    """
    if now is None:
        now = datetime.utcnow()

    emr_conn = EMRJobRunner(**runner_kwargs).make_emr_conn()

    # if --max-days-ago is set, only look at recent jobs
    created_after = None
    if max_days_ago is not None:
        created_after = now - timedelta(days=max_days_ago)

    # use _DELAY to sleep 1 second before each API call (see #1091). Could
    # implement some sort of connection wrapper for this if it becomes more
    # generally useful.
    for cluster_summary in _yield_all_clusters(emr_conn,
                                               created_after=created_after,
                                               _delay=_DELAY):
        cluster_id = cluster_summary.id

        sleep(_DELAY)
        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        cluster.steps = _list_all_steps(emr_conn, cluster_id, _delay=_DELAY)
        cluster.bootstrapactions = list(
            _yield_all_bootstrap_actions(emr_conn, cluster_id, _delay=_DELAY))

        yield cluster
예제 #2
0
def _yield_clusters(max_days_ago=None, now=None, **runner_kwargs):
    """Get relevant cluster information from EMR.

    :param float max_days_ago: If set, don't fetch clusters created longer
                               than this many days ago.
    :param now: the current UTC time, as a :py:class:`datetime.datetime`.
                Defaults to the current time.
    :param runner_kwargs: keyword args to pass through to
                          :py:class:`~mrjob.emr.EMRJobRunner`
    """
    if now is None:
        now = datetime.utcnow()

    emr_conn = EMRJobRunner(**runner_kwargs).make_emr_conn()

    # if --max-days-ago is set, only look at recent jobs
    created_after = None
    if max_days_ago is not None:
        created_after = now - timedelta(days=max_days_ago)

    for cluster_summary in _yield_all_clusters(
            emr_conn, created_after=created_after):
        cluster_id = cluster_summary.id

        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        cluster.steps = _list_all_steps(emr_conn, cluster_id)
        cluster.bootstrapactions = list(
            _yield_all_bootstrap_actions(emr_conn, cluster_id))

        yield cluster
예제 #3
0
def _yield_clusters(max_days_ago=None, now=None, **runner_kwargs):
    """Get relevant cluster information from EMR.

    :param float max_days_ago: If set, don't fetch clusters created longer
                               than this many days ago.
    :param now: the current UTC time, as a :py:class:`datetime.datetime`.
                Defaults to the current time.
    :param runner_kwargs: keyword args to pass through to
                          :py:class:`~mrjob.emr.EMRJobRunner`
    """
    if now is None:
        now = datetime.utcnow()

    emr_conn = EMRJobRunner(**runner_kwargs).make_emr_conn()

    # if --max-days-ago is set, only look at recent jobs
    created_after = None
    if max_days_ago is not None:
        created_after = now - timedelta(days=max_days_ago)

    # use _DELAY to sleep 1 second before each API call (see #1091). Could
    # implement some sort of connection wrapper for this if it becomes more
    # generally useful.
    for cluster_summary in _yield_all_clusters(
            emr_conn, created_after=created_after, _delay=_DELAY):
        cluster_id = cluster_summary.id

        sleep(_DELAY)
        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        cluster.steps = _list_all_steps(emr_conn, cluster_id, _delay=_DELAY)
        cluster.bootstrapactions = list(
            _yield_all_bootstrap_actions(emr_conn, cluster_id, _delay=_DELAY))

        yield cluster
예제 #4
0
def _yield_clusters(max_days_ago=None, now=None, **runner_kwargs):
    """Get relevant cluster information from EMR.

    :param float max_days_ago: If set, don't fetch clusters created longer
                               than this many days ago.
    :param now: the current UTC time, as a :py:class:`datetime.datetime`.
                Defaults to the current time.
    :param runner_kwargs: keyword args to pass through to
                          :py:class:`~mrjob.emr.EMRJobRunner`
    """
    if now is None:
        now = datetime.utcnow()

    emr_conn = EMRJobRunner(**runner_kwargs).make_emr_conn()

    # if --max-days-ago is set, only look at recent jobs
    created_after = None
    if max_days_ago is not None:
        created_after = now - timedelta(days=max_days_ago)

    for cluster_summary in _yield_all_clusters(emr_conn,
                                               created_after=created_after):
        cluster_id = cluster_summary.id

        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        cluster.steps = _list_all_steps(emr_conn, cluster_id)
        cluster.bootstrapactions = list(
            _yield_all_bootstrap_actions(emr_conn, cluster_id))

        yield cluster
예제 #5
0
    def test_terminate_cluster(self):
        cluster_id = self.make_cluster(pool_clusters=True)
        self.monkey_patch_argv('--quiet', '--no-conf', 'j-MOCKCLUSTER0')

        terminate_main()

        emr_conn = EMRJobRunner(conf_paths=[]).make_emr_conn()
        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        emr_conn.simulate_progress(cluster_id)
        self.assertEqual(cluster.status.state, 'TERMINATED')
예제 #6
0
    def test_terminate_cluster(self):
        cluster_id = self.make_cluster(pool_clusters=True)
        self.monkey_patch_argv('--quiet', '--no-conf', 'j-MOCKCLUSTER0')

        terminate_main()

        emr_conn = EMRJobRunner(conf_paths=[]).make_emr_conn()
        cluster = _patched_describe_cluster(emr_conn, cluster_id)
        emr_conn.simulate_progress(cluster_id)
        self.assertEqual(cluster.status.state, 'TERMINATED')