def _to_datetime(iso8601_time): """Convert a ISO8601-formatted datetime (from :py:mod:`boto`) to a :py:class:`datetime.datetime`.""" if iso8601_time is None: return None return iso8601_to_datetime(iso8601_time)
def _time_last_active(cluster_summary, steps): """When did something last happen with the given cluster? Things we look at: * ``cluster.creationdatetime`` (always set) * ``cluster.readydatetime`` (i.e. when bootstrapping finished) * ``step.creationdatetime`` for any step * ``step.startdatetime`` for any step * ``step.enddatetime`` for any step This is not really meant to be run on clusters which are currently running, or done. """ timestamps = [] for key in 'creationdatetime', 'readydatetime': value = getattr(cluster_summary.status.timeline, key, None) if value: timestamps.append(value) for step in steps: for key in 'creationdatetime', 'startdatetime', 'enddatetime': value = getattr(step.status.timeline, key, None) if value: timestamps.append(value) # for ISO8601 timestamps, alpha order == chronological order last_timestamp = max(timestamps) return iso8601_to_datetime(last_timestamp)
def to_datetime(iso8601_time): """Convert a ISO8601-formatted datetime (from :py:mod:`boto`) to a :py:class:`datetime.datetime`.""" if iso8601_time is None: return None return iso8601_to_datetime(iso8601_time)
def time_last_active(job_flow): """When did something last happen with the given job flow? Things we look at: * ``job_flow.creationdatetime`` (always set) * ``job_flow.startdatetime`` * ``job_flow.readydatetime`` (i.e. when bootstrapping finished) * ``step.creationdatetime`` for any step * ``step.startdatetime`` for any step * ``step.enddatetime`` for any step This is not really meant to be run on job flows which are currently running, or done. """ timestamps = [] for key in 'creationdatetime', 'startdatetime', 'readydatetime': value = getattr(job_flow, key, None) if value: timestamps.append(value) steps = getattr(job_flow, 'steps', None) or [] for step in steps: for key in 'creationdatetime', 'startdatetime', 'enddatetime': value = getattr(step, key, None) if value: timestamps.append(value) # for ISO8601 timestamps, alpha order == chronological order last_timestamp = max(timestamps) return iso8601_to_datetime(last_timestamp)
def est_time_to_hour(job_flow, now=None): """How long before job reaches the end of the next full hour since it began. This is important for billing purposes. If it happens to be exactly a whole number of hours, we return one hour, not zero. """ if now is None: now = datetime.utcnow() creationdatetime = getattr(job_flow, 'creationdatetime', None) startdatetime = getattr(job_flow, 'startdatetime', None) if creationdatetime: if startdatetime: start = iso8601_to_datetime(startdatetime) else: start = iso8601_to_datetime(job_flow.creationdatetime) else: # do something reasonable if creationdatetime isn't set return timedelta(minutes=60) run_time = now - start return timedelta(seconds=((-run_time).seconds % 3600.0 or 3600.0))
def _find_long_running_jobs(emr_conn, cluster_summaries, min_time, now=None): """Identify jobs that have been running or pending for a long time. :param clusters: a list of :py:class:`boto.emr.emrobject.Cluster` objects to inspect. :param min_time: a :py:class:`datetime.timedelta`: report jobs running or pending longer than this :param now: the current UTC time, as a :py:class:`datetime.datetime`. Defaults to the current time. For each job that is running or pending longer than *min_time*, yields a dictionary with the following keys: * *cluster_id*: the cluster's unique ID (e.g. ``j-SOMECLUSTER``) * *name*: name of the step, or the cluster when bootstrapping * *state*: state of the step (``'RUNNING'`` or ``'PENDING'``) or, if there is no step, the cluster (``'STARTING'`` or ``'BOOTSTRAPPING'``) * *time*: amount of time step was running or pending, as a :py:class:`datetime.timedelta` """ if now is None: now = datetime.utcnow() for cs in cluster_summaries: # special case for jobs that are taking a long time to bootstrap if cs.status.state in ('STARTING', 'BOOTSTRAPPING'): # there isn't a way to tell when the cluster stopped being # provisioned and started bootstrapping, so just measure # from cluster creation time created_timestamp = cs.status.timeline.creationdatetime created = iso8601_to_datetime(created_timestamp) time_running = now - created if time_running >= min_time: yield ({ 'cluster_id': cs.id, 'name': cs.name, 'state': cs.status.state, 'time': time_running }) # the default case: running clusters if cs.status.state != 'RUNNING': continue steps = _list_all_steps(emr_conn, cs.id) running_steps = [ step for step in steps if step.status.state == 'RUNNING' ] pending_steps = [ step for step in steps if step.status.state == 'PENDING' ] if running_steps: # should be only one, but if not, we should know about it for step in running_steps: start_timestamp = step.status.timeline.startdatetime start = iso8601_to_datetime(start_timestamp) time_running = now - start if time_running >= min_time: yield ({ 'cluster_id': cs.id, 'name': step.name, 'state': step.status.state, 'time': time_running }) # sometimes EMR says it's "RUNNING" but doesn't actually run steps! elif pending_steps: step = pending_steps[0] # PENDING job should have run starting when the cluster # became ready, or the previous step completed start_timestamp = cs.status.timeline.readydatetime for step in steps: if step.status.state == 'COMPLETED': start_timestamp = step.status.timeline.enddatetime start = iso8601_to_datetime(start_timestamp) time_pending = now - start if time_pending >= min_time: yield ({ 'cluster_id': cs.id, 'name': step.name, 'state': step.status.state, 'time': time_pending })
def find_long_running_jobs(job_flows, min_time, now=None): """Identify jobs that have been running or pending for a long time. :param job_flows: a list of :py:class:`boto.emr.emrobject.JobFlow` objects to inspect. :param min_time: a :py:class:`datetime.timedelta`: report jobs running or pending longer than this :param now: the current UTC time, as a :py:class:`datetime.datetime`. Defaults to the current time. For each job that is running or pending longer than *min_time*, yields a dictionary with the following keys: * *job_flow_id*: the job flow's unique ID (e.g. ``j-SOMEJOBFLOW``) * *name*: name of the step, or the job flow when bootstrapping * *step_state*: state of the step, either ``'RUNNING'`` or ``'PENDING'`` * *time*: amount of time step was running or pending, as a :py:class:`datetime.timedelta` """ if now is None: now = datetime.utcnow() for jf in job_flows: # special case for jobs that are taking a long time to bootstrap if jf.state == 'BOOTSTRAPPING': start_timestamp = jf.startdatetime start = iso8601_to_datetime(start_timestamp) time_running = now - start if time_running >= min_time: # we tell bootstrapping info by step_state being empty, # and only use job_flow_id and time in the report yield({'job_flow_id': jf.jobflowid, 'name': jf.name, 'step_state': '', 'time': time_running}) # the default case: running job flows if jf.state != 'RUNNING': continue running_steps = [step for step in jf.steps if step.state == 'RUNNING'] pending_steps = [step for step in jf.steps if step.state == 'PENDING'] if running_steps: # should be only one, but if not, we should know for step in running_steps: start_timestamp = step.startdatetime start = iso8601_to_datetime(start_timestamp) time_running = now - start if time_running >= min_time: yield({'job_flow_id': jf.jobflowid, 'name': step.name, 'step_state': step.state, 'time': time_running}) # sometimes EMR says it's "RUNNING" but doesn't actually run steps! elif pending_steps: step = pending_steps[0] # PENDING job should have run starting when the job flow # became ready, or the previous step completed start_timestamp = jf.readydatetime for step in jf.steps: if step.state == 'COMPLETED': start_timestamp = step.enddatetime start = iso8601_to_datetime(start_timestamp) time_pending = now - start if time_pending >= min_time: yield({'job_flow_id': jf.jobflowid, 'name': step.name, 'step_state': step.state, 'time': time_pending})
def find_long_running_jobs(emr_conn, cluster_summaries, min_time, now=None): """Identify jobs that have been running or pending for a long time. :param clusters: a list of :py:class:`boto.emr.emrobject.Cluster` objects to inspect. :param min_time: a :py:class:`datetime.timedelta`: report jobs running or pending longer than this :param now: the current UTC time, as a :py:class:`datetime.datetime`. Defaults to the current time. For each job that is running or pending longer than *min_time*, yields a dictionary with the following keys: * *cluster_id*: the cluster's unique ID (e.g. ``j-SOMECLUSTER``) * *name*: name of the step, or the cluster when bootstrapping * *state*: state of the step (``'RUNNING'`` or ``'PENDING'``) or, if there is no step, the cluster (``'STARTING'`` or ``'BOOTSTRAPPING'``) * *time*: amount of time step was running or pending, as a :py:class:`datetime.timedelta` """ if now is None: now = datetime.utcnow() for cs in cluster_summaries: # special case for jobs that are taking a long time to bootstrap if cs.status.state in ('STARTING', 'BOOTSTRAPPING'): # there isn't a way to tell when the cluster stopped being # provisioned and started bootstrapping, so just measure # from cluster creation time created_timestamp = cs.status.timeline.creationdatetime created = iso8601_to_datetime(created_timestamp) time_running = now - created if time_running >= min_time: yield({'cluster_id': cs.id, 'name': cs.name, 'state': cs.status.state, 'time': time_running}) # the default case: running clusters if cs.status.state != 'RUNNING': continue steps = list(_yield_all_steps(emr_conn, cs.id)) running_steps = [ step for step in steps if step.status.state == 'RUNNING'] pending_steps = [ step for step in steps if step.status.state == 'PENDING'] if running_steps: # should be only one, but if not, we should know about it for step in running_steps: start_timestamp = step.status.timeline.startdatetime start = iso8601_to_datetime(start_timestamp) time_running = now - start if time_running >= min_time: yield({'cluster_id': cs.id, 'name': step.name, 'state': step.status.state, 'time': time_running}) # sometimes EMR says it's "RUNNING" but doesn't actually run steps! elif pending_steps: step = pending_steps[0] # PENDING job should have run starting when the cluster # became ready, or the previous step completed start_timestamp = cs.status.timeline.readydatetime for step in steps: if step.status.state == 'COMPLETED': start_timestamp = step.status.timeline.enddatetime start = iso8601_to_datetime(start_timestamp) time_pending = now - start if time_pending >= min_time: yield({'cluster_id': cs.id, 'name': step.name, 'state': step.status.state, 'time': time_pending})