def convert(self, job):
"""
Convert a job from AWS Batch representation.
:param job: the job dictionary returned by AWS Batch api.
:return: a Job object containing the parsed data.
"""
container = self._get_container(job)
log_stream, log_stream_url = self._get_log_stream(container, self._get_job_region(job))
return Job(
job_id=self._get_job_id(job),
name=job["jobName"],
creation_time=convert_to_date(job["createdAt"]),
start_time=convert_to_date(job["startedAt"]) if "startedAt" in job else "-",
stop_time=convert_to_date(job["stoppedAt"]) if "stoppedAt" in job else "-",
status=job.get("status", "UNKNOWN"),
status_reason=job.get("statusReason", "-"),
job_definition=self._get_job_definition(job),
queue=self._get_job_queue(job),
command=self._get_command(container),
reason=container.get("reason", "-"),
exit_code=container.get("exitCode", "-"),
vcpus=container.get("vcpus", "-"),
memory=container.get("memory", "-"),
nodes=self._get_number_of_nodes(job),
log_stream=log_stream,
log_stream_url=log_stream_url,
s3_folder_url=self._get_s3_folder_url(container),
)
def __print_events(events):
"""
Print given events.
:param events: events to print
"""
for event in events:
print("{0}: {1}".format(convert_to_date(event["timestamp"]), event["message"]))
def _convert_to_date_utc(*args, **kwargs):
from awsbatch.utils import convert_to_date
from dateutil import tz
# executes convert_to_date but overrides arguments so that timezone is enforced to utc
if "timezone" in kwargs:
del kwargs["timezone"]
return convert_to_date(timezone=tz.tzutc(), *args, **kwargs)
def _convert_to_date_utc(*args, **kwargs):
from awsbatch.utils import convert_to_date
from dateutil import tz
# executes convert_to_date but overrides arguments so that timezone is enforced to utc
if "timezone" in kwargs:
del kwargs["timezone"]
return convert_to_date(timezone=tz.tzutc(), *args, **kwargs)
def __print_events(events):
"""
Print given events.
:param events: events to print
"""
for event in events:
print("{0}: {1}".format(convert_to_date(event["timestamp"]),
event["message"]))
def __print_events(events):
"""
Pring given events
:param events: events to print
:return:
"""
for event in events:
print('{0}: {1}'.format(convert_to_date(event['timestamp']),
event['message']))
def __add_jobs(self, jobs, details):
"""
Get job info from AWS Batch and add to the output
:param jobs: list of jobs items (output of the list_jobs function)
:param details: ask for job details
"""
try:
if jobs:
self.log.debug("Adding jobs to the output (%s)" % jobs)
if details:
self.log.info("Asking for jobs details")
jobs_to_show = []
for index in range(0, len(jobs), 100):
jobs_chunk = jobs[index:index + 100]
job_ids = []
for job in jobs_chunk:
job_ids.append(job['jobId'])
jobs_to_show.extend(
self.batch_client.describe_jobs(
jobs=job_ids)['jobs'])
else:
jobs_to_show = jobs
for job in jobs_to_show:
nodes = 1
if 'nodeProperties' in job:
# MNP job
container = job['nodeProperties'][
'nodeRangeProperties'][0]['container']
nodes = job['nodeProperties']['numNodes']
elif 'container' in job:
container = job['container']
else:
container = {}
if is_job_array(job):
# parent job array
job_id = '{0}[{1}]'.format(
job['jobId'], job['arrayProperties']['size'])
log_stream = '-'
log_stream_url = '-'
else:
job_id = job['jobId']
if 'logStreamName' in container:
log_stream = container.get('logStreamName')
log_stream_url = _compose_log_stream_url(
self.boto3_factory.region, log_stream)
else:
log_stream = '-'
log_stream_url = '-'
command = container.get('command', [])
self.log.debug("Adding job to the output (%s)", job)
job = Job(job_id=job_id,
name=job['jobName'],
creation_time=convert_to_date(job['createdAt']),
start_time=convert_to_date(job['startedAt'])
if 'startedAt' in job else '-',
stop_time=convert_to_date(job['stoppedAt'])
if 'stoppedAt' in job else '-',
status=job.get('status', 'UNKNOWN'),
status_reason=job.get('statusReason', '-'),
job_definition=get_job_definition_name_by_arn(
job['jobDefinition'], version=True)
if 'jobQueue' in job else '-',
queue=job['jobQueue'].split('/')[1]
if 'jobQueue' in job else '-',
command=shell_join(command) if command else '-',
reason=container.get('reason', '-'),
exit_code=container.get('exitCode', '-'),
vcpus=container.get('vcpus', '-'),
memory=container.get('memory', '-'),
nodes=nodes,
log_stream=log_stream,
log_stream_url=log_stream_url)
self.output.add(job)
except KeyError as e:
fail("Error building Job item. Key (%s) not found." % e)
except Exception as e:
fail("Error adding jobs to the output. Failed with exception: %s" %
e)