def __search_for_job_definition(self, base_job_definition, nodes):
"""
Search for existing job definition with the same name of the base_job_definition and the same number of nodes.
:param base_job_definition: job definition arn
:param nodes: number of nodes
:return: the found jobDefinition object or None
"""
job_definition_found = None
base_job_definition_name = get_job_definition_name_by_arn(
base_job_definition)
try:
next_token = ""
while next_token is not None:
response = self.batch_client.describe_job_definitions(
jobDefinitionName=base_job_definition_name,
status="ACTIVE",
nextToken=next_token)
for job_definition in response["jobDefinitions"]:
if job_definition["nodeProperties"]["numNodes"] == nodes:
job_definition_found = job_definition
break
next_token = response.get("nextToken")
except Exception as e:
fail("Error listing job definition. Failed with exception: %s" % e)
return job_definition_found
def _get_job_definition(job):
if "jobQueue" in job:
return get_job_definition_name_by_arn(job["jobDefinition"], version=True)
return "-"
def __add_jobs(self, jobs, details):
"""
Get job info from AWS Batch and add to the output
:param jobs: list of jobs items (output of the list_jobs function)
:param details: ask for job details
"""
try:
if jobs:
self.log.debug("Adding jobs to the output (%s)" % jobs)
if details:
self.log.info("Asking for jobs details")
jobs_to_show = []
for index in range(0, len(jobs), 100):
jobs_chunk = jobs[index:index + 100]
job_ids = []
for job in jobs_chunk:
job_ids.append(job['jobId'])
jobs_to_show.extend(
self.batch_client.describe_jobs(
jobs=job_ids)['jobs'])
else:
jobs_to_show = jobs
for job in jobs_to_show:
nodes = 1
if 'nodeProperties' in job:
# MNP job
container = job['nodeProperties'][
'nodeRangeProperties'][0]['container']
nodes = job['nodeProperties']['numNodes']
elif 'container' in job:
container = job['container']
else:
container = {}
if is_job_array(job):
# parent job array
job_id = '{0}[{1}]'.format(
job['jobId'], job['arrayProperties']['size'])
log_stream = '-'
log_stream_url = '-'
else:
job_id = job['jobId']
if 'logStreamName' in container:
log_stream = container.get('logStreamName')
log_stream_url = _compose_log_stream_url(
self.boto3_factory.region, log_stream)
else:
log_stream = '-'
log_stream_url = '-'
command = container.get('command', [])
self.log.debug("Adding job to the output (%s)", job)
job = Job(job_id=job_id,
name=job['jobName'],
creation_time=convert_to_date(job['createdAt']),
start_time=convert_to_date(job['startedAt'])
if 'startedAt' in job else '-',
stop_time=convert_to_date(job['stoppedAt'])
if 'stoppedAt' in job else '-',
status=job.get('status', 'UNKNOWN'),
status_reason=job.get('statusReason', '-'),
job_definition=get_job_definition_name_by_arn(
job['jobDefinition'], version=True)
if 'jobQueue' in job else '-',
queue=job['jobQueue'].split('/')[1]
if 'jobQueue' in job else '-',
command=shell_join(command) if command else '-',
reason=container.get('reason', '-'),
exit_code=container.get('exitCode', '-'),
vcpus=container.get('vcpus', '-'),
memory=container.get('memory', '-'),
nodes=nodes,
log_stream=log_stream,
log_stream_url=log_stream_url)
self.output.add(job)
except KeyError as e:
fail("Error building Job item. Key (%s) not found." % e)
except Exception as e:
fail("Error adding jobs to the output. Failed with exception: %s" %
e)
