def __populate_output_by_queue(self, job_queue, job_status, expand_children, details):
"""
Add Job items to the output asking for given queue and status.
:param job_queue: job queue name or ARN
:param job_status: list of job status to ask
:param expand_children: if True, the job with children will be expanded by creating a row for each child
:param details: ask for job details
"""
try:
single_jobs = []
jobs_with_children = []
for status in job_status:
next_token = "" # nosec
while next_token is not None:
response = self.batch_client.list_jobs(jobStatus=status, jobQueue=job_queue, nextToken=next_token)
for job in response["jobSummaryList"]:
if get_job_type(job) != "SIMPLE" and expand_children is True:
jobs_with_children.append(job["jobId"])
else:
single_jobs.append(job)
next_token = response.get("nextToken")
# create output items for job array children
self.__populate_output_by_job_ids(jobs_with_children, details)
# add single jobs to the output
self.__add_jobs(single_jobs, details)
except Exception as e:
fail("Error listing jobs from AWS Batch. Failed with exception: %s" % e)
def __add_jobs(self, jobs, details=False):
"""
Get job info from AWS Batch and add to the output.
:param jobs: list of jobs items (output of the list_jobs function)
:param details: ask for job details
"""
try:
if jobs:
self.log.debug("Adding jobs to the output (%s)" % jobs)
if details:
self.log.info("Asking for jobs details")
jobs_to_show = self.__chunked_describe_jobs([job["jobId"] for job in jobs])
else:
jobs_to_show = jobs
for job in jobs_to_show:
self.log.debug("Adding job to the output (%s)", job)
job_converter = self.__JOB_CONVERTERS[get_job_type(job)]
self.output.add(job_converter.convert(job))
except KeyError as e:
fail("Error building Job item. Key (%s) not found." % e)
except Exception as e:
fail("Error adding jobs to the output. Failed with exception: %s" % e)
def __populate_output_by_job_ids(self, job_ids, details, include_parents=False):
"""
Add Job item or jobs array children to the output.
:param job_ids: job ids or ARNs
:param details: ask for job details
"""
try:
if job_ids:
self.log.info("Describing jobs (%s), details (%s)" % (job_ids, details))
parent_jobs = []
jobs_with_children = []
jobs = self.__chunked_describe_jobs(job_ids)
for job in jobs:
# always add parent job
if include_parents or get_job_type(job) == "SIMPLE":
parent_jobs.append(job)
if is_job_array(job):
jobs_with_children.append((job["jobId"], ":", job["arrayProperties"]["size"]))
elif is_mnp_job(job):
jobs_with_children.append((job["jobId"], "#", job["nodeProperties"]["numNodes"]))
# add parent jobs to the output
self.__add_jobs(parent_jobs)
# create output items for jobs' children
self.__populate_output_by_parent_ids(jobs_with_children)
except Exception as e:
fail("Error describing jobs from AWS Batch. Failed with exception: %s" % e)
def __get_log_stream(self, job_id):
"""
Get log stream for the given job.
:param job_id: job id (ARN)
:return: the log_stream if there, or None
"""
log_stream = None
try:
batch_client = self.boto3_factory.get_client("batch")
jobs = batch_client.describe_jobs(jobs=[job_id])["jobs"]
if len(jobs) == 1:
job = jobs[0]
self.log.debug(job)
if "nodeProperties" in job:
# MNP job
container = job["nodeProperties"]["nodeRangeProperties"][0]["container"]
elif "container" in job:
container = job["container"]
else:
container = {}
if get_job_type(job) != "SIMPLE":
fail("No output available for the Job (%s). Please ask for its children." % job["jobId"])
else:
if "logStreamName" in container:
log_stream = container.get("logStreamName")
else:
print("No log stream found for job (%s) in the status (%s)" % (job_id, job["status"]))
else:
fail("Error asking job output for job (%s). Job not found." % job_id)
except Exception as e:
fail("Error listing jobs from AWS Batch. Failed with exception: %s" % e)
return log_stream
def __get_log_stream(self, job_id):
"""
Get log stream for the given job.
:param job_id: job id (ARN)
:return: the log_stream if there, or None
"""
log_stream = None
try:
batch_client = self.boto3_factory.get_client("batch")
jobs = batch_client.describe_jobs(jobs=[job_id])["jobs"]
if len(jobs) == 1:
job = jobs[0]
self.log.debug(job)
if "nodeProperties" in job:
# MNP job
container = job["nodeProperties"]["nodeRangeProperties"][
0]["container"]
elif "container" in job:
container = job["container"]
else:
container = {}
if get_job_type(job) != "SIMPLE":
fail(
"No output available for the Job (%s). Please ask for its children."
% job["jobId"])
else:
if "logStreamName" in container:
log_stream = container.get("logStreamName")
else:
print(
"No log stream found for job (%s) in the status (%s)"
% (job_id, job["status"]))
else:
fail("Error asking job output for job (%s). Job not found." %
job_id)
except Exception as e:
fail("Error listing jobs from AWS Batch. Failed with exception: %s"
% e)
return log_stream