def __populate_output_by_queue(self, job_queue, job_status, expand_children, details): """ Add Job items to the output asking for given queue and status. :param job_queue: job queue name or ARN :param job_status: list of job status to ask :param expand_children: if True, the job with children will be expanded by creating a row for each child :param details: ask for job details """ try: single_jobs = [] jobs_with_children = [] for status in job_status: next_token = "" # nosec while next_token is not None: response = self.batch_client.list_jobs(jobStatus=status, jobQueue=job_queue, nextToken=next_token) for job in response["jobSummaryList"]: if get_job_type(job) != "SIMPLE" and expand_children is True: jobs_with_children.append(job["jobId"]) else: single_jobs.append(job) next_token = response.get("nextToken") # create output items for job array children self.__populate_output_by_job_ids(jobs_with_children, details) # add single jobs to the output self.__add_jobs(single_jobs, details) except Exception as e: fail("Error listing jobs from AWS Batch. Failed with exception: %s" % e)
def __add_jobs(self, jobs, details=False): """ Get job info from AWS Batch and add to the output. :param jobs: list of jobs items (output of the list_jobs function) :param details: ask for job details """ try: if jobs: self.log.debug("Adding jobs to the output (%s)" % jobs) if details: self.log.info("Asking for jobs details") jobs_to_show = self.__chunked_describe_jobs([job["jobId"] for job in jobs]) else: jobs_to_show = jobs for job in jobs_to_show: self.log.debug("Adding job to the output (%s)", job) job_converter = self.__JOB_CONVERTERS[get_job_type(job)] self.output.add(job_converter.convert(job)) except KeyError as e: fail("Error building Job item. Key (%s) not found." % e) except Exception as e: fail("Error adding jobs to the output. Failed with exception: %s" % e)
def __populate_output_by_job_ids(self, job_ids, details, include_parents=False): """ Add Job item or jobs array children to the output. :param job_ids: job ids or ARNs :param details: ask for job details """ try: if job_ids: self.log.info("Describing jobs (%s), details (%s)" % (job_ids, details)) parent_jobs = [] jobs_with_children = [] jobs = self.__chunked_describe_jobs(job_ids) for job in jobs: # always add parent job if include_parents or get_job_type(job) == "SIMPLE": parent_jobs.append(job) if is_job_array(job): jobs_with_children.append((job["jobId"], ":", job["arrayProperties"]["size"])) elif is_mnp_job(job): jobs_with_children.append((job["jobId"], "#", job["nodeProperties"]["numNodes"])) # add parent jobs to the output self.__add_jobs(parent_jobs) # create output items for jobs' children self.__populate_output_by_parent_ids(jobs_with_children) except Exception as e: fail("Error describing jobs from AWS Batch. Failed with exception: %s" % e)
def __get_log_stream(self, job_id): """ Get log stream for the given job. :param job_id: job id (ARN) :return: the log_stream if there, or None """ log_stream = None try: batch_client = self.boto3_factory.get_client("batch") jobs = batch_client.describe_jobs(jobs=[job_id])["jobs"] if len(jobs) == 1: job = jobs[0] self.log.debug(job) if "nodeProperties" in job: # MNP job container = job["nodeProperties"]["nodeRangeProperties"][0]["container"] elif "container" in job: container = job["container"] else: container = {} if get_job_type(job) != "SIMPLE": fail("No output available for the Job (%s). Please ask for its children." % job["jobId"]) else: if "logStreamName" in container: log_stream = container.get("logStreamName") else: print("No log stream found for job (%s) in the status (%s)" % (job_id, job["status"])) else: fail("Error asking job output for job (%s). Job not found." % job_id) except Exception as e: fail("Error listing jobs from AWS Batch. Failed with exception: %s" % e) return log_stream
def __get_log_stream(self, job_id): """ Get log stream for the given job. :param job_id: job id (ARN) :return: the log_stream if there, or None """ log_stream = None try: batch_client = self.boto3_factory.get_client("batch") jobs = batch_client.describe_jobs(jobs=[job_id])["jobs"] if len(jobs) == 1: job = jobs[0] self.log.debug(job) if "nodeProperties" in job: # MNP job container = job["nodeProperties"]["nodeRangeProperties"][ 0]["container"] elif "container" in job: container = job["container"] else: container = {} if get_job_type(job) != "SIMPLE": fail( "No output available for the Job (%s). Please ask for its children." % job["jobId"]) else: if "logStreamName" in container: log_stream = container.get("logStreamName") else: print( "No log stream found for job (%s) in the status (%s)" % (job_id, job["status"])) else: fail("Error asking job output for job (%s). Job not found." % job_id) except Exception as e: fail("Error listing jobs from AWS Batch. Failed with exception: %s" % e) return log_stream