def submit_jobs(self, tasks): # Register job definitions for each container_image for container_image in set(task.drm_options["container_image"] for task in tasks): if container_image not in self.image_to_job_definition: self.log.info(f"Registering base job definition for image: {container_image}") self.image_to_job_definition[container_image] = register_base_job_definition( container_image=container_image, environment=None, command="user-should-override-this", ) if len(tasks) > 1: with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool: rv = list(progress_bar(pool.map(self._submit_job, tasks), len(tasks), "Submitting")) else: # submit in serial without a progress bar rv = list(map(self._submit_job, tasks)) for task, rv in zip(tasks, rv): jobId, s3_command_script_uri, job_definition_arn = rv if jobId is not None: # set task attributes task.drm_jobID = jobId task.status = TaskStatus.submitted task.s3_command_script_uri = s3_command_script_uri task.job_definition_arn = job_definition_arn else: # self.procs[None] = None # task.drm_jobID = None task.status = TaskStatus.killed
def kill_tasks(self, tasks): if len(tasks): with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool: self.log.info("Killing Tasks...") list( progress_bar(pool.map(self.kill, tasks), count=len(tasks), prefix="Killing "))
def submit_jobs(self, tasks): if len(tasks) > 1: with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool: rv = list(progress_bar(pool.map(self._submit_job, tasks), len(tasks), "Submitting")) else: # submit in serial without a progress bar rv = map(self._submit_job, tasks) for task, drm_jobID in zip(tasks, rv): if drm_jobID is not None: task.drm_jobID = drm_jobID task.status = TaskStatus.submitted else: self.procs[None] = None task.drm_jobID = None task.status = TaskStatus.killed