def submit_jobs(self, tasks):
# Register job definitions for each container_image
for container_image in set(task.drm_options["container_image"] for task in tasks):
if container_image not in self.image_to_job_definition:
self.log.info(f"Registering base job definition for image: {container_image}")
self.image_to_job_definition[container_image] = register_base_job_definition(
container_image=container_image, environment=None, command="user-should-override-this",
)
if len(tasks) > 1:
with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool:
rv = list(progress_bar(pool.map(self._submit_job, tasks), len(tasks), "Submitting"))
else:
# submit in serial without a progress bar
rv = list(map(self._submit_job, tasks))
for task, rv in zip(tasks, rv):
jobId, s3_command_script_uri, job_definition_arn = rv
if jobId is not None:
# set task attributes
task.drm_jobID = jobId
task.status = TaskStatus.submitted
task.s3_command_script_uri = s3_command_script_uri
task.job_definition_arn = job_definition_arn
else:
# self.procs[None] = None
# task.drm_jobID = None
task.status = TaskStatus.killed
def kill_tasks(self, tasks):
if len(tasks):
with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool:
self.log.info("Killing Tasks...")
list(
progress_bar(pool.map(self.kill, tasks),
count=len(tasks),
prefix="Killing "))
def submit_jobs(self, tasks):
if len(tasks) > 1:
with ThreadPoolExecutor(min(len(tasks), MAX_THREADS)) as pool:
rv = list(progress_bar(pool.map(self._submit_job, tasks), len(tasks), "Submitting"))
else:
# submit in serial without a progress bar
rv = map(self._submit_job, tasks)
for task, drm_jobID in zip(tasks, rv):
if drm_jobID is not None:
task.drm_jobID = drm_jobID
task.status = TaskStatus.submitted
else:
self.procs[None] = None
task.drm_jobID = None
task.status = TaskStatus.killed