def create_dataset_members(self, dataset, data_list):
"""Creates a dataset member
:param dataset: The dataset the member is a part of
:type dataset: :class:`data.models.DataSet`
:param data_list: Data definitions of the dataset members
:type data_list: [:class:`data.data.data.Data`]
"""
with transaction.atomic():
dataset_members = []
datasetfiles = []
existing_scale_ids = DataSetFile.objects.get_file_ids(
dataset_ids=[dataset.id])
for d in data_list:
dataset_member = DataSetMember()
dataset_member.dataset = dataset
dataset_member.data = convert_data_to_v6_json(d).get_dict()
dataset_member.file_ids = list(data_util.get_file_ids(d))
dataset_members.append(dataset_member)
datasetfiles.extend(
DataSetFile.objects.create_dataset_files(
dataset, d, existing_scale_ids))
existing_scale_ids.append(dataset_member.file_ids)
DataSetFile.objects.bulk_create(datasetfiles)
return DataSetMember.objects.bulk_create(dataset_members)
def queue_jobs(self, jobs, requeue=False, priority=None):
"""Queues the given jobs. The caller must have obtained model locks on the job models in an atomic transaction.
Any jobs that are not in a valid status for being queued, are without job input, or are superseded will be
ignored.
:param jobs: The job models to put on the queue
:type jobs: :func:`list`
:param requeue: Whether this is a re-queue (True) or a first queue (False)
:type requeue: bool
:param priority: An optional argument to reset the jobs' priority when they are queued
:type priority: int
:returns: The list of job IDs that were successfully QUEUED
:rtype: :func:`list`
"""
when_queued = timezone.now()
# Set job models to QUEUED
queued_job_ids = Job.objects.update_jobs_to_queued(jobs,
when_queued,
requeue=requeue)
if not queued_job_ids:
return queued_job_ids # Done if nothing was queued
# Retrieve the related job_type, job_type_rev, and batch models for the queued jobs
queued_jobs = Job.objects.get_jobs_with_related(queued_job_ids)
# Query for all input files of the queued jobs
input_files = {}
input_file_ids = set()
for job in queued_jobs:
input_file_ids.update(get_file_ids(job.get_input_data()))
if input_file_ids:
for input_file in ScaleFile.objects.get_files_for_queued_jobs(
input_file_ids):
input_files[input_file.id] = input_file
# Bulk create queue models
queues = []
job_ids = []
configurator = QueuedExecutionConfigurator(input_files)
for job in queued_jobs:
job_ids.append(job.id)
config = configurator.configure_queued_job(job)
manifest = SeedManifest(job.job_type.manifest)
if priority:
queued_priority = priority
elif job.batch and job.batch.get_configuration().priority:
queued_priority = job.batch.get_configuration().priority
else:
queued_priority = job.get_job_configuration().priority
queue = Queue()
# select_related from get_jobs_with_related above will only make a single query
queue.docker_image = job.job_type_rev.docker_image
queue.job_type_id = job.job_type_id
queue.job_id = job.id
queue.recipe_id = job.recipe_id
queue.batch_id = job.batch_id
queue.exe_num = job.num_exes
queue.input_file_size = job.input_file_size if job.input_file_size else 0.0
queue.is_canceled = False
queue.priority = queued_priority
queue.timeout = manifest.get_timeout() if manifest else job.timeout
queue.interface = job.get_job_interface().get_dict()
queue.configuration = config.get_dict()
if job.get_resources():
queue.resources = job.get_resources().get_json().get_dict()
queue.queued = when_queued
queues.append(queue)
self.cancel_queued_jobs(job_ids)
if queues:
self.bulk_create(queues)
return queued_job_ids