def create_dataset_members(self, dataset, data_list): """Creates a dataset member :param dataset: The dataset the member is a part of :type dataset: :class:`data.models.DataSet` :param data_list: Data definitions of the dataset members :type data_list: [:class:`data.data.data.Data`] """ with transaction.atomic(): dataset_members = [] datasetfiles = [] existing_scale_ids = DataSetFile.objects.get_file_ids( dataset_ids=[dataset.id]) for d in data_list: dataset_member = DataSetMember() dataset_member.dataset = dataset dataset_member.data = convert_data_to_v6_json(d).get_dict() dataset_member.file_ids = list(data_util.get_file_ids(d)) dataset_members.append(dataset_member) datasetfiles.extend( DataSetFile.objects.create_dataset_files( dataset, d, existing_scale_ids)) existing_scale_ids.append(dataset_member.file_ids) DataSetFile.objects.bulk_create(datasetfiles) return DataSetMember.objects.bulk_create(dataset_members)
def queue_jobs(self, jobs, requeue=False, priority=None): """Queues the given jobs. The caller must have obtained model locks on the job models in an atomic transaction. Any jobs that are not in a valid status for being queued, are without job input, or are superseded will be ignored. :param jobs: The job models to put on the queue :type jobs: :func:`list` :param requeue: Whether this is a re-queue (True) or a first queue (False) :type requeue: bool :param priority: An optional argument to reset the jobs' priority when they are queued :type priority: int :returns: The list of job IDs that were successfully QUEUED :rtype: :func:`list` """ when_queued = timezone.now() # Set job models to QUEUED queued_job_ids = Job.objects.update_jobs_to_queued(jobs, when_queued, requeue=requeue) if not queued_job_ids: return queued_job_ids # Done if nothing was queued # Retrieve the related job_type, job_type_rev, and batch models for the queued jobs queued_jobs = Job.objects.get_jobs_with_related(queued_job_ids) # Query for all input files of the queued jobs input_files = {} input_file_ids = set() for job in queued_jobs: input_file_ids.update(get_file_ids(job.get_input_data())) if input_file_ids: for input_file in ScaleFile.objects.get_files_for_queued_jobs( input_file_ids): input_files[input_file.id] = input_file # Bulk create queue models queues = [] job_ids = [] configurator = QueuedExecutionConfigurator(input_files) for job in queued_jobs: job_ids.append(job.id) config = configurator.configure_queued_job(job) manifest = SeedManifest(job.job_type.manifest) if priority: queued_priority = priority elif job.batch and job.batch.get_configuration().priority: queued_priority = job.batch.get_configuration().priority else: queued_priority = job.get_job_configuration().priority queue = Queue() # select_related from get_jobs_with_related above will only make a single query queue.docker_image = job.job_type_rev.docker_image queue.job_type_id = job.job_type_id queue.job_id = job.id queue.recipe_id = job.recipe_id queue.batch_id = job.batch_id queue.exe_num = job.num_exes queue.input_file_size = job.input_file_size if job.input_file_size else 0.0 queue.is_canceled = False queue.priority = queued_priority queue.timeout = manifest.get_timeout() if manifest else job.timeout queue.interface = job.get_job_interface().get_dict() queue.configuration = config.get_dict() if job.get_resources(): queue.resources = job.get_resources().get_json().get_dict() queue.queued = when_queued queues.append(queue) self.cancel_queued_jobs(job_ids) if queues: self.bulk_create(queues) return queued_job_ids