def post(self, request): run_ids = request.data.get('run_ids') pipeline_names = request.data.get('pipelines') job_group_id = request.data.get('job_group_id', None) for_each = request.data.get('for_each', False) if not for_each: for pipeline_name in pipeline_names: get_object_or_404(Pipeline, name=pipeline_name) try: run = Run.objects.get(id=run_ids[0]) req = run.tags.get('requestId', 'Unknown') except Run.DoesNotExist: req = 'Unknown' if not job_group_id: job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) notifier_start(job_group, req) else: try: job_group = JobGroup.objects.get(id=job_group_id) except JobGroup.DoesNotExist: return Response( {'details': 'Invalid JobGroup: %s' % job_group_id}, status=status.HTTP_400_BAD_REQUEST) for pipeline_name in pipeline_names: pipeline = get_object_or_404(Pipeline, name=pipeline_name) try: job_group_notifier = JobGroupNotifier.objects.get( job_group_id=job_group_id, notifier_type_id=pipeline.operator.notifier_id) job_group_notifier_id = str(job_group_notifier.id) except JobGroupNotifier.DoesNotExist: job_group_notifier_id = notifier_start( job_group, req, operator=pipeline.operator) operator_model = Operator.objects.get(id=pipeline.operator_id) operator = OperatorFactory.get_by_model( operator_model, run_ids=run_ids, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id) create_jobs_from_operator( operator, job_group_id, job_group_notifier_id=job_group_notifier_id) else: return Response({'details': 'Not Implemented'}, status=status.HTTP_400_BAD_REQUEST) body = { "details": "Operator Job submitted to pipelines %s, job group id %s, with runs %s" % (pipeline_names, job_group_id, str(run_ids)) } return Response(body, status=status.HTTP_202_ACCEPTED)
def create_jobs_from_request(request_id, operator_id, job_group_id, job_group_notifier_id=None, pipeline=None): logger.info("Creating operator id %s for requestId: %s for job_group: %s" % (operator_id, request_id, job_group_id)) operator_model = Operator.objects.get(id=operator_id) if not job_group_notifier_id: try: job_group = JobGroup.objects.get(id=job_group_id) except JobGroup.DoesNotExist: raise Exception("JobGroup doesn't exist") try: job_group_notifier_id = notifier_start(job_group, request_id, operator=operator_model) except Exception as e: logger.error("Failed to instantiate Notifier: %s" % str(e)) operator = OperatorFactory.get_by_model( operator_model, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id, request_id=request_id, pipeline=pipeline) _set_link_to_run_ticket(request_id, job_group_notifier_id) generate_description(job_group_id, job_group_notifier_id, request_id) generate_label(job_group_notifier_id, request_id) create_jobs_from_operator(operator, job_group_id, job_group_notifier_id)
def post(self, request): normals_override = request.data.get('normals_override', []) tumors_override = request.data.get('tumors_override', []) operator_model = Operator.objects.get(slug="tempo_mpgen_operator") pairing_override = None heading = "TempoMPGen Run %s" % datetime.datetime.now().isoformat() job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) job_group_notifier_id = notifier_start(job_group, heading, operator_model) operator = OperatorFactory.get_by_model( operator_model, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id) if normals_override and tumors_override: pairing_override = dict() pairing_override['normal_samples'] = normals_override pairing_override['tumor_samples'] = tumors_override body = { "details": "Submitting TempoMPGen Job with pairing overrides." } else: body = {"details": "TempoMPGen Job submitted."} create_tempo_mpgen_job(operator, pairing_override, job_group_id, job_group_notifier_id) return Response(body, status=status.HTTP_202_ACCEPTED)
def create_request_job(request_id, redelivery=False): logger.info( "Searching for job: %s for request_id: %s" % (TYPES['REQUEST'], request_id)) count = Job.objects.filter(run=TYPES['REQUEST'], args__request_id=request_id, status__in=[JobStatus.CREATED, JobStatus.IN_PROGRESS, JobStatus.WAITING_FOR_CHILDREN]).count() request_redelivered = Job.objects.filter(run=TYPES['REQUEST'], args__request_id=request_id).count() > 0 assays = ETLConfiguration.objects.first() if request_redelivered and not (assays.redelivery and redelivery): return None, "Request is redelivered, but redelivery deactivated" if count == 0: job_group = JobGroup() job_group.save() job_group_notifier_id = notifier_start(job_group, request_id) job_group_notifier = JobGroupNotifier.objects.get(id=job_group_notifier_id) job = Job(run=TYPES['REQUEST'], args={'request_id': request_id, 'job_group': str(job_group.id), 'job_group_notifier': job_group_notifier_id, 'redelivery': request_redelivered}, status=JobStatus.CREATED, max_retry=1, children=[], callback=TYPES['REQUEST_CALLBACK'], callback_args={'request_id': request_id, 'job_group': str(job_group.id), 'job_group_notifier': job_group_notifier_id}, job_group=job_group, job_group_notifier=job_group_notifier) job.save() if request_redelivered: redelivery_event = RedeliveryEvent(job_group_notifier_id).to_dict() send_notification.delay(redelivery_event) return job, "Job Created"
def post(self, request): pairs = request.data.get("pairs") pipeline_names = request.data.get("pipelines") pipeline_versions = request.data.get("pipeline_versions") name = request.data.get("name") labHeadName = request.data.get("labHeadName") investigatorName = request.data.get("investigatorName") assay = request.data.get("assay") job_group_id = request.data.get("job_group_id", None) output_directory_prefix = request.data.get("output_directory_prefix", None) if not job_group_id: job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) for i, pipeline_name in enumerate(pipeline_names): pipeline_version = pipeline_versions[i] pipeline = get_object_or_404(Pipeline, name=pipeline_name, version=pipeline_version) try: job_group_notifier = JobGroupNotifier.objects.get( job_group_id=job_group_id, notifier_type_id=pipeline.operator.notifier_id) job_group_notifier_id = str(job_group_notifier.id) except JobGroupNotifier.DoesNotExist: metadata = { "assay": assay, "investigatorName": investigatorName, "labHeadName": labHeadName } job_group_notifier_id = notifier_start( job_group, name, operator=pipeline.operator, metadata=metadata) operator_model = Operator.objects.get(id=pipeline.operator_id) operator = OperatorFactory.get_by_model( operator_model, pairing={"pairs": pairs}, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id, output_directory_prefix=output_directory_prefix, ) create_jobs_from_operator( operator, job_group_id, job_group_notifier_id=job_group_notifier_id) body = { "details": "Operator Job submitted to pipelines %s, job group id %s, with pairs %s" % (pipeline_names, job_group_id, str(pairs)) } return Response(body, status=status.HTTP_202_ACCEPTED)
def post(self, request): request_ids = request.data.get('request_ids') pipeline_name = request.data.get('pipeline') job_group_id = request.data.get('job_group_id', None) for_each = request.data.get('for_each', True) pipeline = get_object_or_404(Pipeline, name=pipeline_name) errors = [] if not request_ids: errors.append('request_ids needs to be specified') if not pipeline: errors.append('pipeline needs to be specified') if errors: return Response({'details': errors}, status=status.HTTP_400_BAD_REQUEST) if not job_group_id: if for_each: for req in request_ids: job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) logging.info("Submitting requestId %s to pipeline %s" % (req, pipeline)) create_jobs_from_request.delay(req, pipeline.operator_id, job_group_id, pipeline=str(pipeline.id)) else: return Response({'details': 'Not Implemented'}, status=status.HTTP_400_BAD_REQUEST) else: if for_each: for req in request_ids: logging.info("Submitting requestId %s to pipeline %s" % (req, pipeline)) try: job_group_notifier = JobGroupNotifier.objects.get( job_group_id=job_group_id, notifier_type_id=pipeline.operator.notifier_id) job_group_notifier_id = str(job_group_notifier.id) except JobGroupNotifier.DoesNotExist: job_group_notifier_id = notifier_start( job_group_id, req, pipeline.operator) create_jobs_from_request.delay( req, pipeline.operator_id, job_group_id, job_group_notifier_id=job_group_notifier_id, pipeline=str(pipeline.id)) else: return Response({'details': 'Not Implemented'}, status=status.HTTP_400_BAD_REQUEST) body = {"details": "Operator Job submitted %s" % str(request_ids)} return Response(body, status=status.HTTP_202_ACCEPTED)
def create_jobs_from_request(request_id, operator_id, job_group_id, job_group_notifier_id=None, pipeline=None): logger.info( format_log("Creating operator with %s" % operator_id, job_group_id=job_group_id, request_id=request_id)) operator_model = Operator.objects.get(id=operator_id) if not job_group_notifier_id: try: job_group = JobGroup.objects.get(id=job_group_id) except JobGroup.DoesNotExist: logger.info( format_log("Job group does not exist" % operator_id, job_group_id=job_group_id, request_id=request_id)) return try: job_group_notifier_id = notifier_start(job_group, request_id, operator=operator_model) request_obj = Request.objects.filter(request_id=request_id).first() if request_obj: delivery_date_event = SetDeliveryDateFieldEvent( job_group_notifier_id, str(request_obj.delivery_date)).to_dict() send_notification.delay(delivery_date_event) except Exception as e: logger.info( format_log("Failed to instantiate notifier" % operator_id, job_group_id=job_group_id, request_id=request_id)) operator = OperatorFactory.get_by_model( operator_model, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id, request_id=request_id, pipeline=pipeline, ) _set_link_to_run_ticket(request_id, job_group_notifier_id) generate_description(job_group_id, job_group_notifier_id, request_id) generate_label(job_group_notifier_id, request_id) create_jobs_from_operator(operator, job_group_id, job_group_notifier_id)
def post(self, request): lab_head_email = request.data.get('lab_head_email', []) if lab_head_email: operator_model = Operator.objects.get(class_name="AionOperator") operator = OperatorFactory.get_by_model( operator_model, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id) heading = "Aion Run for %s" % lab_head_email job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) job_group_notifier_id = notifier_start(job_group, heading, operator_model) create_aion_job(operator, lab_head_email, job_group_id, job_group_notifier_id) body = {"details": "Aion Job submitted for %s" % lab_head_email} return Response(body, status=status.HTTP_202_ACCEPTED)
def post(self, request): pairs = request.data.get('pairs') pipeline_names = request.data.get('pipelines') name = request.data.get('name') job_group_id = request.data.get('job_group_id', None) output_directory_prefix = request.data.get('output_directory_prefix', None) if not job_group_id: job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) for pipeline_name in pipeline_names: pipeline = get_object_or_404(Pipeline, name=pipeline_name) try: job_group_notifier = JobGroupNotifier.objects.get( job_group_id=job_group_id, notifier_type_id=pipeline.operator.notifier_id) job_group_notifier_id = str(job_group_notifier.id) except JobGroupNotifier.DoesNotExist: job_group_notifier_id = notifier_start( job_group, name, operator=pipeline.operator) operator_model = Operator.objects.get(id=pipeline.operator_id) operator = OperatorFactory.get_by_model( operator_model, pairing={'pairs': pairs}, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id, output_directory_prefix=output_directory_prefix) create_jobs_from_operator( operator, job_group_id, job_group_notifier_id=job_group_notifier_id) body = { "details": "Operator Job submitted to pipelines %s, job group id %s, with pairs %s" % (pipeline_names, job_group_id, str(pairs)) } return Response(body, status=status.HTTP_202_ACCEPTED)
def create_request_job(request_id, redelivery=False): logger.info("Searching for job: %s for request_id: %s" % (TYPES["REQUEST"], request_id)) count = Job.objects.filter( run=TYPES["REQUEST"], args__request_id=request_id, status__in=[ JobStatus.CREATED, JobStatus.IN_PROGRESS, JobStatus.WAITING_FOR_CHILDREN ], ).count() request_redelivered = Job.objects.filter( run=TYPES["REQUEST"], args__request_id=request_id).count() > 0 delivery_date = None try: request_from_lims = LIMSClient.get_request_samples(request_id) delivery_date = datetime.fromtimestamp( request_from_lims["deliveryDate"] / 1000) except Exception: logger.error("Failed to retrieve deliveryDate for request %s" % request_id) if not Request.objects.filter(request_id=request_id): Request.objects.create(request_id=request_id, delivery_date=delivery_date) assays = ETLConfiguration.objects.first() if request_redelivered and not (assays.redelivery and redelivery): return None, "Request is redelivered, but redelivery deactivated" if count == 0: job_group = JobGroup() job_group.save() job_group_notifier_id = notifier_start(job_group, request_id) job_group_notifier = JobGroupNotifier.objects.get( id=job_group_notifier_id) job = Job( run=TYPES["REQUEST"], args={ "request_id": request_id, "job_group": str(job_group.id), "job_group_notifier": job_group_notifier_id, "redelivery": request_redelivered, }, status=JobStatus.CREATED, max_retry=1, children=[], callback=TYPES["REQUEST_CALLBACK"], callback_args={ "request_id": request_id, "job_group": str(job_group.id), "job_group_notifier": job_group_notifier_id, }, job_group=job_group, job_group_notifier=job_group_notifier, ) job.save() if request_redelivered: redelivery_event = RedeliveryEvent(job_group_notifier_id).to_dict() send_notification.delay(redelivery_event) request_obj = Request.objects.filter(request_id=request_id).first() if request_obj: delivery_date_event = SetDeliveryDateFieldEvent( job_group_notifier_id, str(request_obj.delivery_date)).to_dict() send_notification.delay(delivery_date_event) return job, "Job Created"
def post(self, request): run_ids = request.data.get("run_ids") pipeline_names = request.data.get("pipelines") pipeline_versions = request.data.get("pipeline_versions", None) job_group_id = request.data.get("job_group_id", None) for_each = request.data.get("for_each", False) if not for_each: for i, pipeline_name in enumerate(pipeline_names): pipeline_version = pipeline_versions[i] get_object_or_404(Pipeline, name=pipeline_name, version=pipeline_version) try: run = Run.objects.get(id=run_ids[0]) req = run.tags.get("requestId", "Unknown") except Run.DoesNotExist: req = "Unknown" if not job_group_id: job_group = JobGroup() job_group.save() job_group_id = str(job_group.id) notifier_start(job_group, req) else: try: job_group = JobGroup.objects.get(id=job_group_id) except JobGroup.DoesNotExist: return Response( {"details": "Invalid JobGroup: %s" % job_group_id}, status=status.HTTP_400_BAD_REQUEST) for i, pipeline_name in enumerate(pipeline_names): pipeline_version = pipeline_versions[i] pipeline = get_object_or_404(Pipeline, name=pipeline_name, version=pipeline_version) try: job_group_notifier = JobGroupNotifier.objects.get( job_group_id=job_group_id, notifier_type_id=pipeline.operator.notifier_id) job_group_notifier_id = str(job_group_notifier.id) except JobGroupNotifier.DoesNotExist: job_group_notifier_id = notifier_start( job_group, req, operator=pipeline.operator) operator_model = Operator.objects.get(id=pipeline.operator_id) operator = OperatorFactory.get_by_model( operator_model, run_ids=run_ids, job_group_id=job_group_id, job_group_notifier_id=job_group_notifier_id, pipeline=str(pipeline.id), ) create_jobs_from_operator( operator, job_group_id, job_group_notifier_id=job_group_notifier_id) else: return Response({"details": "Not Implemented"}, status=status.HTTP_400_BAD_REQUEST) body = { "details": "Operator Job submitted to pipelines %s, job group id %s, with runs %s" % (pipeline_names, job_group_id, str(run_ids)) } return Response(body, status=status.HTTP_202_ACCEPTED)