def post(self, request):
run_ids = request.data.get('run_ids')
pipeline_names = request.data.get('pipelines')
job_group_id = request.data.get('job_group_id', None)
for_each = request.data.get('for_each', False)
if not for_each:
for pipeline_name in pipeline_names:
get_object_or_404(Pipeline, name=pipeline_name)
try:
run = Run.objects.get(id=run_ids[0])
req = run.tags.get('requestId', 'Unknown')
except Run.DoesNotExist:
req = 'Unknown'
if not job_group_id:
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
notifier_start(job_group, req)
else:
try:
job_group = JobGroup.objects.get(id=job_group_id)
except JobGroup.DoesNotExist:
return Response(
{'details': 'Invalid JobGroup: %s' % job_group_id},
status=status.HTTP_400_BAD_REQUEST)
for pipeline_name in pipeline_names:
pipeline = get_object_or_404(Pipeline, name=pipeline_name)
try:
job_group_notifier = JobGroupNotifier.objects.get(
job_group_id=job_group_id,
notifier_type_id=pipeline.operator.notifier_id)
job_group_notifier_id = str(job_group_notifier.id)
except JobGroupNotifier.DoesNotExist:
job_group_notifier_id = notifier_start(
job_group, req, operator=pipeline.operator)
operator_model = Operator.objects.get(id=pipeline.operator_id)
operator = OperatorFactory.get_by_model(
operator_model,
run_ids=run_ids,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id)
create_jobs_from_operator(
operator,
job_group_id,
job_group_notifier_id=job_group_notifier_id)
else:
return Response({'details': 'Not Implemented'},
status=status.HTTP_400_BAD_REQUEST)
body = {
"details":
"Operator Job submitted to pipelines %s, job group id %s, with runs %s"
% (pipeline_names, job_group_id, str(run_ids))
}
return Response(body, status=status.HTTP_202_ACCEPTED)
def create_jobs_from_request(request_id,
operator_id,
job_group_id,
job_group_notifier_id=None,
pipeline=None):
logger.info("Creating operator id %s for requestId: %s for job_group: %s" %
(operator_id, request_id, job_group_id))
operator_model = Operator.objects.get(id=operator_id)
if not job_group_notifier_id:
try:
job_group = JobGroup.objects.get(id=job_group_id)
except JobGroup.DoesNotExist:
raise Exception("JobGroup doesn't exist")
try:
job_group_notifier_id = notifier_start(job_group,
request_id,
operator=operator_model)
except Exception as e:
logger.error("Failed to instantiate Notifier: %s" % str(e))
operator = OperatorFactory.get_by_model(
operator_model,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id,
request_id=request_id,
pipeline=pipeline)
_set_link_to_run_ticket(request_id, job_group_notifier_id)
generate_description(job_group_id, job_group_notifier_id, request_id)
generate_label(job_group_notifier_id, request_id)
create_jobs_from_operator(operator, job_group_id, job_group_notifier_id)
def post(self, request):
normals_override = request.data.get('normals_override', [])
tumors_override = request.data.get('tumors_override', [])
operator_model = Operator.objects.get(slug="tempo_mpgen_operator")
pairing_override = None
heading = "TempoMPGen Run %s" % datetime.datetime.now().isoformat()
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
job_group_notifier_id = notifier_start(job_group, heading,
operator_model)
operator = OperatorFactory.get_by_model(
operator_model,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id)
if normals_override and tumors_override:
pairing_override = dict()
pairing_override['normal_samples'] = normals_override
pairing_override['tumor_samples'] = tumors_override
body = {
"details": "Submitting TempoMPGen Job with pairing overrides."
}
else:
body = {"details": "TempoMPGen Job submitted."}
create_tempo_mpgen_job(operator, pairing_override, job_group_id,
job_group_notifier_id)
return Response(body, status=status.HTTP_202_ACCEPTED)
def create_request_job(request_id, redelivery=False):
logger.info(
"Searching for job: %s for request_id: %s" % (TYPES['REQUEST'], request_id))
count = Job.objects.filter(run=TYPES['REQUEST'], args__request_id=request_id,
status__in=[JobStatus.CREATED, JobStatus.IN_PROGRESS,
JobStatus.WAITING_FOR_CHILDREN]).count()
request_redelivered = Job.objects.filter(run=TYPES['REQUEST'], args__request_id=request_id).count() > 0
assays = ETLConfiguration.objects.first()
if request_redelivered and not (assays.redelivery and redelivery):
return None, "Request is redelivered, but redelivery deactivated"
if count == 0:
job_group = JobGroup()
job_group.save()
job_group_notifier_id = notifier_start(job_group, request_id)
job_group_notifier = JobGroupNotifier.objects.get(id=job_group_notifier_id)
job = Job(run=TYPES['REQUEST'],
args={'request_id': request_id, 'job_group': str(job_group.id),
'job_group_notifier': job_group_notifier_id, 'redelivery': request_redelivered},
status=JobStatus.CREATED,
max_retry=1,
children=[],
callback=TYPES['REQUEST_CALLBACK'],
callback_args={'request_id': request_id, 'job_group': str(job_group.id),
'job_group_notifier': job_group_notifier_id},
job_group=job_group,
job_group_notifier=job_group_notifier)
job.save()
if request_redelivered:
redelivery_event = RedeliveryEvent(job_group_notifier_id).to_dict()
send_notification.delay(redelivery_event)
return job, "Job Created"
def post(self, request):
pairs = request.data.get("pairs")
pipeline_names = request.data.get("pipelines")
pipeline_versions = request.data.get("pipeline_versions")
name = request.data.get("name")
labHeadName = request.data.get("labHeadName")
investigatorName = request.data.get("investigatorName")
assay = request.data.get("assay")
job_group_id = request.data.get("job_group_id", None)
output_directory_prefix = request.data.get("output_directory_prefix",
None)
if not job_group_id:
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
for i, pipeline_name in enumerate(pipeline_names):
pipeline_version = pipeline_versions[i]
pipeline = get_object_or_404(Pipeline,
name=pipeline_name,
version=pipeline_version)
try:
job_group_notifier = JobGroupNotifier.objects.get(
job_group_id=job_group_id,
notifier_type_id=pipeline.operator.notifier_id)
job_group_notifier_id = str(job_group_notifier.id)
except JobGroupNotifier.DoesNotExist:
metadata = {
"assay": assay,
"investigatorName": investigatorName,
"labHeadName": labHeadName
}
job_group_notifier_id = notifier_start(
job_group,
name,
operator=pipeline.operator,
metadata=metadata)
operator_model = Operator.objects.get(id=pipeline.operator_id)
operator = OperatorFactory.get_by_model(
operator_model,
pairing={"pairs": pairs},
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id,
output_directory_prefix=output_directory_prefix,
)
create_jobs_from_operator(
operator,
job_group_id,
job_group_notifier_id=job_group_notifier_id)
body = {
"details":
"Operator Job submitted to pipelines %s, job group id %s, with pairs %s"
% (pipeline_names, job_group_id, str(pairs))
}
return Response(body, status=status.HTTP_202_ACCEPTED)
def post(self, request):
request_ids = request.data.get('request_ids')
pipeline_name = request.data.get('pipeline')
job_group_id = request.data.get('job_group_id', None)
for_each = request.data.get('for_each', True)
pipeline = get_object_or_404(Pipeline, name=pipeline_name)
errors = []
if not request_ids:
errors.append('request_ids needs to be specified')
if not pipeline:
errors.append('pipeline needs to be specified')
if errors:
return Response({'details': errors},
status=status.HTTP_400_BAD_REQUEST)
if not job_group_id:
if for_each:
for req in request_ids:
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
logging.info("Submitting requestId %s to pipeline %s" %
(req, pipeline))
create_jobs_from_request.delay(req,
pipeline.operator_id,
job_group_id,
pipeline=str(pipeline.id))
else:
return Response({'details': 'Not Implemented'},
status=status.HTTP_400_BAD_REQUEST)
else:
if for_each:
for req in request_ids:
logging.info("Submitting requestId %s to pipeline %s" %
(req, pipeline))
try:
job_group_notifier = JobGroupNotifier.objects.get(
job_group_id=job_group_id,
notifier_type_id=pipeline.operator.notifier_id)
job_group_notifier_id = str(job_group_notifier.id)
except JobGroupNotifier.DoesNotExist:
job_group_notifier_id = notifier_start(
job_group_id, req, pipeline.operator)
create_jobs_from_request.delay(
req,
pipeline.operator_id,
job_group_id,
job_group_notifier_id=job_group_notifier_id,
pipeline=str(pipeline.id))
else:
return Response({'details': 'Not Implemented'},
status=status.HTTP_400_BAD_REQUEST)
body = {"details": "Operator Job submitted %s" % str(request_ids)}
return Response(body, status=status.HTTP_202_ACCEPTED)
def create_jobs_from_request(request_id,
operator_id,
job_group_id,
job_group_notifier_id=None,
pipeline=None):
logger.info(
format_log("Creating operator with %s" % operator_id,
job_group_id=job_group_id,
request_id=request_id))
operator_model = Operator.objects.get(id=operator_id)
if not job_group_notifier_id:
try:
job_group = JobGroup.objects.get(id=job_group_id)
except JobGroup.DoesNotExist:
logger.info(
format_log("Job group does not exist" % operator_id,
job_group_id=job_group_id,
request_id=request_id))
return
try:
job_group_notifier_id = notifier_start(job_group,
request_id,
operator=operator_model)
request_obj = Request.objects.filter(request_id=request_id).first()
if request_obj:
delivery_date_event = SetDeliveryDateFieldEvent(
job_group_notifier_id,
str(request_obj.delivery_date)).to_dict()
send_notification.delay(delivery_date_event)
except Exception as e:
logger.info(
format_log("Failed to instantiate notifier" % operator_id,
job_group_id=job_group_id,
request_id=request_id))
operator = OperatorFactory.get_by_model(
operator_model,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id,
request_id=request_id,
pipeline=pipeline,
)
_set_link_to_run_ticket(request_id, job_group_notifier_id)
generate_description(job_group_id, job_group_notifier_id, request_id)
generate_label(job_group_notifier_id, request_id)
create_jobs_from_operator(operator, job_group_id, job_group_notifier_id)
def post(self, request):
lab_head_email = request.data.get('lab_head_email', [])
if lab_head_email:
operator_model = Operator.objects.get(class_name="AionOperator")
operator = OperatorFactory.get_by_model(
operator_model,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id)
heading = "Aion Run for %s" % lab_head_email
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
job_group_notifier_id = notifier_start(job_group, heading,
operator_model)
create_aion_job(operator, lab_head_email, job_group_id,
job_group_notifier_id)
body = {"details": "Aion Job submitted for %s" % lab_head_email}
return Response(body, status=status.HTTP_202_ACCEPTED)
def post(self, request):
pairs = request.data.get('pairs')
pipeline_names = request.data.get('pipelines')
name = request.data.get('name')
job_group_id = request.data.get('job_group_id', None)
output_directory_prefix = request.data.get('output_directory_prefix',
None)
if not job_group_id:
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
for pipeline_name in pipeline_names:
pipeline = get_object_or_404(Pipeline, name=pipeline_name)
try:
job_group_notifier = JobGroupNotifier.objects.get(
job_group_id=job_group_id,
notifier_type_id=pipeline.operator.notifier_id)
job_group_notifier_id = str(job_group_notifier.id)
except JobGroupNotifier.DoesNotExist:
job_group_notifier_id = notifier_start(
job_group, name, operator=pipeline.operator)
operator_model = Operator.objects.get(id=pipeline.operator_id)
operator = OperatorFactory.get_by_model(
operator_model,
pairing={'pairs': pairs},
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id,
output_directory_prefix=output_directory_prefix)
create_jobs_from_operator(
operator,
job_group_id,
job_group_notifier_id=job_group_notifier_id)
body = {
"details":
"Operator Job submitted to pipelines %s, job group id %s, with pairs %s"
% (pipeline_names, job_group_id, str(pairs))
}
return Response(body, status=status.HTTP_202_ACCEPTED)
def create_request_job(request_id, redelivery=False):
logger.info("Searching for job: %s for request_id: %s" %
(TYPES["REQUEST"], request_id))
count = Job.objects.filter(
run=TYPES["REQUEST"],
args__request_id=request_id,
status__in=[
JobStatus.CREATED, JobStatus.IN_PROGRESS,
JobStatus.WAITING_FOR_CHILDREN
],
).count()
request_redelivered = Job.objects.filter(
run=TYPES["REQUEST"], args__request_id=request_id).count() > 0
delivery_date = None
try:
request_from_lims = LIMSClient.get_request_samples(request_id)
delivery_date = datetime.fromtimestamp(
request_from_lims["deliveryDate"] / 1000)
except Exception:
logger.error("Failed to retrieve deliveryDate for request %s" %
request_id)
if not Request.objects.filter(request_id=request_id):
Request.objects.create(request_id=request_id,
delivery_date=delivery_date)
assays = ETLConfiguration.objects.first()
if request_redelivered and not (assays.redelivery and redelivery):
return None, "Request is redelivered, but redelivery deactivated"
if count == 0:
job_group = JobGroup()
job_group.save()
job_group_notifier_id = notifier_start(job_group, request_id)
job_group_notifier = JobGroupNotifier.objects.get(
id=job_group_notifier_id)
job = Job(
run=TYPES["REQUEST"],
args={
"request_id": request_id,
"job_group": str(job_group.id),
"job_group_notifier": job_group_notifier_id,
"redelivery": request_redelivered,
},
status=JobStatus.CREATED,
max_retry=1,
children=[],
callback=TYPES["REQUEST_CALLBACK"],
callback_args={
"request_id": request_id,
"job_group": str(job_group.id),
"job_group_notifier": job_group_notifier_id,
},
job_group=job_group,
job_group_notifier=job_group_notifier,
)
job.save()
if request_redelivered:
redelivery_event = RedeliveryEvent(job_group_notifier_id).to_dict()
send_notification.delay(redelivery_event)
request_obj = Request.objects.filter(request_id=request_id).first()
if request_obj:
delivery_date_event = SetDeliveryDateFieldEvent(
job_group_notifier_id,
str(request_obj.delivery_date)).to_dict()
send_notification.delay(delivery_date_event)
return job, "Job Created"
def post(self, request):
run_ids = request.data.get("run_ids")
pipeline_names = request.data.get("pipelines")
pipeline_versions = request.data.get("pipeline_versions", None)
job_group_id = request.data.get("job_group_id", None)
for_each = request.data.get("for_each", False)
if not for_each:
for i, pipeline_name in enumerate(pipeline_names):
pipeline_version = pipeline_versions[i]
get_object_or_404(Pipeline,
name=pipeline_name,
version=pipeline_version)
try:
run = Run.objects.get(id=run_ids[0])
req = run.tags.get("requestId", "Unknown")
except Run.DoesNotExist:
req = "Unknown"
if not job_group_id:
job_group = JobGroup()
job_group.save()
job_group_id = str(job_group.id)
notifier_start(job_group, req)
else:
try:
job_group = JobGroup.objects.get(id=job_group_id)
except JobGroup.DoesNotExist:
return Response(
{"details": "Invalid JobGroup: %s" % job_group_id},
status=status.HTTP_400_BAD_REQUEST)
for i, pipeline_name in enumerate(pipeline_names):
pipeline_version = pipeline_versions[i]
pipeline = get_object_or_404(Pipeline,
name=pipeline_name,
version=pipeline_version)
try:
job_group_notifier = JobGroupNotifier.objects.get(
job_group_id=job_group_id,
notifier_type_id=pipeline.operator.notifier_id)
job_group_notifier_id = str(job_group_notifier.id)
except JobGroupNotifier.DoesNotExist:
job_group_notifier_id = notifier_start(
job_group, req, operator=pipeline.operator)
operator_model = Operator.objects.get(id=pipeline.operator_id)
operator = OperatorFactory.get_by_model(
operator_model,
run_ids=run_ids,
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id,
pipeline=str(pipeline.id),
)
create_jobs_from_operator(
operator,
job_group_id,
job_group_notifier_id=job_group_notifier_id)
else:
return Response({"details": "Not Implemented"},
status=status.HTTP_400_BAD_REQUEST)
body = {
"details":
"Operator Job submitted to pipelines %s, job group id %s, with runs %s"
% (pipeline_names, job_group_id, str(run_ids))
}
return Response(body, status=status.HTTP_202_ACCEPTED)