def process_triggers():
operator_runs = OperatorRun.objects.prefetch_related(
'runs', 'operator__from_triggers').exclude(
status__in=[RunStatus.COMPLETED, RunStatus.FAILED])
for operator_run in operator_runs:
created_chained_job = False
job_group = operator_run.job_group
job_group_id = str(job_group.id) if job_group else None
job_group_notifier = operator_run.job_group_notifier
job_group_notifier_id = str(
job_group_notifier.id) if job_group_notifier else None
try:
for trigger in operator_run.operator.from_triggers.all():
trigger_type = trigger.run_type
if trigger_type == TriggerRunType.AGGREGATE:
condition = trigger.aggregate_condition
if condition == TriggerAggregateConditionType.ALL_RUNS_SUCCEEDED:
if operator_run.percent_runs_succeeded == 100.0:
created_chained_job = True
create_jobs_from_chaining.delay(
trigger.to_operator_id,
trigger.from_operator_id,
list(
operator_run.runs.order_by(
'id').values_list('id', flat=True)),
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id)
continue
elif condition == TriggerAggregateConditionType.NINTY_PERCENT_SUCCEEDED:
if operator_run.percent_runs_succeeded >= 90.0:
created_chained_job = True
create_jobs_from_chaining.delay(
trigger.to_operator_id,
trigger.from_operator_id,
list(
operator_run.runs.order_by(
'id').values_list('id', flat=True)),
job_group_id=job_group_id,
job_group_notifier_id=job_group_notifier_id)
continue
if operator_run.percent_runs_finished == 100.0:
logger.info("Condition never met for operator run %s" %
operator_run.id)
elif trigger_type == TriggerRunType.INDIVIDUAL:
if operator_run.percent_runs_finished == 100.0:
operator_run.complete()
if operator_run.percent_runs_finished == 100.0:
if operator_run.percent_runs_succeeded == 100.0:
operator_run.complete()
if not created_chained_job and job_group_notifier_id:
completed_event = SetPipelineCompletedEvent(
job_group_notifier_id).to_dict()
send_notification.delay(completed_event)
else:
operator_run.fail()
if job_group_notifier_id:
e = OperatorRequestEvent(
job_group_notifier_id,
"[CIReviewEvent] Operator Run %s failed" %
str(operator_run.id)).to_dict()
send_notification.delay(e)
ci_review_event = SetCIReviewEvent(
job_group_notifier_id).to_dict()
send_notification.delay(ci_review_event)
except Exception as e:
logger.info("Trigger %s Fail. Error %s" %
(operator_run.id, str(e)))
operator_run.fail()
def send_message(self, msg):
event = OperatorRequestEvent(self.job_group_notifier_id, msg)
e = event.to_dict()
send_notification.delay(e)
def request_callback(request_id, job_group=None, job_group_notifier=None):
jg = None
jgn = None
try:
jgn = JobGroupNotifier.objects.get(id=job_group_notifier)
logger.debug("[RequestCallback] JobGroup id: %s", job_group)
except JobGroupNotifier.DoesNotExist:
logger.debug("[RequestCallback] JobGroup not set")
job_group_notifier_id = str(jgn.id) if jgn else None
assays = ETLConfiguration.objects.first()
recipe = LIMSClient.get_request_samples(request_id).get("recipe", None)
if (not all([
JobStatus(job["status"]) == JobStatus.COMPLETED for job in
Job.objects.filter(job_group=job_group,
run=TYPES["SAMPLE"],
args__igocomplete=True).values("status")
]) and recipe in settings.WES_ASSAYS):
wes_job_failed = WESJobFailedEvent(job_group_notifier_id, recipe)
send_notification.delay(wes_job_failed.to_dict())
if not recipe:
raise FailedToSubmitToOperatorException(
"Not enough metadata to choose the operator for requestId:%s" %
request_id)
if not all(item in assays.all_recipes for item in [recipe]):
ci_review_e = SetCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(ci_review_e)
set_unknown_assay_label = SetLabelEvent(
job_group_notifier_id, "unrecognized_assay").to_dict()
send_notification.delay(set_unknown_assay_label)
unknown_assay_event = UnknownAssayEvent(job_group_notifier_id,
recipe).to_dict()
send_notification.delay(unknown_assay_event)
return []
if any(item in assays.hold_recipes for item in [
recipe,
]):
admin_hold_event = AdminHoldEvent(job_group_notifier_id).to_dict()
send_notification.delay(admin_hold_event)
custom_capture_event = CustomCaptureCCEvent(job_group_notifier_id,
recipe).to_dict()
send_notification.delay(custom_capture_event)
return []
if any(item in assays.disabled_recipes for item in [
recipe,
]):
not_for_ci = NotForCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(not_for_ci)
disabled_assay_event = DisabledAssayEvent(job_group_notifier_id,
recipe).to_dict()
send_notification.delay(disabled_assay_event)
return []
if len(
FileRepository.filter(metadata={
"requestId": request_id
},
values_metadata="recipe").all()) == 0:
no_samples_event = AdminHoldEvent(job_group_notifier_id).to_dict()
send_notification.delay(no_samples_event)
return []
if not all([
JobStatus(job["status"]) == JobStatus.COMPLETED
for job in Job.objects.filter(job_group=job_group).values("status")
]):
ci_review_e = SetCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(ci_review_e)
lab_head_email = FileRepository.filter(
metadata={
"requestId": request_id
}, values_metadata="labHeadEmail").first()
try:
if lab_head_email.split("@")[1] != "mskcc.org":
event = ExternalEmailEvent(job_group_notifier_id,
request_id).to_dict()
send_notification.delay(event)
except Exception:
logger.error("Failed to check labHeadEmail")
if len(
FileRepository.filter(metadata={
"requestId": request_id,
"tumorOrNormal": "Tumor"
})) == 0:
only_normal_samples_event = OnlyNormalSamplesEvent(
job_group_notifier_id, request_id).to_dict()
send_notification.delay(only_normal_samples_event)
if recipe in settings.ASSAYS_ADMIN_HOLD_ONLY_NORMALS:
admin_hold_event = AdminHoldEvent(job_group_notifier_id).to_dict()
send_notification.delay(admin_hold_event)
return []
operators = Operator.objects.filter(recipes__overlap=[recipe])
if not operators:
# TODO: Import ticket will have CIReviewNeeded
msg = "No operator defined for requestId %s with recipe %s" % (
request_id, recipe)
logger.error(msg)
e = OperatorRequestEvent(job_group_notifier_id,
"[CIReviewEvent] %s" % msg).to_dict()
send_notification.delay(e)
ci_review_e = SetCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(ci_review_e)
raise FailedToSubmitToOperatorException(msg)
for operator in operators:
if not operator.active:
msg = "Operator not active: %s" % operator.class_name
logger.info(msg)
e = OperatorRequestEvent(job_group_notifier_id,
"[CIReviewEvent] %s" % msg).to_dict()
send_notification.delay(e)
error_label = SetLabelEvent(job_group_notifier_id,
"operator_inactive").to_dict()
send_notification.delay(error_label)
ci_review_e = SetCIReviewEvent(job_group_notifier_id).to_dict()
send_notification.delay(ci_review_e)
else:
logger.info("Submitting request_id %s to %s operator" %
(request_id, operator.class_name))
if Job.objects.filter(job_group=job_group,
args__request_id=request_id,
run=TYPES["SAMPLE"],
status=JobStatus.FAILED).all():
partialy_complete_event = ETLImportPartiallyCompleteEvent(
job_notifier=job_group_notifier_id).to_dict()
send_notification.delay(partialy_complete_event)
else:
complete_event = ETLImportCompleteEvent(
job_notifier=job_group_notifier_id).to_dict()
send_notification.delay(complete_event)
create_jobs_from_request.delay(request_id, operator.id, job_group)
return []
