def update_editor_follows( # noqa: C901
instance, action, reverse, model, pk_set, **_
): # noqa: C901
if action not in ["post_add", "pre_remove", "pre_clear"]:
# nothing to do for the other actions
return
if reverse:
groups = [instance]
if pk_set is None:
users = instance.user_set.all()
else:
users = model.objects.filter(pk__in=pk_set).all()
else:
if pk_set is None:
groups = instance.groups.all()
else:
groups = model.objects.filter(pk__in=pk_set).all()
users = [instance]
follow_objects = []
for group in groups:
if hasattr(group, "editors_of_algorithm"):
follow_objects.append(group.editors_of_algorithm)
elif hasattr(group, "editors_of_archive"):
follow_objects.append(group.editors_of_archive)
elif hasattr(group, "editors_of_readerstudy"):
follow_objects.append(group.editors_of_readerstudy)
elif hasattr(group, "admins_of_challenge"):
# NOTE: only admins of a challenge should follow a challenge
# and its phases
follow_objects.append(group.admins_of_challenge)
for phase in group.admins_of_challenge.phase_set.all():
follow_objects.append(phase)
for user in users:
for obj in follow_objects:
if action == "post_add" and obj._meta.model_name != "algorithm":
follow(
user=user, obj=obj, actor_only=False, send_action=False,
)
# only new admins of a challenge get notified
if obj._meta.model_name == "challenge":
Notification.send(
type=NotificationType.NotificationTypeChoices.NEW_ADMIN,
message="added as admin for",
action_object=user,
target=obj,
)
elif action == "post_add" and obj._meta.model_name == "algorithm":
follow(
user=user,
obj=obj,
actor_only=False,
flag="access_request",
send_action=False,
)
elif action == "pre_remove" or action == "pre_clear":
unfollow(user=user, obj=obj, send_action=False)
def _handle_raw_files(
*,
consumed_files: Set[Path],
file_errors: Dict[Path, List[str]],
base_directory: Path,
upload_session: RawImageUploadSession,
):
upload_session.import_result = {
"consumed_files":
[str(f.relative_to(base_directory)) for f in consumed_files],
"file_errors": {
str(k.relative_to(base_directory)): v
for k, v in file_errors.items() if k not in consumed_files
},
}
if upload_session.import_result["file_errors"]:
n_errors = len(upload_session.import_result["file_errors"])
upload_session.error_message = (
f"{n_errors} file{pluralize(n_errors)} could not be imported")
if upload_session.creator:
Notification.send(
type=NotificationType.NotificationTypeChoices.
IMAGE_IMPORT_STATUS,
message=f"failed with {n_errors} error{pluralize(n_errors)}",
action_object=upload_session,
)
def process_access_request(request_object):
if (request_object.base_object.access_request_handling ==
AccessRequestHandlingOptions.ACCEPT_ALL):
# immediately allow access, no need for a notification
request_object.status = request_object.ACCEPTED
request_object.save()
return
elif (request_object.base_object.access_request_handling ==
AccessRequestHandlingOptions.ACCEPT_VERIFIED_USERS):
try:
if request_object.user.verification.is_verified:
# immediately allow access, no need for a notification
request_object.status = request_object.ACCEPTED
request_object.save()
return
except ObjectDoesNotExist:
pass
follow(
user=request_object.user,
obj=request_object,
actor_only=False,
send_action=False,
)
Notification.send(
type=NotificationType.NotificationTypeChoices.ACCESS_REQUEST,
message="requested access to",
actor=request_object.user,
target=request_object.base_object,
)
def create_topic_notification(sender, *, instance, created, **_):
if created:
follow(
user=instance.poster,
obj=instance,
actor_only=False,
send_action=False,
)
if int(instance.type) == int(Topic.TOPIC_ANNOUNCE):
Notification.send(
type=NotificationType.NotificationTypeChoices.FORUM_POST,
actor=instance.poster,
message="announced",
action_object=instance,
target=instance.forum,
context_class="info",
)
else:
Notification.send(
type=NotificationType.NotificationTypeChoices.FORUM_POST,
actor=instance.poster,
message="posted",
action_object=instance,
target=instance.forum,
)
def create_algorithm_jobs_for_session(*, upload_session_pk,
algorithm_image_pk):
session = RawImageUploadSession.objects.get(pk=upload_session_pk)
algorithm_image = AlgorithmImage.objects.get(pk=algorithm_image_pk)
# Editors group should be able to view session jobs for debugging
algorithm_editors = [algorithm_image.algorithm.editors_group]
# Send an email to the algorithm editors and creator on job failure
task_on_success = send_failed_session_jobs_notifications.signature(
kwargs={
"session_pk": str(session.pk),
"algorithm_pk": str(algorithm_image.algorithm.pk),
},
immutable=True,
)
default_input_interface = ComponentInterface.objects.get(
slug=DEFAULT_INPUT_INTERFACE_SLUG)
with transaction.atomic():
civ_sets = [{
ComponentInterfaceValue.objects.create(
interface=default_input_interface, image=image)
} for image in session.image_set.all()]
new_jobs = create_algorithm_jobs(
algorithm_image=algorithm_image,
civ_sets=civ_sets,
creator=session.creator,
extra_viewer_groups=algorithm_editors,
extra_logs_viewer_groups=algorithm_editors,
task_on_success=task_on_success,
)
unscheduled_jobs = len(civ_sets) - len(new_jobs)
if session.creator is not None and unscheduled_jobs:
experiment_url = reverse(
"algorithms:execution-session-detail",
kwargs={
"slug": algorithm_image.algorithm.slug,
"pk": upload_session_pk,
},
)
Notification.send(
type=NotificationType.NotificationTypeChoices.JOB_STATUS,
actor=session.creator,
message=
f"Unfortunately {unscheduled_jobs} of the jobs for algorithm "
f"{algorithm_image.algorithm.title} were not started because "
f"the number of allowed jobs was reached.",
target=algorithm_image.algorithm,
description=experiment_url,
)
def save(self, *args, **kwargs):
adding = self._state.adding
super().save(*args, **kwargs)
if adding:
follow(
user=self.user, obj=self, actor_only=False, send_action=False,
)
Notification.send(
type=NotificationType.NotificationTypeChoices.ACCESS_REQUEST,
message="requested access to",
actor=self.user,
target=self.base_object,
)
def create_post_notification(sender, *, instance, created, **_):
if (created and instance.topic.posts_count != 0
and not instance.is_topic_head):
follow(
user=instance.poster,
obj=instance.topic,
actor_only=False,
send_action=False,
)
Notification.send(
type=NotificationType.NotificationTypeChoices.FORUM_POST_REPLY,
actor=instance.poster,
message="replied to",
target=instance.topic,
)
def send_failed_job_notification(*, job_pk):
job = Job.objects.get(pk=job_pk)
if job.status == Job.FAILURE and job.creator is not None:
algorithm = job.algorithm_image.algorithm
experiment_url = reverse("algorithms:job-list",
kwargs={"slug": algorithm.slug})
Notification.send(
type=NotificationType.NotificationTypeChoices.JOB_STATUS,
actor=job.creator,
message=
f"Unfortunately one of the jobs for algorithm {algorithm.title} "
f"failed with an error",
target=algorithm,
description=experiment_url,
)
def send_failed_session_jobs_notifications(*, session_pk, algorithm_pk):
session = RawImageUploadSession.objects.get(pk=session_pk)
algorithm = Algorithm.objects.get(pk=algorithm_pk)
queryset = Job.objects.filter(
inputs__image__in=session.image_set.all()).distinct()
pending_jobs = queryset.exclude(
status__in=[Job.SUCCESS, Job.FAILURE, Job.CANCELLED])
failed_jobs = queryset.filter(status=Job.FAILURE)
if pending_jobs.exists():
# Nothing to do
return
elif session.creator is not None:
# TODO this task isn't really idempotent
# This task is not guaranteed to only be delivered once after
# all jobs have completed. We could end up in a situation where
# this is run multiple times after the action is sent and
# multiple notifications sent with the same message.
# We cannot really check if the action has already been sent
# which would then reduce this down to a race condition, but
# still a problem.
# We could of course just notify on each failure, but then
# this should be an on_error task for each job.
failed_jobs_count = failed_jobs.count()
if failed_jobs_count:
experiment_url = reverse(
"algorithms:execution-session-detail",
kwargs={
"slug": algorithm.slug,
"pk": session_pk
},
)
Notification.send(
type=NotificationType.NotificationTypeChoices.JOB_STATUS,
actor=session.creator,
message=f"Unfortunately {failed_jobs_count} of the jobs for "
f"algorithm {algorithm.title} failed with an error ",
target=algorithm,
description=experiment_url,
)
def update_status(self, *args, **kwargs):
res = super().update_status(*args, **kwargs)
if self.status == self.FAILURE:
Notification.send(
type=NotificationType.NotificationTypeChoices.EVALUATION_STATUS,
actor=self.submission.creator,
message="failed",
action_object=self,
target=self.submission.phase,
)
if self.status == self.SUCCESS:
Notification.send(
type=NotificationType.NotificationTypeChoices.EVALUATION_STATUS,
actor=self.submission.creator,
message="succeeded",
action_object=self,
target=self.submission.phase,
)
return res
def process_permission_request_update(sender, instance, *_, **__):
try:
old_values = sender.objects.get(pk=instance.pk)
except ObjectDoesNotExist:
old_values = None
old_status = old_values.status if old_values else None
if instance.status != old_status:
if instance.status == instance.ACCEPTED:
instance.add_method(instance.user)
Notification.send(
type=NotificationType.NotificationTypeChoices.REQUEST_UPDATE,
message="was accepted",
target=instance,
)
elif instance.status == instance.REJECTED:
instance.remove_method(instance.user)
Notification.send(
type=NotificationType.NotificationTypeChoices.REQUEST_UPDATE,
message="was rejected",
target=instance,
)
def process_registration(instance: RegistrationRequest = None,
created: bool = False,
*_,
**__):
if created and not instance.challenge.require_participant_review:
instance.status = RegistrationRequest.ACCEPTED
RegistrationRequest.objects.filter(pk=instance.pk).update(
status=instance.status)
elif created and instance.challenge.require_participant_review:
Notification.send(
type=NotificationType.NotificationTypeChoices.ACCESS_REQUEST,
message="requested access to",
actor=instance.user,
target=instance.challenge,
)
if not is_following(instance.user, instance):
follow(
user=instance.user,
obj=instance,
actor_only=False,
send_action=False,
)
if instance.status == RegistrationRequest.ACCEPTED:
instance.challenge.add_participant(instance.user)
Notification.send(
type=NotificationType.NotificationTypeChoices.REQUEST_UPDATE,
message="was approved",
target=instance,
)
elif instance.status == RegistrationRequest.REJECTED:
instance.challenge.remove_participant(instance.user)
Notification.send(
type=NotificationType.NotificationTypeChoices.REQUEST_UPDATE,
message="was rejected",
target=instance,
)
def create_evaluation(*, submission_pk, max_initial_jobs=1):
"""
Creates an Evaluation for a Submission
Parameters
----------
submission_pk
The primary key of the Submission
max_initial_jobs
The maximum number of algorithm jobs to schedule first
"""
Submission = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Submission")
Evaluation = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Evaluation")
submission = Submission.objects.get(pk=submission_pk)
if not submission.predictions_file and submission.user_upload:
with transaction.atomic():
submission.user_upload.copy_object(
to_field=submission.predictions_file)
submission.user_upload.delete()
# TODO - move this to the form and make it an input here
method = submission.latest_ready_method
if not method:
logger.info("No method ready for this submission")
Notification.send(
type=NotificationType.NotificationTypeChoices.MISSING_METHOD,
message="missing method",
actor=submission.creator,
action_object=submission,
target=submission.phase,
)
return
evaluation, created = Evaluation.objects.get_or_create(
submission=submission, method=method)
if not created:
logger.info("Evaluation already created for this submission")
return
if submission.algorithm_image:
on_commit(lambda: create_algorithm_jobs_for_evaluation.apply_async(
kwargs={
"evaluation_pk": evaluation.pk,
"max_jobs": max_initial_jobs,
}))
elif submission.predictions_file:
mimetype = get_file_mimetype(submission.predictions_file)
if mimetype == "application/zip":
interface = ComponentInterface.objects.get(
slug="predictions-zip-file")
elif mimetype in ["text/plain", "application/csv"]:
interface = ComponentInterface.objects.get(
slug="predictions-csv-file")
else:
evaluation.update_status(
status=Evaluation.FAILURE,
stderr=f"{mimetype} files are not supported.",
error_message=f"{mimetype} files are not supported.",
)
return
civ = ComponentInterfaceValue(interface=interface,
file=submission.predictions_file)
civ.full_clean()
civ.save()
evaluation.inputs.set([civ])
on_commit(evaluation.execute)
else:
raise RuntimeError("No algorithm or predictions file found")