class ChallengePhase(TimeStampedModel):
"""Model representing a Challenge Phase"""
def __init__(self, *args, **kwargs):
super(ChallengePhase, self).__init__(*args, **kwargs)
self._original_test_annotation = self.test_annotation
name = models.CharField(max_length=100, db_index=True)
description = models.TextField()
leaderboard_public = models.BooleanField(default=False)
start_date = models.DateTimeField(null=True,
blank=True,
verbose_name="Start Date (UTC)",
db_index=True)
end_date = models.DateTimeField(null=True,
blank=True,
verbose_name="End Date (UTC)",
db_index=True)
challenge = models.ForeignKey("Challenge", on_delete=models.CASCADE)
is_public = models.BooleanField(default=False)
is_submission_public = models.BooleanField(default=False)
test_annotation = models.FileField(
upload_to=RandomFileName("test_annotations"), null=True, blank=True)
max_submissions_per_day = models.PositiveIntegerField(default=100000,
db_index=True)
max_submissions_per_month = models.PositiveIntegerField(default=100000,
db_index=True)
max_submissions = models.PositiveIntegerField(default=100000,
db_index=True)
max_concurrent_submissions_allowed = models.PositiveIntegerField(default=3)
codename = models.CharField(max_length=100, default="Phase Code Name")
dataset_split = models.ManyToManyField(DatasetSplit,
blank=True,
through="ChallengePhaseSplit")
allowed_email_ids = ArrayField(
models.TextField(null=True, blank=True),
default=list,
blank=True,
null=True,
)
slug = models.SlugField(max_length=200, null=True, unique=True)
environment_image = models.CharField(
max_length=2128, null=True, blank=True
) # Max length of repository name and tag is 2000 and 128 respectively
allowed_submission_file_types = models.CharField(
max_length=200,
default=".json, .zip, .txt, .tsv, .gz, .csv, .h5, .npy")
# Flag to restrict user to select only one submission for leaderboard
is_restricted_to_select_one_submission = models.BooleanField(default=False)
# Store the schema for the submission meta attributes of this challenge phase.
submission_meta_attributes = JSONField(default=None, blank=True, null=True)
# Flag to allow reporting partial metrics for submission evaluation
is_partial_submission_evaluation_enabled = models.BooleanField(
default=False)
# Id in the challenge config file. Needed to map the object to the value in the config file while updating through Github
config_id = models.IntegerField(default=None, blank=True, null=True)
# Store the default metadata for a submission meta attributes of a challenge phase.
default_submission_meta_attributes = JSONField(default=None,
blank=True,
null=True)
class Meta:
app_label = "challenges"
db_table = "challenge_phase"
unique_together = (("codename", "challenge"), )
def __str__(self):
"""Returns the name of Phase"""
return self.name
def get_start_date(self):
"""Returns the start date of Phase"""
return self.start_date
def get_end_date(self):
"""Returns the end date of Challenge"""
return self.end_date
@property
def is_active(self):
"""Returns if the challenge is active or not"""
if self.start_date < timezone.now() and self.end_date > timezone.now():
return True
return False
def save(self, *args, **kwargs):
# If the max_submissions_per_day is less than the max_concurrent_submissions_allowed.
if (self.max_submissions_per_day <
self.max_concurrent_submissions_allowed):
self.max_concurrent_submissions_allowed = (
self.max_submissions_per_day)
challenge_phase_instance = super(ChallengePhase,
self).save(*args, **kwargs)
return challenge_phase_instance
class Challenge(TimeStampedModel):
"""Model representing a hosted Challenge"""
def __init__(self, *args, **kwargs):
super(Challenge, self).__init__(*args, **kwargs)
self._original_evaluation_script = self.evaluation_script
title = models.CharField(max_length=100, db_index=True)
short_description = models.TextField(null=True, blank=True)
description = models.TextField(null=True, blank=True)
terms_and_conditions = models.TextField(null=True, blank=True)
submission_guidelines = models.TextField(null=True, blank=True)
evaluation_details = models.TextField(null=True, blank=True)
image = models.ImageField(
upload_to=RandomFileName('logos'), null=True, blank=True, verbose_name="Logo")
start_date = models.DateTimeField(
null=True, blank=True, verbose_name="Start Date (UTC)", db_index=True)
end_date = models.DateTimeField(
null=True, blank=True, verbose_name="End Date (UTC)", db_index=True)
creator = models.ForeignKey(
'hosts.ChallengeHostTeam', related_name='challenge_creator')
published = models.BooleanField(
default=False, verbose_name="Publicly Available", db_index=True)
enable_forum = models.BooleanField(default=True)
forum_url = models.URLField(max_length=100, blank=True, null=True)
anonymous_leaderboard = models.BooleanField(default=False)
participant_teams = models.ManyToManyField(ParticipantTeam, blank=True)
is_disabled = models.BooleanField(default=False, db_index=True)
evaluation_script = models.FileField(
default=False, upload_to=RandomFileName("evaluation_scripts")) # should be zip format
approved_by_admin = models.BooleanField(
default=False, verbose_name="Approved By Admin", db_index=True)
featured = models.BooleanField(
default=False, verbose_name="Featured", db_index=True)
allowed_email_domains = ArrayField(
models.CharField(max_length=50, blank=True),
default=[], blank=True)
blocked_email_domains = ArrayField(
models.CharField(max_length=50, blank=True),
default=[], blank=True)
class Meta:
app_label = 'challenges'
db_table = 'challenge'
def __str__(self):
"""Returns the title of Challenge"""
return self.title
def get_image_url(self):
"""Returns the url of logo of Challenge"""
if self.image:
return self.image.url
return None
def get_evaluation_script_path(self):
"""Returns the path of evaluation script"""
if self.evaluation_script:
return self.evaluation_script.url
return None
def get_start_date(self):
"""Returns the start date of Challenge"""
return self.start_date
def get_end_date(self):
"""Returns the end date of Challenge"""
return self.end_date
@property
def is_active(self):
"""Returns if the challenge is active or not"""
if self.start_date < timezone.now() and self.end_date > timezone.now():
return True
return False
class Challenge(TimeStampedModel):
"""Model representing a hosted Challenge"""
def __init__(self, *args, **kwargs):
super(Challenge, self).__init__(*args, **kwargs)
self._original_evaluation_script = self.evaluation_script
self._original_approved_by_admin = self.approved_by_admin
title = models.CharField(max_length=100, db_index=True)
short_description = models.TextField(null=True, blank=True)
description = models.TextField(null=True, blank=True)
terms_and_conditions = models.TextField(null=True, blank=True)
submission_guidelines = models.TextField(null=True, blank=True)
evaluation_details = models.TextField(null=True, blank=True)
image = models.ImageField(
upload_to=RandomFileName("logos"),
null=True,
blank=True,
verbose_name="Logo",
)
start_date = models.DateTimeField(null=True,
blank=True,
verbose_name="Start Date (UTC)",
db_index=True)
end_date = models.DateTimeField(null=True,
blank=True,
verbose_name="End Date (UTC)",
db_index=True)
creator = models.ForeignKey("hosts.ChallengeHostTeam",
related_name="challenge_creator")
published = models.BooleanField(default=False,
verbose_name="Publicly Available",
db_index=True)
is_registration_open = models.BooleanField(default=True)
enable_forum = models.BooleanField(default=True)
forum_url = models.URLField(max_length=100, blank=True, null=True)
leaderboard_description = models.TextField(null=True, blank=True)
anonymous_leaderboard = models.BooleanField(default=False)
participant_teams = models.ManyToManyField(ParticipantTeam, blank=True)
is_disabled = models.BooleanField(default=False, db_index=True)
evaluation_script = models.FileField(
default=False,
upload_to=RandomFileName("evaluation_scripts")) # should be zip format
approved_by_admin = models.BooleanField(default=False,
verbose_name="Approved By Admin",
db_index=True)
featured = models.BooleanField(default=False,
verbose_name="Featured",
db_index=True)
allowed_email_domains = ArrayField(models.CharField(max_length=50,
blank=True),
default=[],
blank=True)
blocked_email_domains = ArrayField(models.CharField(max_length=50,
blank=True),
default=[],
blank=True)
banned_email_ids = ArrayField(
models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True,
)
remote_evaluation = models.BooleanField(default=False,
verbose_name="Remote Evaluation",
db_index=True)
queue = models.CharField(
max_length=200,
default="",
verbose_name="SQS queue name",
db_index=True,
)
is_docker_based = models.BooleanField(default=False,
verbose_name="Is Docker Based",
db_index=True)
slug = models.SlugField(max_length=200, null=True, unique=True)
max_docker_image_size = models.BigIntegerField(
default=42949672960, null=True, blank=True) # Default is 40 GB
max_concurrent_submission_evaluation = models.PositiveIntegerField(
default=100000)
aws_account_id = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_access_key_id = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_secret_access_key = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_region = models.CharField(max_length=50,
default="us-east-1",
null=True,
blank=True)
use_host_credentials = models.BooleanField(default=False)
cli_version = models.CharField(max_length=20,
verbose_name="evalai-cli version",
null=True,
blank=True)
# The number of active workers on Fargate of the challenge.
workers = models.IntegerField(null=True, blank=True, default=None)
# The task definition ARN for the challenge, used for updating and creating service.
task_def_arn = models.CharField(null=True,
blank=True,
max_length=2048,
default="")
slack_webhook_url = models.URLField(max_length=200, blank=True, null=True)
class Meta:
app_label = "challenges"
db_table = "challenge"
def __str__(self):
"""Returns the title of Challenge"""
return self.title
def get_image_url(self):
"""Returns the url of logo of Challenge"""
if self.image:
return self.image.url
return None
def get_evaluation_script_path(self):
"""Returns the path of evaluation script"""
if self.evaluation_script:
return self.evaluation_script.url
return None
def get_start_date(self):
"""Returns the start date of Challenge"""
return self.start_date
def get_end_date(self):
"""Returns the end date of Challenge"""
return self.end_date
@property
def is_active(self):
"""Returns if the challenge is active or not"""
if self.start_date < timezone.now() and self.end_date > timezone.now():
return True
return False
class ChallengePhase(TimeStampedModel):
"""Model representing a Challenge Phase"""
def __init__(self, *args, **kwargs):
super(ChallengePhase, self).__init__(*args, **kwargs)
self._original_test_annotation = self.test_annotation
name = models.CharField(max_length=100, db_index=True)
description = models.TextField()
leaderboard_public = models.BooleanField(default=False)
start_date = models.DateTimeField(null=True,
blank=True,
verbose_name="Start Date (UTC)",
db_index=True)
end_date = models.DateTimeField(null=True,
blank=True,
verbose_name="End Date (UTC)",
db_index=True)
challenge = models.ForeignKey("Challenge")
is_public = models.BooleanField(default=False)
is_submission_public = models.BooleanField(default=False)
test_annotation = models.FileField(
upload_to=RandomFileName("test_annotations"), default=False)
max_submissions_per_day = models.PositiveIntegerField(default=100000,
db_index=True)
max_submissions_per_month = models.PositiveIntegerField(default=100000,
db_index=True)
max_submissions = models.PositiveIntegerField(default=100000,
db_index=True)
max_concurrent_submissions_allowed = models.PositiveIntegerField(default=3)
codename = models.CharField(max_length=100, default="Phase Code Name")
dataset_split = models.ManyToManyField(DatasetSplit,
blank=True,
through="ChallengePhaseSplit")
allowed_email_ids = ArrayField(
models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True,
)
slug = models.SlugField(max_length=200, null=True, unique=True)
class Meta:
app_label = "challenges"
db_table = "challenge_phase"
unique_together = (("codename", "challenge"), )
def __str__(self):
"""Returns the name of Phase"""
return self.name
def get_start_date(self):
"""Returns the start date of Phase"""
return self.start_date
def get_end_date(self):
"""Returns the end date of Challenge"""
return self.end_date
@property
def is_active(self):
"""Returns if the challenge is active or not"""
if self.start_date < timezone.now() and self.end_date > timezone.now():
return True
return False
def save(self, *args, **kwargs):
# If the max_submissions_per_day is less than the max_concurrent_submissions_allowed.
if (self.max_submissions_per_day <
self.max_concurrent_submissions_allowed):
self.max_concurrent_submissions_allowed = (
self.max_submissions_per_day)
challenge_phase_instance = super(ChallengePhase,
self).save(*args, **kwargs)
return challenge_phase_instance
class Submission(TimeStampedModel):
SUBMITTED = "submitted"
RUNNING = "running"
FAILED = "failed"
CANCELLED = "cancelled"
FINISHED = "finished"
SUBMITTING = "submitting"
STATUS_OPTIONS = (
(SUBMITTED, SUBMITTED),
(RUNNING, RUNNING),
(FAILED, FAILED),
(CANCELLED, CANCELLED),
(FINISHED, FINISHED),
(SUBMITTING, SUBMITTING),
)
participant_team = models.ForeignKey(ParticipantTeam,
related_name='submissions')
challenge_phase = models.ForeignKey(ChallengePhase,
related_name='submissions')
created_by = models.ForeignKey(User)
status = models.CharField(max_length=30,
choices=STATUS_OPTIONS,
db_index=True)
is_public = models.BooleanField(default=False)
is_flagged = models.BooleanField(default=False)
submission_number = models.PositiveIntegerField(default=0)
download_count = models.IntegerField(default=0)
output = models.TextField(blank=True, null=True)
submitted_at = models.DateTimeField(auto_now_add=True, db_index=True)
started_at = models.DateTimeField(null=True, blank=True, db_index=True)
completed_at = models.DateTimeField(null=True, blank=True, db_index=True)
when_made_public = models.DateTimeField(null=True, blank=True)
input_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"))
stdout_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
stderr_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
submission_result_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
submission_metadata_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
execution_time_limit = models.PositiveIntegerField(default=300,
db_index=True)
method_name = models.CharField(max_length=1000, null=True, db_index=True)
method_description = models.TextField(blank=True, null=True, db_index=True)
publication_url = models.CharField(max_length=1000,
null=True,
db_index=True)
project_url = models.CharField(max_length=1000, null=True, db_index=True)
def __unicode__(self):
return '{}'.format(self.id)
class Meta:
app_label = 'jobs'
db_table = 'submission'
@property
def execution_time(self):
"""Returns the execution time of a submission"""
# if self.self.completed_at and self.started_at:
try:
return (self.completed_at - self.started_at).total_seconds()
except:
return "None"
# else:
# return None
def save(self, *args, **kwargs):
# Only save the completed_at time when the object is created.
# Dont update the completed_at field at the time of updating submission meta-data or status.
if self._state.adding is True:
if hasattr(self, 'status'):
if self.status == Submission.RUNNING:
self.started_at = timezone.now()
if self.status == Submission.FINISHED:
self.completed_at = timezone.now()
if not self.pk:
sub_num = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team).aggregate(
Max('submission_number'))['submission_number__max']
if sub_num:
self.submission_number = sub_num + 1
else:
self.submission_number = 1
failed_count = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
status=Submission.FAILED).count()
successful_count = self.submission_number - failed_count
if successful_count > self.challenge_phase.max_submissions:
logger.info(
"Checking to see if the successful_count {0} is greater than maximum allowed {1}"
.format(successful_count,
self.challenge_phase.max_submissions))
logger.info(
"The submission request is submitted by user {0} from participant_team {1} "
.format(self.created_by.pk, self.participant_team.pk))
raise PermissionDenied({
'error':
'The maximum number of submissions has been reached'
})
else:
logger.info(
"Submission is below for user {0} form participant_team {1} for challenge_phase {2}"
.format(self.created_by.pk, self.participant_team.pk,
self.challenge_phase.pk))
if hasattr(self.challenge_phase, 'max_submissions_per_day'):
submissions_done_today_count = Submission.objects.filter(
challenge_phase__challenge=self.challenge_phase.challenge,
participant_team=self.participant_team,
challenge_phase=self.challenge_phase,
submitted_at__gte=datetime.date.today()).count()
failed_count = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
status=Submission.FAILED,
submitted_at__gte=datetime.date.today()).count()
if ((submissions_done_today_count + 1 - failed_count >
self.challenge_phase.max_submissions_per_day) or
(self.challenge_phase.max_submissions_per_day == 0)):
logger.info(
"Permission Denied: The maximum number of submission for today has been reached"
)
raise PermissionDenied({
'error':
'The maximum number of submission for today has been reached'
})
self.is_public = (True if self.challenge_phase.is_submission_public
else False)
self.status = Submission.SUBMITTED
submission_instance = super(Submission, self).save(*args, **kwargs)
return submission_instance
@cached_property
def submissions_count_for_challenge_phase(self):
# To get total submissions count on a challenge phase
submission_count = Submission.objects.filter(
challenge_phase=self.challenge_phase,
challenge_phase__challenge=self.challenge_phase.challenge).count()
return submission_count
@cached_property
def participated_teams_count_for_challenge_phase(self):
# To get total participant teams count on a challenge phase
submission_count = Submission.objects.filter(
challenge_phase=self.challenge_phase,
challenge_phase__challenge=self.challenge_phase.challenge)
submission_count = submission_count.values_list(
'participant_team', flat=True).distinct().count()
return submission_count
@cached_property
def last_submission_timestamp_in_challenge_phase(self):
# To get the last submission time in a challenge phase
last_submitted_timestamp = Submission.objects.filter(
challenge_phase=self.challenge_phase,
challenge_phase__challenge=self.challenge_phase.challenge)
last_submitted_timestamp = last_submitted_timestamp.order_by(
'-submitted_at')[0].created_at
return last_submitted_timestamp
@cached_property
def last_submission_timestamp_in_challenge(self):
# To get the last submission time in a challenge
last_submitted_timestamp = Submission.objects.filter(
challenge_phase__challenge=self.challenge_phase.challenge)
last_submitted_timestamp = last_submitted_timestamp.order_by(
'-submitted_at')[0].created_at
return last_submitted_timestamp
class Submission(TimeStampedModel):
SUBMITTED = "submitted"
RUNNING = "running"
FAILED = "failed"
CANCELLED = "cancelled"
FINISHED = "finished"
SUBMITTING = "submitting"
ARCHIVED = "archived"
PARTIALLY_EVALUATED = "partially_evaluated"
STATUS_OPTIONS = (
(SUBMITTED, SUBMITTED),
(RUNNING, RUNNING),
(FAILED, FAILED),
(CANCELLED, CANCELLED),
(FINISHED, FINISHED),
(SUBMITTING, SUBMITTING),
(ARCHIVED, ARCHIVED),
(PARTIALLY_EVALUATED, PARTIALLY_EVALUATED),
)
participant_team = models.ForeignKey(ParticipantTeam,
related_name="submissions")
challenge_phase = models.ForeignKey(ChallengePhase,
related_name="submissions")
created_by = models.ForeignKey(User)
status = models.CharField(max_length=30,
choices=STATUS_OPTIONS,
db_index=True)
is_public = models.BooleanField(default=True)
is_flagged = models.BooleanField(default=False)
submission_number = models.PositiveIntegerField(default=0)
download_count = models.IntegerField(default=0)
output = models.TextField(blank=True, null=True)
submitted_at = models.DateTimeField(auto_now_add=True, db_index=True)
started_at = models.DateTimeField(null=True, blank=True, db_index=True)
completed_at = models.DateTimeField(null=True, blank=True, db_index=True)
when_made_public = models.DateTimeField(null=True, blank=True)
# Model to store submitted submission files by the user
input_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"))
# Model to store large submission file (> 400 MB's) URLs submitted by the user
input_file_url = models.URLField(max_length=1000, null=True, blank=True)
stdout_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
stderr_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
submission_result_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
submission_metadata_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True,
)
execution_time_limit = models.PositiveIntegerField(default=300)
method_name = models.CharField(max_length=1000,
default="",
db_index=True,
blank=True)
method_description = models.TextField(blank=True, default="")
publication_url = models.CharField(max_length=1000, default="", blank=True)
project_url = models.CharField(max_length=1000, default="", blank=True)
is_baseline = models.BooleanField(default=False)
job_name = ArrayField(
models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True,
)
ignore_submission = models.BooleanField(default=False)
# Store the values of meta attributes for the submission here.
submission_metadata = JSONField(blank=True, null=True)
def __str__(self):
return "{}".format(self.id)
class Meta:
app_label = "jobs"
db_table = "submission"
@property
def execution_time(self):
"""Returns the execution time of a submission"""
# if self.self.completed_at and self.started_at:
try:
return (self.completed_at - self.started_at).total_seconds()
except: # noqa: E722
return "None"
# else:
# return None
def save(self, *args, **kwargs):
if not self.pk:
sub_num = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
).aggregate(Max("submission_number"))["submission_number__max"]
if sub_num:
self.submission_number = sub_num + 1
else:
self.submission_number = 1
submissions = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
)
num_submissions_to_ignore = submissions.filter(
status__in=submission_status_to_exclude).count()
successful_count = (self.submission_number -
num_submissions_to_ignore)
if successful_count > self.challenge_phase.max_submissions:
logger.info(
"Checking to see if the successful_count {0} is greater than maximum allowed {1}"
.format(successful_count,
self.challenge_phase.max_submissions))
logger.info(
"The submission request is submitted by user {0} from participant_team {1} "
.format(self.created_by.pk, self.participant_team.pk))
raise PermissionDenied({
"error":
"The maximum number of submissions has been reached"
})
else:
logger.info(
"Submission is below for user {0} form participant_team {1} for challenge_phase {2}"
.format(
self.created_by.pk,
self.participant_team.pk,
self.challenge_phase.pk,
))
total_submissions_done = Submission.objects.filter(
challenge_phase__challenge=self.challenge_phase.challenge,
participant_team=self.participant_team,
challenge_phase=self.challenge_phase,
)
submissions_done_today_count = (total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
hour=0, minute=0, second=0, microsecond=0)).exclude(
status__in=submission_status_to_exclude).count())
submissions_done_in_month_count = (total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)).exclude(
status__in=submission_status_to_exclude).count())
if (self.challenge_phase.max_submissions_per_month -
submissions_done_in_month_count == 0):
logger.info(
"Permission Denied: The maximum number of submission for this month has been reached"
)
raise PermissionDenied({
"error":
"The maximum number of submission for this month has been reached"
})
if (self.challenge_phase.max_submissions_per_day -
submissions_done_today_count == 0):
logger.info(
"Permission Denied: The maximum number of submission for today has been reached"
)
raise PermissionDenied({
"error":
"The maximum number of submission for today has been reached"
})
self.status = Submission.SUBMITTED
submission_instance = super(Submission, self).save(*args, **kwargs)
return submission_instance
class Submission(TimeStampedModel):
SUBMITTED = "submitted"
RUNNING = "running"
FAILED = "failed"
CANCELLED = "cancelled"
FINISHED = "finished"
SUBMITTING = "submitting"
STATUS_OPTIONS = (
(SUBMITTED, SUBMITTED),
(RUNNING, RUNNING),
(FAILED, FAILED),
(CANCELLED, CANCELLED),
(FINISHED, FINISHED),
(SUBMITTING, SUBMITTING),
)
participant_team = models.ForeignKey(ParticipantTeam,
related_name='submissions')
challenge_phase = models.ForeignKey(ChallengePhase,
related_name='submissions')
created_by = models.ForeignKey(User)
status = models.CharField(max_length=30,
choices=STATUS_OPTIONS,
db_index=True)
is_public = models.BooleanField(default=True)
is_flagged = models.BooleanField(default=False)
submission_number = models.PositiveIntegerField(default=0)
download_count = models.IntegerField(default=0)
output = models.TextField(blank=True, null=True)
submitted_at = models.DateTimeField(auto_now_add=True, db_index=True)
started_at = models.DateTimeField(null=True, blank=True, db_index=True)
completed_at = models.DateTimeField(null=True, blank=True, db_index=True)
when_made_public = models.DateTimeField(null=True, blank=True)
input_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"))
stdout_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
stderr_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
submission_result_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
submission_metadata_file = models.FileField(
upload_to=RandomFileName("submission_files/submission_{id}"),
null=True,
blank=True)
execution_time_limit = models.PositiveIntegerField(default=300)
method_name = models.CharField(max_length=1000,
default="",
db_index=True,
blank=True)
method_description = models.TextField(blank=True, default="")
publication_url = models.CharField(max_length=1000, default="", blank=True)
project_url = models.CharField(max_length=1000, default="", blank=True)
def __str__(self):
return '{}'.format(self.id)
class Meta:
app_label = 'jobs'
db_table = 'submission'
@property
def execution_time(self):
"""Returns the execution time of a submission"""
# if self.self.completed_at and self.started_at:
try:
return (self.completed_at - self.started_at).total_seconds()
except:
return "None"
# else:
# return None
def save(self, *args, **kwargs):
if not self.pk:
sub_num = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team).aggregate(
Max('submission_number'))['submission_number__max']
if sub_num:
self.submission_number = sub_num + 1
else:
self.submission_number = 1
failed_count = Submission.objects.filter(
challenge_phase=self.challenge_phase,
participant_team=self.participant_team,
status=Submission.FAILED).count()
successful_count = self.submission_number - failed_count
if successful_count > self.challenge_phase.max_submissions:
logger.info(
"Checking to see if the successful_count {0} is greater than maximum allowed {1}"
.format(successful_count,
self.challenge_phase.max_submissions))
logger.info(
"The submission request is submitted by user {0} from participant_team {1} "
.format(self.created_by.pk, self.participant_team.pk))
raise PermissionDenied({
'error':
'The maximum number of submissions has been reached'
})
else:
logger.info(
"Submission is below for user {0} form participant_team {1} for challenge_phase {2}"
.format(self.created_by.pk, self.participant_team.pk,
self.challenge_phase.pk))
total_submissions_done = Submission.objects.filter(
challenge_phase__challenge=self.challenge_phase.challenge,
participant_team=self.participant_team,
challenge_phase=self.challenge_phase,
)
submissions_done_today_count = total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
hour=0, minute=0, second=0, microsecond=0)).exclude(
status=Submission.FAILED).count()
submissions_done_in_month_count = total_submissions_done.filter(
submitted_at__gte=timezone.now().replace(
day=1, hour=0, minute=0, second=0, microsecond=0)).exclude(
status=Submission.FAILED).count()
if self.challenge_phase.max_submissions_per_month - submissions_done_in_month_count == 0:
logger.info(
'Permission Denied: The maximum number of submission for this month has been reached'
)
raise PermissionDenied({
'error':
'The maximum number of submission for this month has been reached'
})
if self.challenge_phase.max_submissions_per_day - submissions_done_today_count == 0:
logger.info(
'Permission Denied: The maximum number of submission for today has been reached'
)
raise PermissionDenied({
'error':
'The maximum number of submission for today has been reached'
})
self.is_public = (True if self.challenge_phase.is_submission_public
else False)
self.status = Submission.SUBMITTED
submission_instance = super(Submission, self).save(*args, **kwargs)
return submission_instance
def test_random_file_name_without_id(self):
obj = RandomFileName("evaluation_scripts")
filepath = obj.__call__(self.challenge, self.test_file_path)
expected = "evaluation_scripts/{}".format(filepath.split('/')[1])
self.assertEqual(filepath, expected)
class Challenge(TimeStampedModel):
"""Model representing a hosted Challenge"""
def __init__(self, *args, **kwargs):
super(Challenge, self).__init__(*args, **kwargs)
self._original_evaluation_script = self.evaluation_script
title = models.CharField(max_length=100, db_index=True)
short_description = models.TextField(null=True, blank=True)
description = models.TextField(null=True, blank=True)
terms_and_conditions = models.TextField(null=True, blank=True)
submission_guidelines = models.TextField(null=True, blank=True)
evaluation_details = models.TextField(null=True, blank=True)
image = models.ImageField(
upload_to=RandomFileName("logos"),
null=True,
blank=True,
verbose_name="Logo",
)
start_date = models.DateTimeField(null=True,
blank=True,
verbose_name="Start Date (UTC)",
db_index=True)
end_date = models.DateTimeField(null=True,
blank=True,
verbose_name="End Date (UTC)",
db_index=True)
creator = models.ForeignKey("hosts.ChallengeHostTeam",
related_name="challenge_creator")
published = models.BooleanField(default=False,
verbose_name="Publicly Available",
db_index=True)
is_registration_open = models.BooleanField(default=True)
enable_forum = models.BooleanField(default=True)
forum_url = models.URLField(max_length=100, blank=True, null=True)
leaderboard_description = models.TextField(null=True, blank=True)
anonymous_leaderboard = models.BooleanField(default=False)
participant_teams = models.ManyToManyField(ParticipantTeam, blank=True)
is_disabled = models.BooleanField(default=False, db_index=True)
evaluation_script = models.FileField(
default=False,
upload_to=RandomFileName("evaluation_scripts")) # should be zip format
approved_by_admin = models.BooleanField(default=False,
verbose_name="Approved By Admin",
db_index=True)
featured = models.BooleanField(default=False,
verbose_name="Featured",
db_index=True)
allowed_email_domains = ArrayField(models.CharField(max_length=50,
blank=True),
default=[],
blank=True)
blocked_email_domains = ArrayField(models.CharField(max_length=50,
blank=True),
default=[],
blank=True)
banned_email_ids = ArrayField(models.TextField(null=True, blank=True),
default=[],
blank=True,
null=True)
remote_evaluation = models.BooleanField(default=False,
verbose_name="Remote Evaluation",
db_index=True)
queue = models.CharField(
max_length=200,
default="",
verbose_name="SQS queue name",
db_index=True,
)
is_docker_based = models.BooleanField(default=False,
verbose_name="Is Docker Based",
db_index=True)
slug = models.SlugField(max_length=200, null=True, unique=True)
max_docker_image_size = models.BigIntegerField(
default=42949672960, null=True, blank=True) # Default is 40 GB
max_concurrent_submission_evaluation = models.PositiveIntegerField(
default=100000)
aws_account_id = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_access_key_id = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_secret_access_key = models.CharField(max_length=200,
default="",
null=True,
blank=True)
aws_region = models.CharField(max_length=50,
default="us-east-1",
null=True,
blank=True)
use_host_credentials = models.BooleanField(default=False)
cli_version = models.CharField(max_length=20,
verbose_name="evalai-cli version",
null=True,
blank=True)
# The number of active workers on Fargate of the challenge.
workers = models.IntegerField(null=True, blank=True, default=None)
# The task definition ARN for the challenge, used for updating and creating service.
task_def_arn = models.CharField(null=True,
blank=True,
max_length=2048,
default="")
class Meta:
app_label = "challenges"
db_table = "challenge"
def __str__(self):
"""Returns the title of Challenge"""
return self.title
def get_image_url(self):
"""Returns the url of logo of Challenge"""
if self.image:
return self.image.url
return None
def get_evaluation_script_path(self):
"""Returns the path of evaluation script"""
if self.evaluation_script:
return self.evaluation_script.url
return None
def get_start_date(self):
"""Returns the start date of Challenge"""
return self.start_date
def get_end_date(self):
"""Returns the end date of Challenge"""
return self.end_date
@property
def is_active(self):
"""Returns if the challenge is active or not"""
if self.start_date < timezone.now() and self.end_date > timezone.now():
return True
return False
@classmethod
def load_from_zip(cls, zip_file_path, challenge_host_team):
from .utils import get_file_content
from .serializers import (
ChallengePhaseCreateSerializer,
DatasetSplitSerializer,
LeaderboardSerializer,
ZipChallengePhaseSplitSerializer,
ZipChallengeSerializer,
)
zip_ref = zipfile.ZipFile(zip_file_path, "r")
BASE_LOCATION = tempfile.mkdtemp()
zip_ref.extractall(BASE_LOCATION)
zip_ref.close()
for name in zip_ref.namelist():
if (name.endswith(".yaml") or name.endswith(".yml")) and (
not name.startswith("__MACOSX")
): # Ignore YAML File in __MACOSX Directory
yaml_file = name
extracted_folder_name = yaml_file.split(basename(yaml_file))[0]
break
else:
raise Exception('No yaml file found in zip root!')
with open(join(BASE_LOCATION, yaml_file), "r") as stream:
yaml_file_data = yaml.safe_load(stream)
evaluation_script = yaml_file_data["evaluation_script"]
evaluation_script_path = join(
BASE_LOCATION,
extracted_folder_name,
evaluation_script,
)
# Check for evaluation script file in extracted zip folder.
with open(evaluation_script_path, "rb") as challenge_evaluation_script:
challenge_evaluation_script_file = ContentFile(
challenge_evaluation_script.read(), evaluation_script_path)
challenge_phases_data = yaml_file_data["challenge_phases"]
for data in challenge_phases_data:
test_annotation_file = data["test_annotation_file"]
test_annotation_file_path = join(
BASE_LOCATION,
extracted_folder_name,
test_annotation_file,
)
image = yaml_file_data.get("image")
if image and (image.endswith(".jpg") or image.endswith(".jpeg")
or image.endswith(".png")):
challenge_image_path = join(BASE_LOCATION, extracted_folder_name,
image)
if isfile(challenge_image_path):
challenge_image_file = ContentFile(
get_file_content(challenge_image_path, "rb"), image)
else:
challenge_image_file = None
else:
challenge_image_file = None
challenge_description_file_path = join(
BASE_LOCATION,
extracted_folder_name,
yaml_file_data["description"],
)
if challenge_description_file_path.endswith(".html") and isfile(
challenge_description_file_path):
yaml_file_data["description"] = get_file_content(
challenge_description_file_path, "rb").decode("utf-8")
challenge_evaluation_details_file_path = join(
BASE_LOCATION,
extracted_folder_name,
yaml_file_data["evaluation_details"],
)
if challenge_evaluation_details_file_path.endswith(".html") and isfile(
challenge_evaluation_details_file_path):
yaml_file_data["evaluation_details"] = get_file_content(
challenge_evaluation_details_file_path, "rb").decode("utf-8")
else:
yaml_file_data["evaluation_details"] = None
challenge_terms_and_cond_file_path = join(
BASE_LOCATION,
extracted_folder_name,
yaml_file_data["terms_and_conditions"],
)
if challenge_terms_and_cond_file_path.endswith(".html") and isfile(
challenge_terms_and_cond_file_path):
yaml_file_data["terms_and_conditions"] = get_file_content(
challenge_terms_and_cond_file_path, "rb").decode("utf-8")
else:
yaml_file_data["terms_and_conditions"] = None
submission_guidelines_file_path = join(
BASE_LOCATION,
extracted_folder_name,
yaml_file_data["submission_guidelines"],
)
if submission_guidelines_file_path.endswith(".html") and isfile(
submission_guidelines_file_path):
yaml_file_data["submission_guidelines"] = get_file_content(
submission_guidelines_file_path, "rb").decode("utf-8")
else:
yaml_file_data["submission_guidelines"] = None
serializer = ZipChallengeSerializer(
data=yaml_file_data,
context={
"request": namedtuple('Request', 'method')(method='NOTGET'),
"challenge_host_team": challenge_host_team,
"image": challenge_image_file,
"evaluation_script": challenge_evaluation_script_file,
},
)
if serializer.is_valid():
serializer.save()
challenge = serializer.instance
queue_name = get_queue_name(challenge.title)
challenge.queue = queue_name
challenge.save()
else:
raise Exception(serializer.errors)
# Create Leaderboard
yaml_file_data_of_leaderboard = yaml_file_data["leaderboard"]
leaderboard_ids = {}
for data in yaml_file_data_of_leaderboard:
serializer = LeaderboardSerializer(data=data)
if serializer.is_valid():
serializer.save()
leaderboard_ids[str(data["id"])] = serializer.instance.pk
else:
raise Exception(serializer.errors)
# Create Challenge Phase
challenge_phase_ids = {}
for data in challenge_phases_data:
# Check for challenge phase description file
phase_description_file_path = join(
BASE_LOCATION,
extracted_folder_name,
data["description"],
)
if phase_description_file_path.endswith(".html") and isfile(
phase_description_file_path):
data["description"] = get_file_content(
phase_description_file_path, "rb").decode("utf-8")
else:
data["description"] = None
test_annotation_file = data["test_annotation_file"]
data["slug"] = "{}-{}-{}".format(
challenge.title.split(" ")[0].lower(),
data["codename"].replace(" ", "-").lower(),
challenge.pk,
)[:198]
if test_annotation_file:
test_annotation_file_path = join(
BASE_LOCATION,
extracted_folder_name,
test_annotation_file,
)
if isfile(test_annotation_file_path):
with open(test_annotation_file_path,
"rb") as test_annotation_file:
challenge_test_annotation_file = ContentFile(
test_annotation_file.read(),
test_annotation_file_path,
)
serializer = ChallengePhaseCreateSerializer(
data=data,
context={
"challenge": challenge,
"test_annotation": challenge_test_annotation_file,
},
)
if serializer.is_valid():
serializer.save()
challenge_phase_ids[str(data["id"])] = serializer.instance.pk
else:
raise Exception(serializer.errors)
# Create Dataset Splits
yaml_file_data_of_dataset_split = yaml_file_data["dataset_splits"]
dataset_split_ids = {}
for data in yaml_file_data_of_dataset_split:
serializer = DatasetSplitSerializer(data=data)
if serializer.is_valid():
serializer.save()
dataset_split_ids[str(data["id"])] = serializer.instance.pk
else:
# Return error when dataset split name is not unique.
raise Exception(serializer.errors)
# Create Challenge Phase Splits
challenge_phase_splits_data = yaml_file_data["challenge_phase_splits"]
for data in challenge_phase_splits_data:
challenge_phase = challenge_phase_ids[str(
data["challenge_phase_id"])]
leaderboard = leaderboard_ids[str(data["leaderboard_id"])]
dataset_split = dataset_split_ids[str(data["dataset_split_id"])]
visibility = data["visibility"]
data = {
"challenge_phase": challenge_phase,
"leaderboard": leaderboard,
"dataset_split": dataset_split,
"visibility": visibility,
}
serializer = ZipChallengePhaseSplitSerializer(data=data)
if serializer.is_valid():
serializer.save()
else:
raise Exception(serializer.errors)
if not challenge.is_docker_based:
# Add the Challenge Host as a test participant.
emails = challenge_host_team.get_all_challenge_host_email()
team_name = "Host_{}_Team".format(random.randint(1, 100000))
participant_host_team = ParticipantTeam(
team_name=team_name,
created_by=challenge_host_team.created_by,
)
participant_host_team.save()
for email in emails:
user = User.objects.get(email=email)
host = Participant(
user=user,
status=Participant.ACCEPTED,
team=participant_host_team,
)
host.save()
challenge.participant_teams.add(participant_host_team)
print("Success creating challenge")
return challenge
def test_random_file_name_with_id(self):
obj = RandomFileName("submission_files/submission_{id}")
filepath = obj.__call__(self.submission, self.test_file_path)
expected = "submission_files/submission_{}/{}".format(self.submission.pk, filepath.split('/')[2])
self.assertEqual(filepath, expected)
