def invalidate_submission(self,
submission_id=None,
dataset_id=None,
participation_id=None,
task_id=None,
contest_id=None):
"""Invalidate (and re-score) some submission results.
Invalidate the scores of the submission results that:
- belong to submission_id or, if None, to any submission of
participation_id and/or task_id or, if both None, to any
submission of contest_id or, if None, to any submission in
the database.
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of contest_id or, if None, to any
dataset in the database.
submission_id (int|None): id of the submission whose results
should be invalidated, or None.
dataset_id (int|None): id of the dataset whose results should
be invalidated, or None.
participation_id (int|None): id of the participation whose results
should be invalidated, or None.
task_id (int|None): id of the task whose results should be
invalidated, or None.
contest_id (int|None): id of the contest whose results should
be invalidated, or None.
"""
logger.info("Invalidation request received.")
# We can put results in the scorer queue only after they have
# been invalidated (and committed to the database). Therefore
# we temporarily save them somewhere else.
temp_queue = list()
with SessionGen() as session:
submission_results = \
get_submission_results(contest_id,
participation_id, task_id,
submission_id, dataset_id,
session=session)
for sr in submission_results:
if sr.scored():
sr.invalidate_score()
# We also save the timestamp of the submission, to
# rescore them in order (for fairness, not for a
# specific need).
temp_queue.append(
(ScoringOperation(sr.submission_id, sr.dataset_id),
sr.submission.timestamp))
session.commit()
for item, timestamp in temp_queue:
self.enqueue(item, timestamp=timestamp)
logger.info("Invalidated %d submission results.", len(temp_queue))
def invalidate_submission(self,
submission_id=None,
dataset_id=None,
user_id=None,
task_id=None):
"""Request for invalidating some scores.
Invalidate the scores of the SubmissionResults that:
- belong to submission_id or, if None, to any submission of
user_id and/or task_id or, if both None, to any submission
of the contest this service is running for.
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of any task of the contest this
service is running for.
submission_id (int): id of the submission to invalidate, or
None.
dataset_id (int): id of the dataset to invalidate, or None.
user_id (int): id of the user to invalidate, or None.
task_id (int): id of the task to invalidate, or None.
"""
logger.info("Invalidation request received.")
# Validate arguments
# TODO Check that all these objects belong to this contest.
with SessionGen(commit=True) as session:
submission_results = get_submission_results(
# Give contest_id only if all others are None.
self.contest_id
if {user_id, task_id, submission_id, dataset_id} == {None}
else None,
user_id, task_id, submission_id, dataset_id, session)
logger.info("Submission results to invalidate scores for: %d." %
len(submission_results))
if len(submission_results) == 0:
return
new_submission_results_to_score = set()
for submission_result in submission_results:
# If the submission is not evaluated, it does not have
# a score to invalidate, and, when evaluated,
# ScoringService will be prompted to score it. So in
# that case we do not have to do anything.
if submission_result.evaluated():
submission_result.invalidate_score()
new_submission_results_to_score.add(
(submission_result.submission_id,
submission_result.dataset_id))
old_s = len(self.submission_results_to_score)
old_t = len(self.submissions_to_token)
self.submission_results_to_score |= new_submission_results_to_score
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions, None,
0.5, immediately=False)
def invalidate_submission(self, submission_id=None, dataset_id=None,
participation_id=None, task_id=None,
contest_id=None):
"""Invalidate (and re-score) some submission results.
Invalidate the scores of the submission results that:
- belong to submission_id or, if None, to any submission of
participation_id and/or task_id or, if both None, to any
submission of contest_id or, if None, to any submission in
the database.
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of contest_id or, if None, to any
dataset in the database.
submission_id (int|None): id of the submission whose results
should be invalidated, or None.
dataset_id (int|None): id of the dataset whose results should
be invalidated, or None.
participation_id (int|None): id of the participation whose results
should be invalidated, or None.
task_id (int|None): id of the task whose results should be
invalidated, or None.
contest_id (int|None): id of the contest whose results should
be invalidated, or None.
"""
logger.info("Invalidation request received.")
# We can put results in the scorer queue only after they have
# been invalidated (and committed to the database). Therefore
# we temporarily save them somewhere else.
temp_queue = list()
with SessionGen() as session:
submission_results = \
get_submission_results(contest_id,
participation_id, task_id,
submission_id, dataset_id,
session=session)
for sr in submission_results:
if sr.scored():
sr.invalidate_score()
# We also save the timestamp of the submission, to
# rescore them in order (for fairness, not for a
# specific need).
temp_queue.append((
ScoringOperation(sr.submission_id, sr.dataset_id),
sr.submission.timestamp))
session.commit()
for item, timestamp in temp_queue:
self.enqueue(item, timestamp=timestamp)
logger.info("Invalidated %d submission results.", len(temp_queue))
def invalidate_submission(self,
submission_id=None,
dataset_id=None,
user_id=None,
task_id=None,
contest_id=None):
"""Invalidate (and re-score) some submission results.
Invalidate the scores of the submission results that:
- belong to submission_id or, if None, to any submission of
user_id and/or task_id or, if both None, to any submission
of contest_id or, if None, to any submission in the database.
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of contest_id or, if None, to any
dataset in the database.
submission_id (int): id of the submission whose results should
be invalidated, or None.
dataset_id (int): id of the dataset whose results should be
invalidated, or None.
user_id (int): id of the user whose results should be
invalidated, or None.
task_id (int): id of the task whose results should be
invalidated, or None.
contest_id (int): id of the contest whose results should be
invalidated, or None.
"""
logger.info("Invalidation request received.")
# We can put results in the scorer queue only after they have
# been invalidated (and committed to the database). Therefore
# we temporarily save them somewhere else.
temp_queue = list()
with SessionGen() as session:
submission_results = \
get_submission_results(contest_id, user_id, task_id,
submission_id, dataset_id,
session=session)
for sr in submission_results:
if sr.scored():
sr.invalidate_score()
temp_queue.append((sr.submission_id, sr.dataset_id))
session.commit()
for item in temp_queue:
self._scorer_queue.put(item)
logger.info("Invalidated %d submissions.", len(temp_queue))
def _missing_operations(self):
"""Return a generator of unscored submission results.
Obtain a list of all the submission results in the database,
check each of them to see if it's still unscored and if so
enqueue them.
"""
counter = 0
with SessionGen() as session:
for sr in get_submission_results(session=session):
if sr is not None and sr.needs_scoring():
self.enqueue(ScoringOperation(sr.submission_id, sr.dataset_id), timestamp=sr.submission.timestamp)
counter += 1
return counter
def invalidate_submission(self, submission_id=None, dataset_id=None,
user_id=None, task_id=None, contest_id=None):
"""Invalidate (and re-score) some submission results.
Invalidate the scores of the submission results that:
- belong to submission_id or, if None, to any submission of
user_id and/or task_id or, if both None, to any submission
of contest_id or, if None, to any submission in the database.
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of contest_id or, if None, to any
dataset in the database.
submission_id (int|None): id of the submission whose results
should be invalidated, or None.
dataset_id (int|None): id of the dataset whose results should
be invalidated, or None.
user_id (int|None): id of the user whose results should be
invalidated, or None.
task_id (int|None): id of the task whose results should be
invalidated, or None.
contest_id (int|None): id of the contest whose results should
be invalidated, or None.
"""
logger.info("Invalidation request received.")
# We can put results in the scorer queue only after they have
# been invalidated (and committed to the database). Therefore
# we temporarily save them somewhere else.
temp_queue = list()
with SessionGen() as session:
submission_results = \
get_submission_results(contest_id, user_id, task_id,
submission_id, dataset_id,
session=session)
for sr in submission_results:
if sr.scored():
sr.invalidate_score()
temp_queue.append((sr.submission_id, sr.dataset_id))
session.commit()
for item in temp_queue:
self._scorer_queue.put(item)
logger.info("Invalidated %d submissions.", len(temp_queue))
def _missing_operations(self):
"""Return a generator of unscored submission results.
Obtain a list of all the submission results in the database,
check each of them to see if it's still unscored and if so
enqueue them.
"""
counter = 0
with SessionGen() as session:
for sr in get_submission_results(session=session):
if sr is not None and sr.needs_scoring():
self.enqueue(ScoringOperation(sr.submission_id,
sr.dataset_id),
timestamp=sr.submission.timestamp)
counter += 1
return counter
def _sweep(self):
"""Check the database for unscored submission results.
Obtain a list of all the submission results in the database,
check each of them to see if it's still unscored and, in case,
put it in the queue.
"""
counter = 0
with SessionGen() as session:
for sr in get_submission_results(session=session):
if sr is not None and sr.needs_scoring():
self._scorer_queue.put((sr.submission_id, sr.dataset_id))
counter += 1
if counter > 0:
logger.info("Found %d unscored submissions.", counter)
def invalidate_submission(self,
contest_id=None,
submission_id=None,
dataset_id=None,
participation_id=None,
task_id=None,
level="compilation"):
"""Request to invalidate some computed data.
Invalidate the compilation and/or evaluation data of the
SubmissionResults that:
- belong to submission_id or, if None, to any submission of
participation_id and/or task_id or, if both None, to any
submission of the contest asked for, or, if all three are
None, the contest this service is running for (or all contests).
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of any task of the contest this
service is running for.
The data is cleared, the operations involving the submissions
currently enqueued are deleted, and the ones already assigned to
the workers are ignored. New appropriate operations are
enqueued.
submission_id (int|None): id of the submission to invalidate,
or None.
dataset_id (int|None): id of the dataset to invalidate, or
None.
participation_id (int|None): id of the participation to
invalidate, or None.
task_id (int|None): id of the task to invalidate, or None.
level (string): 'compilation' or 'evaluation'
"""
logger.info("Invalidation request received.")
# Validate arguments
# TODO Check that all these objects belong to this contest.
if level not in ("compilation", "evaluation"):
raise ValueError("Unexpected invalidation level `%s'." % level)
if contest_id is None:
contest_id = self.contest_id
with SessionGen() as session:
# When invalidating a dataset we need to know the task_id, otherwise
# get_submissions will return all the submissions of the contest.
if dataset_id is not None and task_id is None \
and submission_id is None:
task_id = Dataset.get_from_id(dataset_id, session).task_id
# First we load all involved submissions.
submissions = get_submissions(
# Give contest_id only if all others are None.
contest_id if {participation_id, task_id, submission_id}
== {None} else None,
participation_id,
task_id,
submission_id,
session)
# Then we get all relevant operations, and we remove them
# both from the queue and from the pool (i.e., we ignore
# the workers involved in those operations).
operations = get_relevant_operations(level, submissions,
dataset_id)
for operation in operations:
try:
self.dequeue(operation)
except KeyError:
pass # Ok, the operation wasn't in the queue.
try:
self.get_executor().pool.ignore_operation(operation)
except LookupError:
pass # Ok, the operation wasn't in the pool.
# Then we find all existing results in the database, and
# we remove them.
submission_results = get_submission_results(
# Give contest_id only if all others are None.
contest_id if {
participation_id, task_id, submission_id, dataset_id
} == {None} else None,
participation_id,
# Provide the task_id only if the entire task has to be
# reevaluated and not only a specific dataset.
task_id if dataset_id is None else None,
submission_id,
dataset_id,
session)
logger.info("Submission results to invalidate %s for: %d.", level,
len(submission_results))
for submission_result in submission_results:
# We invalidate the appropriate data and queue the
# operations to recompute those data.
if level == "compilation":
submission_result.invalidate_compilation()
elif level == "evaluation":
submission_result.invalidate_evaluation()
# Finally, we re-enqueue the operations for the
# submissions.
for submission in submissions:
self.submission_enqueue_operations(submission)
session.commit()
logger.info("Invalidate successfully completed.")
def invalidate_submission(self,
contest_id=None,
submission_id=None,
dataset_id=None,
participation_id=None,
task_id=None,
testcases=None,
overwrite=None,
force_priority=None,
level="compilation"):
"""Request to invalidate some computed data.
Invalidate the compilation and/or evaluation data of the
SubmissionResults that:
- belong to submission_id or, if None, to any submission of
participation_id and/or task_id or, if both None, to any
submission of the contest asked for, or, if all three are
None, the contest this service is running for (or all contests).
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of any task of the contest this
service is running for.
The data is cleared, the operations involving the submissions
currently enqueued are deleted, and the ones already assigned to
the workers are ignored. New appropriate operations are
enqueued.
submission_id (int|None): id of the submission to invalidate,
or None.
dataset_id (int|None): id of the dataset to invalidate, or
None.
participation_id (int|None): id of the participation to
invalidate, or None.
task_id (int|None): id of the task to invalidate, or None.
level (string): 'compilation' or 'evaluation'
"""
logger.info("Invalidation request received.")
# Avoid running the sweeper for the next 10-ish minutes, to
# avoid race conditions between invalidate's requeuing of
# submissions and the sweeper's.
self.avoid_next_sweepers = 1
# Validate arguments
# TODO Check that all these objects belong to this contest.
if level not in ("compilation", "evaluation"):
raise ValueError("Unexpected invalidation level `%s'." % level)
if contest_id is None:
contest_id = self.contest_id
with SessionGen() as session:
# First we load all involved submissions.
if (dataset_id is not None) and (submission_id is None):
dataset = Dataset.get_from_id(dataset_id, session)
task_id_for_submissions = dataset.task_id
else:
task_id_for_submissions = task_id
submissions = get_submissions(
# Give contest_id only if all others are None.
contest_id if {
participation_id, task_id_for_submissions, submission_id
} == {None} else None,
participation_id,
task_id_for_submissions,
submission_id,
session)
# Then we get all relevant operations, and we remove them
# both from the queue and from the pool (i.e., we ignore
# the workers involved in those operations).
operations = get_relevant_operations(level, submissions,
dataset_id, testcases)
for operation in operations:
try:
self.dequeue(operation)
except KeyError:
pass # Ok, the operation wasn't in the queue.
try:
self.get_executor().pool.ignore_operation(operation)
except LookupError:
pass # Ok, the operation wasn't in the pool.
# Then we find all existing results in the database, and
# we remove them.
submission_results = get_submission_results(
# Give contest_id only if all others are None.
contest_id if {
participation_id, task_id, submission_id, dataset_id
} == {None} else None,
participation_id,
task_id,
submission_id,
dataset_id,
session)
logger.info("Submission results to invalidate %s for: %d.", level,
len(submission_results))
for submission_result in submission_results:
# We invalidate the appropriate data and queue the
# operations to recompute those data.
if level == "compilation":
submission_result.invalidate_compilation(
testcases if overwrite is not False else [])
elif level == "evaluation":
submission_result.invalidate_evaluation(
testcases if overwrite is not False else [])
# There is a small chance that an invalidate won't
# succeed: if a result is in the pending structure, it
# might be written after the commit here.
session.commit()
# Collecting the ids so that we can close the session
# before the rpcs.
submission_ids = [submission.id for submission in submissions]
# Finally, we re-enqueue the operations for the submissions.
for idx in range(0, len(submission_ids), 500):
random_service(self.evaluation_services).new_submissions(
submission_ids=submission_ids[idx:min(idx +
500, len(submission_ids
))],
dataset_id=dataset_id,
force_priority=force_priority)
logger.info("Invalidate successfully completed.")
def invalidate_submission(self,
contest_id=None,
submission_id=None,
dataset_id=None,
participation_id=None,
task_id=None,
level="compilation"):
"""Request to invalidate some computed data.
Invalidate the compilation and/or evaluation data of the
SubmissionResults that:
- belong to submission_id or, if None, to any submission of
participation_id and/or task_id or, if both None, to any
submission of the contest asked for, or, if all three are
None, the contest this service is running for (or all contests).
- belong to dataset_id or, if None, to any dataset of task_id
or, if None, to any dataset of any task of the contest this
service is running for.
The data is cleared, the operations involving the submissions
currently enqueued are deleted, and the ones already assigned to
the workers are ignored. New appropriate operations are
enqueued.
submission_id (int|None): id of the submission to invalidate,
or None.
dataset_id (int|None): id of the dataset to invalidate, or
None.
participation_id (int|None): id of the participation to
invalidate, or None.
task_id (int|None): id of the task to invalidate, or None.
level (string): 'compilation' or 'evaluation'
"""
logger.info("Invalidation request received.")
# Validate arguments
# TODO Check that all these objects belong to this contest.
if level not in ("compilation", "evaluation"):
raise ValueError(
"Unexpected invalidation level `%s'." % level)
if contest_id is None:
contest_id = self.contest_id
with SessionGen() as session:
# First we load all involved submissions.
submissions = get_submissions(
# Give contest_id only if all others are None.
contest_id
if {participation_id, task_id, submission_id} == {None}
else None,
participation_id, task_id, submission_id, session)
# Then we get all relevant operations, and we remove them
# both from the queue and from the pool (i.e., we ignore
# the workers involved in those operations).
operations = get_relevant_operations(
level, submissions, dataset_id)
for operation in operations:
try:
self.dequeue(operation)
except KeyError:
pass # Ok, the operation wasn't in the queue.
try:
self.get_executor().pool.ignore_operation(operation)
except LookupError:
pass # Ok, the operation wasn't in the pool.
# Then we find all existing results in the database, and
# we remove them.
submission_results = get_submission_results(
# Give contest_id only if all others are None.
contest_id
if {participation_id,
task_id,
submission_id,
dataset_id} == {None}
else None,
participation_id, task_id, submission_id, dataset_id, session)
logger.info("Submission results to invalidate %s for: %d.",
level, len(submission_results))
for submission_result in submission_results:
# We invalidate the appropriate data and queue the
# operations to recompute those data.
if level == "compilation":
submission_result.invalidate_compilation()
elif level == "evaluation":
submission_result.invalidate_evaluation()
# Finally, we re-enqueue the operations for the
# submissions.
for submission in submissions:
self.submission_enqueue_operations(submission)
session.commit()
logger.info("Invalidate successfully completed.")
