def acquire_worker(self, job, side_data=None):
"""Tries to assign a job to an available worker. If no workers
are available then this returns None, otherwise this returns
the chosen worker.
job (job): the job to assign to a worker
side_data (object): object to attach to the worker for later
use
returns (int): None if no workers are available, the worker
assigned to the job otherwise
"""
# We look for an available worker
try:
shard = self.find_worker(WorkerPool.WORKER_INACTIVE, require_connection=True, random_worker=True)
except LookupError:
return None
# Then we fill the info for future memory
self._job[shard] = job
self._start_time[shard] = make_datetime()
self._side_data[shard] = side_data
logger.debug("Worker %s acquired." % shard)
# And finally we ask the worker to do the job
action, object_id = job
timestamp = side_data[1]
queue_time = self._start_time[shard] - timestamp
logger.info(
"Asking worker %s to %s submission/user test %d "
" (%s after submission)." % (shard, action, object_id, queue_time)
)
with SessionGen(commit=False) as session:
if action == EvaluationService.JOB_TYPE_COMPILATION:
submission = Submission.get_from_id(object_id, session)
job_ = CompilationJob.from_submission(submission)
elif action == EvaluationService.JOB_TYPE_EVALUATION:
submission = Submission.get_from_id(object_id, session)
job_ = EvaluationJob.from_submission(submission)
elif action == EvaluationService.JOB_TYPE_TEST_COMPILATION:
user_test = UserTest.get_from_id(object_id, session)
job_ = CompilationJob.from_user_test(user_test)
elif action == EvaluationService.JOB_TYPE_TEST_EVALUATION:
user_test = UserTest.get_from_id(object_id, session)
job_ = EvaluationJob.from_user_test(user_test)
job_.get_output = True
job_.only_execution = True
self._worker[shard].execute_job(
job_dict=job_.export_to_dict(),
callback=self._service.action_finished.im_func,
plus=(action, object_id, side_data, shard),
)
return shard
def get_submission_data(self, submission_id):
"""Given the id, returns the submission object and a new task
type object of the correct type.
submission_id (int): id of the submission.
return (Submission, TaskType): corresponding objects.
raise: JobException if id or task type not found.
"""
submission = Submission.get_from_id(submission_id, self.session)
if submission is None:
err_msg = "Couldn't find submission %s " \
"in the database." % submission_id
logger.critical(err_msg)
raise JobException(err_msg)
try:
task_type = get_task_type(submission, self.file_cacher)
except KeyError as error:
err_msg = "Task type `%s' not known for " \
"submission %s (error: %s)." % (
submission.task.task_type, submission_id, error)
logger.error(err_msg)
raise JobException(err_msg)
return (submission, task_type)
def submission_tokened(self, submission_id):
"""This RPC inform ScoringService that the user has played the
token on a submission.
submission_id (int): the id of the submission that changed.
timestamp (int): the time of the token.
"""
with SessionGen(commit=False) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[submission_tokened] Received token request for "
"unexistent submission id %s." % submission_id)
raise KeyError
if submission.user.hidden:
logger.info("[submission_tokened] Token for submission %d "
"not sent because user is hidden." % submission_id)
return
# Mark submission as tokened.
self.submissions_tokened.add(submission_id)
# Update RWS.
self.rankings_send_token(submission)
def post(self):
submission_id = self.get_argument("submission_id", "")
# Decrypt submission_id.
try:
submission_id = decrypt_number(submission_id)
except ValueError:
# We reply with Forbidden if the given ID cannot be
# decrypted.
logger.warning(
"User %s tried to play a token " "on an undecryptable submission_id." % self.current_user.username
)
raise tornado.web.HTTPError(403)
# Find submission and check it is of the current user.
submission = Submission.get_from_id(submission_id, self.sql_session)
if submission is None or submission.user != self.current_user:
logger.warning(
"User %s tried to play a token " "on an unexisting submission_id." % self.current_user.username
)
raise tornado.web.HTTPError(404)
# Don't trust the user, check again if (s)he can really play
# the token.
timestamp = int(time.time())
if self.contest.tokens_available(self.current_user.username, submission.task.name, timestamp)[0] <= 0:
logger.warning("User %s tried to play a token " "when it shouldn't." % self.current_user.username)
# Add "no luck" notification
self.application.service.add_notification(
self.current_user.username,
timestamp,
self._("Token request discarded"),
self._("Your request has been discarded because you have no " "tokens available."),
)
self.redirect("/tasks/%s" % encrypt_number(submission.task.id))
return
token = Token(timestamp, submission)
self.sql_session.add(token)
self.sql_session.commit()
# Inform ScoringService and eventually the ranking that the
# token has been played.
self.application.service.scoring_service.submission_tokened(submission_id=submission_id, timestamp=timestamp)
logger.info("Token played by user %s on task %s." % (self.current_user.username, submission.task.name))
# Add "All ok" notification
self.application.service.add_notification(
self.current_user.username,
timestamp,
self._("Token request received"),
self._("Your request has been received " "and applied to the submission."),
)
self.redirect("/tasks/%s" % encrypt_number(submission.task.id))
def dataset_updated(self, task_id):
"""This function updates RWS with new data about a task. It should be
called after the live dataset of a task is changed.
task_id (int): id of the task whose dataset has changed.
"""
with SessionGen(commit=False) as session:
task = Task.get_from_id(task_id, session)
dataset_id = task.active_dataset_id
logger.info("Dataset update for task %d (dataset now is %d)." % (
task_id, dataset_id))
submission_ids = get_submissions(self.contest_id, task_id=task_id)
subchanges = []
with SessionGen(commit=False) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
submission_result = SubmissionResult.get_from_id(
(submission_id, dataset_id), session)
if submission_result is None:
# Not yet compiled, evaluated or scored.
score = None
ranking_score_details = None
else:
score = submission_result.score
try:
ranking_score_details = json.loads(
submission_result.ranking_score_details)
except (json.decoder.JSONDecodeError, TypeError):
# It may be blank.
ranking_score_details = None
# Data to send to remote rankings.
subchange_id = "%s%ss" % \
(int(make_timestamp(submission.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.timestamp))}
# We're sending the unrounded score to RWS
if score is not None:
subchange_put_data["score"] = score
if ranking_score_details is not None:
subchange_put_data["extra"] = ranking_score_details
subchanges.append((subchange_id, subchange_put_data))
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
for subchange_id, data in subchanges:
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = data
def submission_tokened(self, submission_id):
"""This RPC inform ScoringService that the user has played the
token on a submission.
submission_id (int): the id of the submission that changed.
timestamp (int): the time of the token.
"""
with SessionGen(commit=False) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[submission_tokened] Received token request for "
"unexistent submission id %s." % submission_id)
raise KeyError
elif submission.user.hidden:
logger.info("[submission_tokened] Token for submission %d "
"not sent because user is hidden." % submission_id)
return
# Mark submission as tokened.
self.submissions_tokened.add(submission_id)
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))
}
subchange_id = "%s%st" % \
(int(make_timestamp(submission.token.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.token.timestamp)),
"token": True
}
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(
ranking,
dict())[encode_id(submission_id)] = \
submission_put_data
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = \
subchange_put_data
def submission_tokened(self, submission_id):
"""This RPC inform ScoringService that the user has played the
token on a submission.
submission_id (int): the id of the submission that changed.
timestamp (int): the time of the token.
"""
with SessionGen(commit=False) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[submission_tokened] Received token request for "
"unexistent submission id %s." % submission_id)
raise KeyError
elif submission.user.hidden:
logger.info("[submission_tokened] Token for submission %d "
"not sent because user is hidden." % submission_id)
return
# Mark submission as tokened.
self.submissions_tokened.add(submission_id)
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))}
subchange_id = "%s%st" % \
(int(make_timestamp(submission.token.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.token.timestamp)),
"token": True}
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(
ranking,
dict())[encode_id(submission_id)] = \
submission_put_data
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = \
subchange_put_data
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None):
"""Request for invalidating the scores of some submissions.
The scores to be cleared are the one regarding 1) a submission
or 2) all submissions of a user or 3) all submissions of a
task or 4) all submission (if all parameters are None).
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
"""
logger.info("Invalidation request received.")
submission_ids = get_submissions(self.contest_id, submission_id,
user_id, task_id)
logger.info("Submissions to invalidate: %s" % len(submission_ids))
if len(submission_ids) == 0:
return
new_submission_ids = []
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
# If the submission is not evaluated, it does not have
# a score to invalidate, and, when evaluated,
# ScoringService will be prompted to score it. So in
# that case we do not have to do anything.
if submission.evaluated():
submission.invalidate_score()
new_submission_ids.append(submission_id)
old_s = len(self.submission_ids_to_score)
old_t = len(self.submission_ids_to_token)
self.submission_ids_to_score |= new_submission_ids
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions,
None,
0.5,
immediately=False)
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None):
"""Request for invalidating the scores of some submissions.
The scores to be cleared are the one regarding 1) a submission
or 2) all submissions of a user or 3) all submissions of a
task or 4) all submission (if all parameters are None).
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
"""
logger.info("Invalidation request received.")
submission_ids = get_submissions(
self.contest_id,
submission_id, user_id, task_id)
logger.info("Submissions to invalidate: %s" % len(submission_ids))
if len(submission_ids) == 0:
return
new_submission_ids = []
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
# If the submission is not evaluated, it does not have
# a score to invalidate, and, when evaluated,
# ScoringService will be prompted to score it. So in
# that case we do not have to do anything.
if submission.evaluated():
submission.invalidate_score()
new_submission_ids.append(submission_id)
old_s = len(self.submission_ids_to_score)
old_t = len(self.submission_ids_to_token)
self.submission_ids_to_score |= set(new_submission_ids)
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions, None,
0.5, immediately=False)
def submission_tokened(self, submission_id, timestamp):
"""This RPC inform ScoringService that the user has played the
token on a submission.
submission_id (int): the id of the submission that changed.
timestamp (int): the time of the token.
"""
with SessionGen(commit=False) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("Received request for "
"unexistent submission id %s." % submission_id)
raise KeyError
if submission.user.hidden:
return
# Mark submission as tokened.
self.submission_ids_tokened.add(submission_id)
# Data to send to remote rankings.
submission_url = "/submissions/%s" % encode_id(submission_id)
submission_put_data = {"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": submission.timestamp}
subchange_url = "/subchanges/%s" % encode_id("%s%st" %
(timestamp,
submission_id))
subchange_put_data = {"submission": encode_id(submission_id),
"time": timestamp,
"token": True}
# Adding operations to the queue.
for ranking in self.rankings:
self.operation_queue.append((send_submission,
[ranking, submission_url,
submission_put_data]))
self.operation_queue.append((send_change,
[ranking, subchange_url,
subchange_put_data]))
def new_submission(self, submission_id):
"""This RPC prompts ES of the existence of a new
submission. ES takes the right countermeasures, i.e., it
schedules it for compilation.
submission_id (string): the id of the new submission.
returns (bool): True if everything went well.
"""
with SessionGen(commit=False) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[new_submission] Couldn't find submission "
"%s in the database." % submission_id)
return
if to_compile(submission):
self.push_in_queue((EvaluationService.JOB_TYPE_COMPILATION,
submission_id),
EvaluationService.JOB_PRIORITY_HIGH,
submission.timestamp)
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None):
"""Request for invalidating the scores of some submissions.
The scores to be cleared are the one regarding 1) a submission
or 2) all submissions of a user or 3) all submissions of a
task or 4) all submission (if all parameters are None).
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
"""
logger.info("Invalidation request received.")
submission_ids = get_submissions(self.contest_id, submission_id,
user_id, task_id)
logger.info("Submissions to invalidate: %s" % len(submission_ids))
if len(submission_ids) == 0:
return
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
submission.invalidate_score()
old_s = len(self.submission_ids_to_score)
old_t = len(self.submission_ids_to_token)
self.submission_ids_to_score = submission_ids + \
self.submission_ids_to_score
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions,
None,
0.5,
immediately=False)
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None):
"""Request for invalidating the scores of some submissions.
The scores to be cleared are the one regarding 1) a submission
or 2) all submissions of a user or 3) all submissions of a
task or 4) all submission (if all parameters are None).
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
"""
logger.info("Invalidation request received.")
submission_ids = get_submissions(
self.contest_id,
submission_id, user_id, task_id)
logger.info("Submissions to invalidate: %s" % len(submission_ids))
if len(submission_ids) == 0:
return
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
submission.invalidate_score()
old_s = len(self.submission_ids_to_score)
old_t = len(self.submission_ids_to_token)
self.submission_ids_to_score = submission_ids + \
self.submission_ids_to_score
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions, None,
0.5, immediately=False)
def new_evaluation(self, submission_id, dataset_id):
"""This RPC inform ScoringService that ES finished the work on
a submission (either because it has been evaluated, or because
the compilation failed).
submission_id (int): the id of the submission that changed.
dataset_id (int): the id of the dataset to use.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[new_evaluation] Couldn't find submission %d "
"in the database." % submission_id)
raise ValueError
dataset = Dataset.get_from_id(dataset_id, session)
if dataset is None:
logger.error("[new_evaluation] Couldn't find dataset %d "
"in the database." % dataset_id)
raise ValueError
submission_result = submission.get_result(dataset)
if submission_result is None or not submission_result.compiled():
logger.warning("[new_evaluation] Submission %d(%d) is not "
"compiled." %
(submission_id, dataset_id))
return
elif submission_result.compilation_outcome == "ok" and \
not submission_result.evaluated():
logger.warning("[new_evaluation] Submission %d(%d) compiled "
"correctly but is not evaluated." %
(submission_id, dataset_id))
return
elif submission.user.hidden:
logger.info("[new_evaluation] Submission %d not scored "
"because user is hidden." % submission_id)
return
# Assign score to the submission.
scorer = self.scorers[dataset_id]
scorer.add_submission(submission_id, submission.timestamp,
submission.user.username,
submission_result.evaluated(),
dict((ev.codename,
{"outcome": ev.outcome,
"text": ev.text,
"time": ev.execution_time,
"memory": ev.memory_used})
for ev in submission_result.evaluations),
submission.tokened())
# Mark submission as scored.
self.submission_results_scored.add((submission_id, dataset_id))
# Filling submission's score info in the db.
submission_result.score = \
scorer.pool[submission_id]["score"]
submission_result.public_score = \
scorer.pool[submission_id]["public_score"]
# And details.
submission_result.score_details = \
scorer.pool[submission_id]["details"]
submission_result.public_score_details = \
scorer.pool[submission_id]["public_details"]
submission_result.ranking_score_details = \
scorer.pool[submission_id]["ranking_details"]
try:
ranking_score_details = json.loads(
submission_result.ranking_score_details)
except (TypeError, ValueError):
# It may be blank.
ranking_score_details = None
# If we are not a live dataset then we can bail out here,
# and avoid updating RWS.
if dataset is not submission.task.active_dataset:
return
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))}
subchange_id = "%s%ss" % \
(int(make_timestamp(submission.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(str(submission_id)),
"time": int(make_timestamp(submission.timestamp)),
# We're sending the unrounded score to RWS
"score": submission_result.score}
if ranking_score_details is not None:
subchange_put_data["extra"] = ranking_score_details
# TODO: ScoreRelative here does not work with remote
# rankings (it does in the ranking view) because we
# update only the user owning the submission.
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(
ranking,
dict())[encode_id(str(submission_id))] = \
submission_put_data
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = \
subchange_put_data
def new_evaluation(self, submission_id):
"""This RPC inform ScoringService that ES finished the work on
a submission (either because it has been evaluated, or because
the compilation failed).
submission_id (int): the id of the submission that changed.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[new_evaluation] Couldn't find "
" submission %d in the database." %
submission_id)
raise KeyError
elif not submission.compiled():
logger.warning("[new_evaluation] Submission %d "
"is not compiled." % submission_id)
return
elif submission.compilation_outcome == "ok" \
and not submission.evaluated():
logger.warning("[new_evaluation] Submission %d compiled "
"correctly but is not evaluated."
% submission_id)
return
elif submission.user.hidden:
logger.info("[new_evaluation] Submission %d not scored "
"because user is hidden." % submission_id)
return
# Assign score to the submission.
scorer = self.scorers[submission.task_id]
scorer.add_submission(submission_id, submission.timestamp,
submission.user.username,
submission.evaluated(),
dict((ev.num,
{"outcome": ev.outcome,
"text": ev.text,
"time": ev.execution_time,
"memory": ev.memory_used})
for ev in submission.evaluations),
submission.tokened())
# Mark submission as scored.
self.submission_ids_scored.add(submission_id)
# Filling submission's score info in the db.
submission.score = scorer.pool[submission_id]["score"]
submission.public_score = \
scorer.pool[submission_id]["public_score"]
# And details.
submission.score_details = scorer.pool[submission_id]["details"]
submission.public_score_details = \
scorer.pool[submission_id]["public_details"]
submission.ranking_score_details = \
scorer.pool[submission_id]["ranking_details"]
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))}
subchange_id = "%s%ss" % \
(int(make_timestamp(submission.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.timestamp)),
# We're sending the unrounded score to RWS
"score": submission.score,
"extra": submission.ranking_score_details}
# TODO: ScoreRelative here does not work with remote
# rankings (it does in the ranking view) because we
# update only the user owning the submission.
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(
ranking,
dict())[encode_id(submission_id)] = \
submission_put_data
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = \
subchange_put_data
def invalidate_submission(self,
submission_id=None,
user_id=None,
dataset_id=None,
task_id=None):
"""Request for invalidating the scores of some submissions.
The scores to be cleared are the one regarding
1) a submission (only submission_id and dataset_id given)
or 2) all submissions of a user for the active dataset (only user_id
given)
or 3) all submissions for a dataset (only dataset_id given)
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
dataset_id (int): id of the dataset we want to invalidate, or None.
"""
logger.info("Invalidation request received.")
# If we are invalidating a dataset, get the task so we can find all the
# relevant submissions.
if submission_id is None and dataset_id is not None:
with SessionGen(commit=False) as session:
dataset = Dataset.get_from_id(dataset_id, session)
task_id = dataset.task_id
else:
task_id = None
submission_ids = get_submissions(
self.contest_id,
submission_id, user_id, task_id)
logger.info("Submissions to invalidate: %s" % len(submission_ids))
if len(submission_ids) == 0:
return
new_submission_ids = []
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
if dataset_id is None:
this_dataset_id = submission.task.active_dataset_id
else:
this_dataset_id = dataset_id
sr = SubmissionResult.get_from_id(
(submission_id, this_dataset_id), session)
# If the submission is not evaluated, it does not have
# a score to invalidate, and, when evaluated,
# ScoringService will be prompted to score it. So in
# that case we do not have to do anything.
if sr is not None and sr.evaluated():
sr.invalidate_score()
new_submission_ids.append((submission_id, this_dataset_id))
old_s = len(self.submission_ids_to_score)
old_t = len(self.submission_ids_to_token)
self.submission_ids_to_score |= set(new_submission_ids)
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions, None,
0.5, immediately=False)
def new_evaluation(self, submission_id, dataset_id):
"""This RPC inform ScoringService that ES finished the work on
a submission (either because it has been evaluated, or because
the compilation failed).
submission_id (int): the id of the submission that changed.
dataset_id (int): the id of the dataset to use.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[new_evaluation] Couldn't find submission %d "
"in the database." % submission_id)
raise ValueError
if submission.user.hidden:
logger.info("[new_evaluation] Submission %d not scored "
"because user is hidden." % submission_id)
return
dataset = Dataset.get_from_id(dataset_id, session)
if dataset is None:
logger.error("[new_evaluation] Couldn't find dataset %d "
"in the database." % dataset_id)
raise ValueError
submission_result = submission.get_result(dataset)
# We'll accept only submissions that either didn't compile
# at all or that did evaluate successfully.
if submission_result is None or not submission_result.compiled():
logger.warning("[new_evaluation] Submission %d(%d) is "
"not compiled." % (submission_id, dataset_id))
return
elif submission_result.compilation_outcome == "ok" and \
not submission_result.evaluated():
logger.warning("[new_evaluation] Submission %d(%d) is "
"compiled but is not evaluated." %
(submission_id, dataset_id))
return
# Assign score to the submission.
score_type = get_score_type(dataset=dataset)
score, details, public_score, public_details, ranking_details = \
score_type.compute_score(submission_result)
# Mark submission as scored.
self.submission_results_scored.add((submission_id, dataset_id))
# Filling submission's score info in the db.
submission_result.score = score
submission_result.public_score = public_score
# And details.
submission_result.score_details = details
submission_result.public_score_details = public_details
submission_result.ranking_score_details = ranking_details
# If dataset is the active one, update RWS.
if dataset is submission.task.active_dataset:
self.rankings_send_score(submission)
def new_evaluation(self, submission_id):
"""This RPC inform ScoringService that ES finished the
evaluation for a submission.
submission_id (int): the id of the submission that changed.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.critical("[action_finished] Couldn't find "
" submission %d in the database" %
submission_id)
return
if submission.user.hidden:
return
# Assign score to the submission.
scorer = self.scorers[submission.task_id]
scorer.add_submission(submission_id, submission.timestamp,
submission.user.username,
dict((ev.num,
{"outcome": float(ev.outcome),
"text": ev.text,
"time": ev.execution_time,
"memory": ev.memory_used})
for ev in submission.evaluations),
submission.tokened())
# Mark submission as scored.
self.submission_ids_scored.add(submission_id)
# Filling submission's score info in the db.
submission.score = scorer.pool[submission_id]["score"]
submission.public_score = \
scorer.pool[submission_id]["public_score"]
# And details.
submission.score_details = scorer.pool[submission_id]["details"]
submission.public_score_details = \
scorer.pool[submission_id]["public_details"]
submission.ranking_score_details = \
scorer.pool[submission_id]["ranking_details"]
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))}
subchange_id = "%s%ss" % (int(make_timestamp(submission.timestamp)), submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.timestamp)),
"score": submission.score,
"extra": submission.ranking_score_details}
# TODO: ScoreRelative here does not work with remote
# rankings (it does in the ranking view) because we
# update only the user owning the submission.
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(ranking, dict())[encode_id(submission_id)] = submission_put_data
self.subchange_queue.setdefault(ranking, dict())[encode_id(subchange_id)] = subchange_put_data
def get(self, sub_id):
submission = Submission.get_from_id(sub_id, self.sql_session)
if submission.user.id != self.current_user.id or submission.task.contest.id != self.contest.id:
raise tornado.web.HTTPError(403)
self.render("submission_snippet.html", s=submission)
def action_finished(self, data, plus, error=None):
"""Callback from a worker, to signal that is finished some
action (compilation or evaluation).
data (dict): a dictionary that describes a Job instance.
plus (tuple): the tuple (job_type,
object_id,
side_data=(priority, timestamp),
shard_of_worker)
"""
# TODO - The next two comments are in the wrong place and
# really little understandable anyway.
# We notify the pool that the worker is free (even if it
# replied with an error), but if the pool wants to disable the
# worker, it's because it already assigned its job to someone
# else, so we discard the data from the worker.
job_type, object_id, side_data, shard = plus
# If worker was ignored, do nothing.
if self.pool.release_worker(shard):
return
job_success = True
if error is not None:
logger.error("Received error from Worker: `%s'." % error)
job_success = False
else:
try:
job = Job.import_from_dict_with_type(data)
except:
logger.error("[action_finished] Couldn't build Job for data" " %s." % (data))
job_success = False
else:
if not job.success:
logger.error("Worker %s signaled action " "not successful." % shard)
job_success = False
_, timestamp = side_data
logger.info("Action %s for submission %s completed. Success: %s." % (job_type, object_id, data["success"]))
# We get the submission from DB and update it.
with SessionGen(commit=False) as session:
if job_type == EvaluationService.JOB_TYPE_COMPILATION:
submission = Submission.get_from_id(object_id, session)
if submission is None:
logger.error("[action_finished] Couldn't find " "submission %d in the database." % object_id)
return
submission.compilation_tries += 1
if job_success:
submission.compilation_outcome = "ok" if job.compilation_success else "fail"
submission.compilation_text = job.text
submission.compilation_shard = job.shard
submission.compilation_sandbox = ":".join(job.sandboxes)
for executable in job.executables.itervalues():
submission.executables[executable.filename] = executable
session.add(executable)
self.compilation_ended(submission)
elif job_type == EvaluationService.JOB_TYPE_EVALUATION:
submission = Submission.get_from_id(object_id, session)
if submission is None:
logger.error("[action_finished] Couldn't find " "submission %s in the database." % object_id)
return
submission.evaluation_tries += 1
if job_success:
submission.evaluation_outcome = "ok"
for test_number, info in job.evaluations.iteritems():
evaluation = Evaluation(
text=info["text"],
outcome=info["outcome"],
num=test_number,
memory_used=info["plus"].get("memory_used", None),
execution_time=info["plus"].get("execution_time", None),
execution_wall_clock_time=info["plus"].get("execution_wall_clock_time", None),
evaluation_shard=job.shard,
evaluation_sandbox=":".join(info["sandboxes"]),
submission=submission,
)
session.add(evaluation)
self.evaluation_ended(submission)
elif job_type == EvaluationService.JOB_TYPE_TEST_COMPILATION:
user_test = UserTest.get_from_id(object_id, session)
if user_test is None:
logger.error("[action_finished] Couldn't find " "user test %d in the database." % object_id)
return
user_test.compilation_tries += 1
if job_success:
user_test.compilation_outcome = "ok" if job.compilation_success else "fail"
user_test.compilation_text = job.text
user_test.compilation_shard = job.shard
user_test.compilation_sandbox = ":".join(job.sandboxes)
for executable in job.executables.itervalues():
ut_executable = UserTestExecutable.import_from_dict(executable.export_to_dict())
user_test.executables[ut_executable.filename] = ut_executable
session.add(ut_executable)
self.user_test_compilation_ended(user_test)
elif job_type == EvaluationService.JOB_TYPE_TEST_EVALUATION:
user_test = UserTest.get_from_id(object_id, session)
if user_test is None:
logger.error("[action_finished] Couldn't find " "user test %d in the database." % object_id)
return
user_test.evaluation_tries += 1
if job_success:
try:
[evaluation] = job.evaluations.values()
except ValueError:
logger.error(
"[action_finished] I expected the job "
"for a user test to contain a single "
"evaluation, while instead it has %d." % (len(job.evaluations.values()))
)
return
user_test.evaluation_outcome = "ok"
user_test.evaluation_shard = job.shard
user_test.output = evaluation["output"]
user_test.evaluation_text = evaluation["text"]
user_test.evaluation_sandbox = ":".join(evaluation["sandboxes"])
user_test.memory_used = (evaluation["plus"].get("memory_used", None),)
user_test.execution_time = (evaluation["plus"].get("execution_time", None),)
self.user_test_evaluation_ended(user_test)
else:
logger.error("Invalid job type %r." % (job_type))
return
session.commit()
def action_finished(self, data, plus, error=None):
"""Callback from a worker, to signal that is finished some
action (compilation or evaluation).
data (bool): report success of the action
plus (tuple): the tuple (job=(job_type, submission_id),
side_data=(priority, timestamp),
shard_of_worker)
"""
# We notify the pool that the worker is free (even if it
# replied with ane error), but if the pool wants to disable the
# worker, it's because it already assigned its job to someone
# else, so we discard the data from the worker.
job, side_data, shard = plus
# If worker was ignored, do nothing.
if self.pool.release_worker(shard):
return
if error is not None:
logger.error("Received error from Worker: `%s'." % error)
return
if not data["success"]:
logger.error("Worker %s signaled action not successful." % shard)
return
job_type, submission_id = job
unused_priority, timestamp = side_data
logger.info("Action %s for submission %s completed. Success: %s." %
(job_type, submission_id, data["success"]))
# We get the submission from db.
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.critical("[action_finished] Couldn't find submission "
"%s in the database." % submission_id)
return
if job_type == EvaluationService.JOB_TYPE_COMPILATION:
submission.compilation_tries += 1
submission.compilation_outcome = data["compilation_outcome"]
submission.compilation_text = data["compilation_text"]
submission.compilation_shard = data["compilation_shard"]
submission.compilation_sandbox = data["compilation_sandbox"]
for filename, digest in data.get("executables", []):
session.add(Executable(digest, filename, submission))
if job_type == EvaluationService.JOB_TYPE_EVALUATION:
submission.evaluation_tries += 1
submission.evaluation_outcome = "ok"
for test_number, info in data["evaluations"].iteritems():
ewct = info["execution_wall_clock_time"] # Too long... :(
session.add(Evaluation(
text=info["text"],
outcome=info["outcome"],
num=test_number,
memory_used=info["memory_used"],
execution_time=info["execution_time"],
execution_wall_clock_time=ewct,
evaluation_shard=info["evaluation_shard"],
evaluation_sandbox=info["evaluation_sandbox"],
submission=submission))
compilation_tries = submission.compilation_tries
compilation_outcome = submission.compilation_outcome
evaluation_tries = submission.evaluation_tries
evaluated = submission.evaluated()
# Compilation.
if job_type == EvaluationService.JOB_TYPE_COMPILATION:
self.compilation_ended(submission_id,
timestamp,
compilation_tries,
compilation_outcome)
# Evaluation.
elif job_type == EvaluationService.JOB_TYPE_EVALUATION:
self.evaluation_ended(submission_id,
timestamp,
evaluation_tries,
evaluated)
# Other (i.e. error).
else:
logger.error("Invalid job type %r." % job_type)
return
def new_evaluation(self, submission_id):
"""This RPC inform ScoringService that ES finished the
evaluation for a submission.
submission_id (int): the id of the submission that changed.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.critical("[action_finished] Couldn't find "
" submission %d in the database" %
submission_id)
return
if submission.user.hidden:
return
# Assign score to the submission.
scorer = self.scorers[submission.task_id]
scorer.add_submission(submission_id, submission.timestamp,
submission.user.username,
[float(ev.outcome)
for ev in submission.evaluations],
submission.tokened())
# Mark submission as scored.
self.submission_ids_scored.add(submission_id)
# Update the ranking view.
contest = session.query(Contest).\
filter_by(id=self.contest_id).first()
contest.update_ranking_view(self.scorers,
task=submission.task)
# Filling submission's score info in the db.
submission.score = scorer.pool[submission_id]["score"]
submission.public_score = \
scorer.pool[submission_id]["public_score"]
details = scorer.pool[submission_id]["details"]
if details is None:
details = []
submission.score_details = json.dumps(details)
public_details = scorer.pool[submission_id]["public_details"]
if public_details is None:
public_details = []
submission.public_score_details = json.dumps(public_details)
# Data to send to remote rankings.
submission_url = "/submissions/%s" % encode_id(submission_id)
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": submission.timestamp}
subchange_url = "/subchanges/%s" % encode_id("%s%ss" %
(submission.timestamp,
submission_id))
subchange_put_data = {"submission": encode_id(submission_id),
"time": submission.timestamp,
"score": submission.score,
"extra": details}
# TODO: ScoreRelative here does not work with remote
# rankings (it does in the ranking view) because we
# update only the user owning the submission.
# Adding operations to the queue.
for ranking in self.rankings:
self.operation_queue.append((send_submission,
[ranking, submission_url,
submission_put_data]))
self.operation_queue.append((send_change,
[ranking, subchange_url,
subchange_put_data]))
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None,
level="compilation"):
"""Request for invalidating some computed data.
The data (compilations and evaluations, or evaluations only)
to be cleared are the one regarding 1) a submission or 2) all
submissions of a user or 3) all submissions of a task or 4)
all submission (if all parameters are None).
The data are cleared, the jobs involving the submissions
currently enqueued are deleted, and the one already assigned
to the workers are ignored. New appropriate jobs are enqueued.
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
level (string): 'compilation' or 'evaluation'
"""
logger.info("Invalidation request received.")
if level not in ["compilation", "evaluation"]:
err_msg = "Unexpected invalidation level `%s'." % level
logger.warning(err_msg)
raise ValueError(err_msg)
submission_ids = get_submissions(
self.contest_id,
submission_id, user_id, task_id)
logger.info("Submissions to invalidate for %s: %s." %
(level, len(submission_ids)))
if len(submission_ids) == 0:
return
for submission_id in submission_ids:
jobs = [(EvaluationService.JOB_TYPE_COMPILATION, submission_id),
(EvaluationService.JOB_TYPE_EVALUATION, submission_id)]
for job in jobs:
try:
self.queue.remove(job)
except KeyError:
pass # Ok, the job wasn't in the queue.
try:
self.pool.ignore_job(job)
except LookupError:
pass # Ok, the job wasn't in the pool.
# We invalidate the appropriate data and queue the jobs to
# recompute those data.
with SessionGen(commit=True) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
if level == "compilation":
submission.invalidate_compilation()
if to_compile(submission):
self.push_in_queue(
(EvaluationService.JOB_TYPE_COMPILATION,
submission_id),
EvaluationService.JOB_PRIORITY_HIGH,
submission.timestamp)
elif level == "evaluation":
submission.invalidate_evaluation()
if to_evaluate(submission):
self.push_in_queue(
(EvaluationService.JOB_TYPE_EVALUATION,
submission_id),
EvaluationService.JOB_PRIORITY_MEDIUM,
submission.timestamp)
def new_evaluation(self, submission_id):
"""This RPC inform ScoringService that ES finished the work on
a submission (either because it has been evaluated, or because
the compilation failed).
submission_id (int): the id of the submission that changed.
"""
with SessionGen(commit=True) as session:
submission = Submission.get_from_id(submission_id, session)
if submission is None:
logger.error("[new_evaluation] Couldn't find "
" submission %d in the database." % submission_id)
raise KeyError
elif not submission.compiled():
logger.warning("[new_evaluation] Submission %d "
"is not compiled." % submission_id)
return
elif submission.compilation_outcome == "ok" \
and not submission.evaluated():
logger.warning("[new_evaluation] Submission %d compiled "
"correctly but is not evaluated." %
submission_id)
return
elif submission.user.hidden:
logger.info("[new_evaluation] Submission %d not scored "
"because user is hidden." % submission_id)
return
# Assign score to the submission.
scorer = self.scorers[submission.task_id]
scorer.add_submission(
submission_id, submission.timestamp, submission.user.username,
submission.evaluated(),
dict((ev.num, {
"outcome": ev.outcome,
"text": ev.text,
"time": ev.execution_time,
"memory": ev.memory_used
}) for ev in submission.evaluations), submission.tokened())
# Mark submission as scored.
self.submissions_scored.add(submission_id)
# Filling submission's score info in the db.
submission.score = scorer.pool[submission_id]["score"]
submission.public_score = \
scorer.pool[submission_id]["public_score"]
# And details.
submission.score_details = scorer.pool[submission_id]["details"]
submission.public_score_details = \
scorer.pool[submission_id]["public_details"]
submission.ranking_score_details = \
scorer.pool[submission_id]["ranking_details"]
# Data to send to remote rankings.
submission_put_data = {
"user": encode_id(submission.user.username),
"task": encode_id(submission.task.name),
"time": int(make_timestamp(submission.timestamp))
}
subchange_id = "%s%ss" % \
(int(make_timestamp(submission.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.timestamp)),
# We're sending the unrounded score to RWS
"score": submission.score,
"extra": submission.ranking_score_details
}
# TODO: ScoreRelative here does not work with remote
# rankings (it does in the ranking view) because we
# update only the user owning the submission.
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
self.submission_queue.setdefault(
ranking,
dict())[encode_id(submission_id)] = \
submission_put_data
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = \
subchange_put_data
