def dataset_updated(self, task_id):
"""This function updates RWS with new data about a task. It should be
called after the live dataset of a task is changed.
task_id (int): id of the task whose dataset has changed.
"""
with SessionGen(commit=False) as session:
task = Task.get_from_id(task_id, session)
dataset_id = task.active_dataset_id
logger.info("Dataset update for task %d (dataset now is %d)." % (
task_id, dataset_id))
submission_ids = get_submissions(self.contest_id, task_id=task_id)
subchanges = []
with SessionGen(commit=False) as session:
for submission_id in submission_ids:
submission = Submission.get_from_id(submission_id, session)
submission_result = SubmissionResult.get_from_id(
(submission_id, dataset_id), session)
if submission_result is None:
# Not yet compiled, evaluated or scored.
score = None
ranking_score_details = None
else:
score = submission_result.score
try:
ranking_score_details = json.loads(
submission_result.ranking_score_details)
except (json.decoder.JSONDecodeError, TypeError):
# It may be blank.
ranking_score_details = None
# Data to send to remote rankings.
subchange_id = "%s%ss" % \
(int(make_timestamp(submission.timestamp)),
submission_id)
subchange_put_data = {
"submission": encode_id(submission_id),
"time": int(make_timestamp(submission.timestamp))}
# We're sending the unrounded score to RWS
if score is not None:
subchange_put_data["score"] = score
if ranking_score_details is not None:
subchange_put_data["extra"] = ranking_score_details
subchanges.append((subchange_id, subchange_put_data))
# Adding operations to the queue.
with self.operation_queue_lock:
for ranking in self.rankings:
for subchange_id, data in subchanges:
self.subchange_queue.setdefault(
ranking,
dict())[encode_id(subchange_id)] = data
def add_statements():
"""Support for statement translations.
Add external Statement objects to support statement translations.
The "official" statement is identified by its language code.
"""
with SessionGen(commit=True) as session:
session.execute("""\
CREATE TABLE IF NOT EXISTS statements (
id SERIAL NOT NULL,
language VARCHAR NOT NULL,
digest VARCHAR NOT NULL,
task_id INTEGER NOT NULL,
PRIMARY KEY (id),
CONSTRAINT cst_statements_task_id_language UNIQUE (task_id, language),
FOREIGN KEY(task_id) REFERENCES tasks (id)
ON DELETE CASCADE ON UPDATE CASCADE
)""")
session.execute("DROP INDEX IF EXISTS ix_statements_task_id;")
session.execute("CREATE INDEX ix_statements_task_id "
"ON statements (task_id)")
for task_id, digest in session.execute(
"SELECT id, statement FROM tasks;"):
session.execute("INSERT INTO statements "
"(language, digest, task_id) "
"VALUES ('', '%s', %s);" % (digest, task_id))
session.execute("ALTER TABLE tasks " "DROP COLUMN statement;")
session.execute("ALTER TABLE tasks "
"ADD COLUMN official_language VARCHAR;")
session.execute("UPDATE tasks " "SET official_language = '';")
session.execute("ALTER TABLE tasks "
"ALTER COLUMN official_language SET NOT NULL;")
def add_unique_constraints():
"""Add a bunch of constraints to the DB...
...that we were too lazy to do right away. See code for the
constraints.
"""
# First value of the pair is the table name, the second is the
# list of columns.
constraints = [
["attachments", ["task_id", "filename"]],
["evaluations", ["submission_id", "num"]],
["executables", ["submission_id", "filename"]],
["files", ["submission_id", "filename"]],
["managers", ["task_id", "filename"]],
["scores", ["rankingview_id", "task_id", "user_id"]],
["task_testcases", ["task_id", "num"]],
["tasks", ["contest_id", "num"]],
["tokens", ["submission_id"]],
["users", ["contest_id", "username"]],
["tasks", ["contest_id", "name"]],
]
with SessionGen(commit=True) as session:
for table, columns in constraints:
name = "cst_" + table + "_" + "_".join(columns)
columns_list = "(" + ", ".join(columns) + ")"
session.execute("ALTER TABLE %s "
"ADD CONSTRAINT %s "
"UNIQUE %s;" % (table, name, columns_list))
def changed_batch_iofile_parameters():
"""Params for Batch tasktype changed to support custom I/O filenames.
"""
import simplejson as json
with SessionGen(commit=True) as session:
for task_id, task_type_parameters in session.execute(
"SELECT id, task_type_parameters "
"FROM tasks WHERE task_type = 'Batch';"):
try:
parameters = json.loads(task_type_parameters)
except json.decoder.JSONDecodeError:
raise ValueError("Unable to decode parameter string "
"`%s'." % task_type_parameters)
if parameters[1] == "file":
parameters[1] = ["input.txt", "output.txt"]
elif parameters[1] == "nofile":
parameters[1] = ["", ""]
elif isinstance(parameters[1], list):
print "WARNING: already updated or unrecognized "\
"parameters in task %d" % task_id
else:
raise ValueError("I/O type `%s' not recognized." %
parameters[1])
session.execute(
"UPDATE tasks SET "
"task_type_parameters = :parameters "
"WHERE id = :task_id", {
"parameters": json.dumps(parameters),
"task_id": task_id
})
def put_file(self, digest, origin, description=""):
"""See FileCacherBackend.put_file().
"""
with SessionGen() as session:
# Check digest uniqueness
if FSObject.get_from_digest(digest, session) is not None:
logger.debug("File %s already on database, "
"dropping this one." % digest)
session.rollback()
# If it is not already present, copy the file into the
# lobject
else:
fso = FSObject(description=description)
logger.debug("Sending file %s to the database." % digest)
with open(origin, 'rb') as temp_file:
with fso.get_lobject(session, mode='wb') as lobject:
logger.debug("Large object created.")
buf = temp_file.read(self.CHUNK_SIZE)
while buf != '':
while len(buf) > 0:
written = lobject.write(buf)
buf = buf[written:]
if self.service is not None:
self.service._step()
buf = temp_file.read(self.CHUNK_SIZE)
fso.digest = digest
session.add(fso)
session.commit()
logger.debug("File %s sent to the database." % digest)
def fix_primary_statements():
"""Change the way primary submissions are stored.
"""
import simplejson as json
with SessionGen(commit=True) as session:
t_primary = dict()
for t_id, lang in session.execute(
"SELECT id, official_language "
"FROM tasks;"):
t_primary[t_id] = json.dumps([lang],
sort_keys=True,
separators=(',', ':'))
u_primary = dict()
ids = dict()
for s_id, task, lang in session.execute(
"SELECT s.id, t.name, s.language "
"FROM statements AS s, tasks AS t "
"WHERE s.task_id = t.id;"):
ids[s_id] = (task, lang)
for u_id, statements in session.execute(
"SELECT id, statements FROM users;"):
data = dict()
for s_id in statements:
task, lang = ids[s_id]
data.setdefault(task, []).append(lang)
for v in data.itervalues():
v.sort()
u_primary[u_id] = json.dumps(data,
sort_keys=True,
separators=(',', ':'))
session.execute("""\
ALTER TABLE tasks
DROP COLUMN official_language,
ADD COLUMN primary_statements VARCHAR;""")
session.execute("""\
ALTER TABLE users
DROP COLUMN statements,
ADD COLUMN primary_statements VARCHAR;""")
for t_id, txt in t_primary.iteritems():
session.execute(
"UPDATE tasks "
"SET primary_statements = '%s' "
"WHERE id = %d" % (txt, t_id))
for u_id, txt in u_primary.iteritems():
session.execute(
"UPDATE users "
"SET primary_statements = '%s' "
"WHERE id = %d" % (txt, u_id))
session.execute("""\
ALTER TABLE tasks
ALTER COLUMN primary_statements SET NOT NULL;""")
session.execute("""\
ALTER TABLE users
ALTER COLUMN primary_statements SET NOT NULL;""")
def add_evaluation_text_to_user_test():
"""Add the evaluation_text column to table user tests.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE user_tests "
"ADD COLUMN evaluation_text VARCHAR;")
def add_time_and_memory_on_tests():
"""Add memory_used and execution_time on UserTests.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE user_tests "
"ADD COLUMN memory_used INTEGER, "
"ADD COLUMN execution_time DOUBLE PRECISION;")
def delete(self, digest):
"""See FileCacherBackend.delete().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
fso.delete()
session.commit()
def add_ranking_score_details():
"""Add a field with the score details to send to RWS
Please rescore solutions.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE submissions "
"ADD COLUMN ranking_score_details VARCHAR;")
def change_token_constraints():
"""Fix token constraints to avoid ambiguos corner cases.
Set some token_* fields to != None or to != zero since these
values would cause a "behavior" (from the user-perspective)
that is identical to other, more "reasonable" and natural
value combinations.
"""
with SessionGen(commit=True) as session:
for table in ["contests", "tasks"]:
session.execute("""
UPDATE %s
SET token_initial = token_max
WHERE token_initial > token_max;
""" % table)
session.execute("""
UPDATE %s
SET (token_initial, token_max, token_total) = (NULL, NULL, NULL)
WHERE token_max = 0 OR token_total = 0;
""" % table)
session.execute("""
UPDATE %s
SET token_min_interval = 0
WHERE token_min_interval IS NULL;
""" % table)
session.execute("""
UPDATE %s
SET (token_gen_time, token_gen_number) = (1, 0)
WHERE token_gen_time IS NULL OR token_gen_number IS NULL;
""" % table)
session.execute("""
ALTER TABLE contests
ADD CONSTRAINT contests_check CHECK (token_initial <= token_max),
DROP CONSTRAINT contests_token_max_check,
DROP CONSTRAINT contests_token_total_check,
DROP CONSTRAINT contests_token_gen_time_check,
ADD CONSTRAINT contests_token_max_check CHECK (token_max > 0),
ADD CONSTRAINT contests_token_total_check CHECK (token_total > 0),
ADD CONSTRAINT contests_token_gen_time_check CHECK (token_gen_time >= 0),
ALTER COLUMN token_min_interval SET NOT NULL,
ALTER COLUMN token_gen_time SET NOT NULL,
ALTER COLUMN token_gen_number SET NOT NULL;
""")
session.execute("""
ALTER TABLE tasks
ADD CONSTRAINT tasks_check CHECK (token_initial <= token_max),
DROP CONSTRAINT tasks_token_max_check,
DROP CONSTRAINT tasks_token_total_check,
DROP CONSTRAINT tasks_token_gen_time_check,
ADD CONSTRAINT tasks_token_max_check CHECK (token_max > 0),
ADD CONSTRAINT tasks_token_total_check CHECK (token_total > 0),
ADD CONSTRAINT tasks_token_gen_time_check CHECK (token_gen_time >= 0),
ALTER COLUMN token_min_interval SET NOT NULL,
ALTER COLUMN token_gen_time SET NOT NULL,
ALTER COLUMN token_gen_number SET NOT NULL;
""")
def change_scoretype_names():
"""Remove the ScoreType prefix from every score type name.
"""
with SessionGen(commit=True) as session:
for score_type in ["Sum", "Relative", "GroupMin", "GroupMul"]:
session.execute("UPDATE tasks SET score_type = '%s' "
"WHERE score_type = 'ScoreType%s';" %
(score_type, score_type))
def change_tasktype_names():
"""Remove the TaskType prefix from every task type name.
"""
with SessionGen(commit=True) as session:
session.execute("UPDATE tasks SET task_type = 'Batch' "
"WHERE task_type = 'TaskTypeBatch';")
session.execute("UPDATE tasks SET task_type = 'OutputOnly' "
"WHERE task_type = 'TaskTypeOutputOnly';")
def task_score(user, task):
"""Return the score of a user on a task.
user (User): the user for which to compute the score.
task (Task): the task for which to compute the score.
return (float, bool): the score of user on task, and True if the
score could change because of a submission
yet to score.
"""
def waits_for_score(submission):
"""Return if submission could be scored but it currently is
not.
submission (Submission): the submission to check.
"""
return submission.compilation_outcome != "fail" and \
not submission.scored()
# The score of the last submission (if valid, otherwise 0.0).
last_score = 0.0
# The maximum score amongst the tokened submissions (invalid
# scores count as 0.0).
max_tokened_score = 0.0
# If the score could change due to submission still being compiled
# / evaluated / scored.
partial = False
with SessionGen(commit=False) as session:
submissions = session.query(Submission).\
filter(Submission.user == user).\
filter(Submission.task == task).\
order_by(Submission.timestamp).all()
if submissions == []:
return 0.0, False
# Last score: if the last submission is scored we use that,
# otherwise we use 0.0 (and mark that the score is partial
# when the last submission could be scored).
if submissions[-1].scored():
last_score = submissions[-1].score
elif waits_for_score(submissions[-1]):
partial = True
for submission in submissions:
if submission.token is not None:
if submission.scored():
max_tokened_score = max(max_tokened_score,
submission.score)
elif waits_for_score(submission):
partial = True
return max(last_score, max_tokened_score), partial
def invalidate_submission(self,
submission_id=None,
user_id=None,
task_id=None):
"""Request for invalidating some scores.
Invalidate the scores of the Submission whose ID is
submission_id or, if None, of those whose user is user_id
and/or whose task is task_id or, if both None, of those that
belong to the contest this service is running for.
submission_id (int): id of the submission to invalidate, or
None.
user_id (int): id of the user to invalidate, or None.
task_id (int): id of the task to invalidate, or None.
"""
logger.info("Invalidation request received.")
# Validate arguments
# TODO Check that all these objects belong to this contest.
with SessionGen(commit=True) as session:
submissions = get_submissions(
# Give contest_id only if all others are None.
self.contest_id
if {user_id, task_id, submission_id} == {None} else None,
user_id,
task_id,
submission_id,
session)
logger.info("Submissions to invalidate scores for: %d." %
len(submissions))
if len(submissions) == 0:
return
new_submissions_to_score = set()
for submission in submissions:
# If the submission is not evaluated, it does not have
# a score to invalidate, and, when evaluated,
# ScoringService will be prompted to score it. So in
# that case we do not have to do anything.
if submission.evaluated():
submission.invalidate_score()
new_submissions_to_score.add(submission.id)
old_s = len(self.submissions_to_score)
old_t = len(self.submissions_to_token)
self.submissions_to_score |= new_submissions_to_score
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions,
None,
0.5,
immediately=False)
def is_contest_id(contest_id):
"""Return if there is a contest with the given id in the database.
contest_id (int): the id to query.
return (boolean): True if there is such a contest.
"""
with SessionGen(commit=False) as session:
contest = session.query(Contest).filter_by(id=contest_id).first()
return contest is not None
def get_submissions(contest_id,
submission_id=None,
user_id=None,
task_id=None):
"""Return a list of submission_ids restricted with the given
information.
We allow at most one of the parameters to be non-None. If all
parameters are, we return all submissions for the given contest.
contest_id (int): the id of the contest.
submission_id (int): id of the submission to invalidate, or None.
user_id (int): id of the user we want to invalidate, or None.
task_id (int): id of the task we want to invalidate, or None.
level (string): 'compilation' or 'evaluation'
"""
if [x is not None
for x in [submission_id, user_id, task_id]].count(True) > 1:
err_msg = "Too many arguments for invalidate_submission."
logger.warning(err_msg)
raise ValueError(err_msg)
submission_ids = []
if submission_id is not None:
submission_ids = [submission_id]
elif user_id is not None:
with SessionGen(commit=False) as session:
user = User.get_from_id(user_id, session)
submission_ids = [x.id for x in user.submissions]
elif task_id is not None:
with SessionGen(commit=False) as session:
submissions = session.query(Submission)\
.join(Task).filter(Task.id == task_id)
submission_ids = [x.id for x in submissions]
else:
with SessionGen(commit=False) as session:
contest = session.query(Contest).\
filter_by(id=contest_id).first()
submission_ids = [x.id for x in contest.get_submissions()]
return submission_ids
def describe(self, digest):
"""See FileCacherBackend.describe().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
return fso.description
else:
return None
def drop_ranking_view():
"""Remove the useless tables.
Ranking views and the accessory scores tables were intended to
be used, but this is not true anymore.
"""
with SessionGen(commit=True) as session:
session.execute("DROP TABLE scores;")
session.execute("DROP TABLE rankingviews;")
def support_output_only():
"""Increase support for output only tasks
By allowing NULL time and memory limits
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE tasks "
"ALTER COLUMN time_limit DROP NOT NULL,"
"ALTER COLUMN memory_limit DROP NOT NULL;")
def get_size(self, digest):
"""See FileCacherBackend.get_size().
"""
with SessionGen() as session:
fso = FSObject.get_from_digest(digest, session)
if fso is not None:
with fso.get_lobject(session, mode='rb') as lobject:
return lobject.seek(0, os.SEEK_END)
else:
return None
def make_contest_description_not_null():
"""Make the Contest.description field NOT NULL.
"""
with SessionGen(commit=True) as session:
session.execute("""\
UPDATE contests
SET description = '' WHERE description IS NULL;""")
session.execute("""\
ALTER TABLE contests
ALTER COLUMN description SET NOT NULL;""")
def constraints_on_tokens():
"""Better constraints for token information.
We allow token_initial to be NULL, which means that the tokens
are disabled for that contest/task. Moreover, all information
are required to be non-negative (or positive when
appropriate).
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE contests "
"ALTER COLUMN token_initial "
"DROP NOT NULL;")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_initial_check "
"CHECK (token_initial >= 0);")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_max_check "
"CHECK (token_max >= 0);")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_total_check "
"CHECK (token_total >= 0);")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_min_interval_check "
"CHECK (token_min_interval >= 0);")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_gen_time_check "
"CHECK (token_gen_time > 0);")
session.execute("ALTER TABLE contests "
"ADD CONSTRAINT contests_token_gen_number_check "
"CHECK (token_gen_number >= 0);")
session.execute("ALTER TABLE tasks "
"ALTER COLUMN token_initial "
"DROP NOT NULL;")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_initial_check "
"CHECK (token_initial >= 0);")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_max_check "
"CHECK (token_max >= 0);")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_total_check "
"CHECK (token_total >= 0);")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_min_interval_check "
"CHECK (token_min_interval >= 0);")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_gen_time_check "
"CHECK (token_gen_time > 0);")
session.execute("ALTER TABLE tasks "
"ADD CONSTRAINT tasks_token_gen_number_check "
"CHECK (token_gen_number >= 0);")
def add_ignore_on_questions():
"""Possibility to ignore users' questions.
We simply add a field "ignored" in the questions table.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE questions "
"ADD COLUMN ignored BOOLEAN;")
session.execute("UPDATE questions SET ignored = False;")
session.execute("ALTER TABLE questions "
"ALTER COLUMN ignored SET NOT NULL;")
def add_extra_time():
"""Add extra time for the users
Defaults to zero.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE users "
"ADD COLUMN extra_time INTERVAL;")
session.execute("UPDATE users " "SET extra_time = '0 seconds';")
session.execute("ALTER TABLE users "
"ALTER COLUMN extra_time SET NOT NULL;")
def add_timezones():
"""Add support for per-contest and per-user timezones
By adding the Contest.timezone and User.timezone fields.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE contests "
"ADD COLUMN timezone VARCHAR;")
session.execute("ALTER TABLE users "
"ALTER timezone DROP NOT NULL,"
"ALTER timezone TYPE VARCHAR USING NULL;")
def add_languages():
"""Add a list of languages to each user
Defaults to empty list.
"""
with SessionGen(commit=True) as session:
session.execute("ALTER TABLE users "
"ADD COLUMN languages VARCHAR[];")
session.execute("UPDATE users " "SET languages = '{}';")
session.execute("ALTER TABLE users "
"ALTER COLUMN languages SET NOT NULL;")
def add_user(contest_id, first_name, last_name, username, password, ip_address,
email, hidden):
with SessionGen(commit=True) as session:
contest = Contest.get_from_id(contest_id, session)
user = User(first_name=first_name,
last_name=last_name,
username=username,
password=password,
email=email,
ip=ip_address,
hidden=hidden,
contest=contest)
session.add(user)
def default_argument_parser(description, cls, ask_contest=None):
"""Default argument parser for services - in two versions: needing
a contest_id, or not.
description (string): description of the service.
cls (class): service's class.
ask_contest (function): None if the service does not require a
contest, otherwise a function that returns
a contest_id (after asking the admins?)
return (object): an instance of a service.
"""
parser = ArgumentParser(description=description)
parser.add_argument("shard", nargs="?", type=int, default=-1)
# We need to allow using the switch "-c" also for services that do
# not need the contest_id because RS needs to be able to restart
# everything without knowing which is which.
contest_id_help = "id of the contest to automatically load"
if ask_contest is None:
contest_id_help += " (ignored)"
parser.add_argument("-c",
"--contest-id",
help=contest_id_help,
nargs="?",
type=int)
args = parser.parse_args()
# If the shard is -1 (i.e., unspecified) we find it basing on the
# local IP addresses
if args.shard == -1:
addrs = find_local_addresses()
args.shard = get_shard_from_addresses(cls.__name__, addrs)
if ask_contest is not None:
if args.contest_id is not None:
# Test if there is a contest with the given contest id.
from cms.db.SQLAlchemyAll import Contest, SessionGen
with SessionGen(commit=False) as session:
contest = session.query(Contest).\
filter_by(id=args.contest_id).first()
if contest is None:
print >> sys.stderr, "There is no contest " \
"with the specified id. Please try again."
sys.exit(1)
return cls(args.shard, args.contest_id)
else:
return cls(args.shard, ask_contest())
else:
return cls(args.shard)
def search_jobs_not_done(self):
"""Look in the database for submissions that have not been
scored for no good reasons. Put the missing job in the queue.
"""
# Do this only if we are not still loading old submission
# (from the start of the service).
if self.scoring_old_submission:
return True
with SessionGen(commit=False) as session:
new_submission_ids_to_score = set([])
new_submission_ids_to_token = set([])
contest = session.query(Contest).\
filter_by(id=self.contest_id).first()
for submission in contest.get_submissions():
for dataset in get_autojudge_datasets(submission.task):
# If a submission result does not yet exist, then we don't
# need to score it.
r = SubmissionResult.get_from_id(
(submission.id, dataset.id),
session)
if r is None:
continue
x = (r.submission_id, r.dataset_id)
if r is not None and (r.evaluated()
or r.compilation_outcome == "fail") \
and x not in self.submission_ids_scored:
new_submission_ids_to_score.add(x)
if r.submission.tokened() and r.submission_id not in \
self.submission_ids_tokened:
new_submission_ids_to_token.add(
(r.submission_id,
make_timestamp(r.submission.token.timestamp)))
new_s = len(new_submission_ids_to_score)
old_s = len(self.submission_ids_to_score)
new_t = len(new_submission_ids_to_token)
old_t = len(self.submission_ids_to_token)
logger.info("Submissions found to score/token: %d, %d." %
(new_s, new_t))
if new_s + new_t > 0:
self.submission_ids_to_score |= new_submission_ids_to_score
self.submission_ids_to_token |= new_submission_ids_to_token
if old_s + old_t == 0:
self.add_timeout(self.score_old_submissions, None,
0.5, immediately=False)
# Run forever.
return True