class Reimporter:
"""This script reimports a contest from disk using YamlLoader
The data parsed by YamlLoader is used to update a Contest that's
already existing in the database.
"""
def __init__(self, path, contest_id, force):
self.old_contest_id = contest_id
self.force = force
self.file_cacher = FileCacher()
self.loader = YamlLoader(os.path.realpath(path), self.file_cacher)
def _update_columns(self, old_object, new_object):
for prp in old_object._col_props:
if hasattr(new_object, prp.key):
setattr(old_object, prp.key, getattr(new_object, prp.key))
def _update_object(self, old_object, new_object):
# This method copies the scalar column properties from the new
# object into the old one, and then tries to do the same for
# relationships too. The data model isn't a tree: for example
# there are two distinct paths from Contest to Submission, one
# through User and one through Task. Yet, at the moment, if we
# ignore Submissions and UserTest (and thus their results, too)
# we get a tree-like structure and Task.active_dataset and
# Submission.token are the only scalar relationships that don't
# refer to the parent. Therefore, if we catch these as special
# cases, we can use a simple DFS to explore the whole data
# graph, recursing only on "vector" relationships.
# TODO Find a better way to handle all of this.
self._update_columns(old_object, new_object)
for prp in old_object._rel_props:
old_value = getattr(old_object, prp.key)
new_value = getattr(new_object, prp.key)
# Special case #1: Contest.announcements, User.questions,
# User.messages
if _is_rel(prp, Contest.announcements) or \
_is_rel(prp, User.questions) or \
_is_rel(prp, User.messages):
# A loader should not provide new Announcements,
# Questions or Messages, since they are data generated
# by the users during the contest: don't update them.
# TODO Warn the admin if these attributes are non-empty
# collections.
pass
# Special case #2: Task.datasets
if _is_rel(prp, Task.datasets):
old_datasets = dict((d.description, d) for d in old_value)
new_datasets = dict((d.description, d) for d in new_value)
for key in set(new_datasets.keys()):
if key not in old_datasets:
# create
temp = new_datasets[key]
new_value.remove(temp)
old_value.append(temp)
else:
# update
self._update_object(old_datasets[key],
new_datasets[key])
# Special case #3: Task.active_dataset
elif _is_rel(prp, Task.active_dataset):
# We don't want to update the existing active dataset.
pass
# Special case #4: User.submissions, Task.submissions,
# User.user_tests, Task.user_tests
elif _is_rel(prp, User.submissions) or \
_is_rel(prp, Task.submissions) or \
_is_rel(prp, User.user_tests) or \
_is_rel(prp, Task.user_tests):
# A loader should not provide new Submissions or
# UserTests, since they are data generated by the users
# during the contest: don't update them.
# TODO Warn the admin if these attributes are non-empty
# collections.
pass
# Special case #5: Submission.token
elif _is_rel(prp, Submission.token):
# We should never reach this point! We should never try
# to update Submissions! We could even assert False...
pass
# General case #1: a dict
elif isinstance(old_value, dict):
for key in set(old_value.keys()) | set(new_value.keys()):
if key in new_value:
if key not in old_value:
# create
# FIXME This hack is needed because of some
# funny behavior of SQLAlchemy-instrumented
# collections when copying values, that
# resulted in new objects being added to
# the session. We need to investigate it.
temp = new_value[key]
del new_value[key]
old_value[key] = temp
else:
# update
self._update_object(old_value[key], new_value[key])
else:
# delete
del old_value[key]
# General case #2: a list
elif isinstance(old_value, list):
old_len = len(old_value)
new_len = len(new_value)
for i in xrange(min(old_len, new_len)):
self._update_object(old_value[i], new_value[i])
if old_len > new_len:
del old_value[new_len:]
elif new_len > old_len:
for i in xrange(old_len, new_len):
# FIXME This hack is needed because of some
# funny behavior of SQLAlchemy-instrumented
# collections when copying values, that
# resulted in new objects being added to the
# session. We need to investigate it.
temp = new_value[i]
del new_value[i]
old_value.append(temp)
# General case #3: a parent object
elif isinstance(old_value, Base):
# No need to climb back up the recursion tree...
pass
# General case #4: None
elif old_value is None:
# That should only happen in case of a scalar
# relationship (i.e. a many-to-one or a one-to-one)
# that is nullable. "Parent" relationships aren't
# nullable, so the only possible cases are the active
# datasets and the tokens, but we should have already
# caught them. We could even assert False...
pass
else:
raise RuntimeError(
"Unknown type of relationship for %s.%s." %
(prp.parent.class_.__name__, prp.key))
def do_reimport(self):
"""Get the contest from the Loader and merge it."""
with SessionGen(commit=False) as session:
# Load the old contest from the database.
old_contest = Contest.get_from_id(self.old_contest_id, session)
old_users = dict((x.username, x) for x in old_contest.users)
old_tasks = dict((x.name, x) for x in old_contest.tasks)
# Load the new contest from the filesystem.
new_contest, new_tasks, new_users = self.loader.get_contest()
new_users = dict((x["username"], x) for x in new_users)
new_tasks = dict((x["name"], x) for x in new_tasks)
# Updates contest-global settings that are set in new_contest.
self._update_columns(old_contest, new_contest)
# Do the actual merge: compare all users of the old and of
# the new contest and see if we need to create, update or
# delete them. Delete only if authorized, fail otherwise.
users = set(old_users.keys()) | set(new_users.keys())
for user in users:
old_user = old_users.get(user, None)
new_user = new_users.get(user, None)
if old_user is None:
# Create a new user.
logger.info("Creating user %s" % user)
new_user = self.loader.get_user(new_user)
old_contest.users.append(new_user)
elif new_user is not None:
# Update an existing user.
logger.info("Updating user %s" % user)
new_user = self.loader.get_user(new_user)
self._update_object(old_user, new_user)
else:
# Delete an existing user.
if self.force:
logger.info("Deleting user %s" % user)
old_contest.users.remove(old_user)
else:
logger.critical(
"User %s exists in old contest, but "
"not in the new one. Use -f to force." %
user)
return False
# The same for tasks.
tasks = set(old_tasks.keys()) | set(new_tasks.keys())
for task in tasks:
old_task = old_tasks.get(task, None)
new_task = new_tasks.get(task, None)
if old_task is None:
# Create a new task.
logger.info("Creating task %s" % task)
new_task = self.loader.get_task(new_task)
old_contest.tasks.append(new_task)
elif new_task is not None:
# Update an existing task.
logger.info("Updating task %s" % task)
new_task = self.loader.get_task(new_task)
self._update_object(old_task, new_task)
else:
# Delete an existing task.
if self.force:
logger.info("Deleting task %s" % task)
old_contest.tasks.remove(old_task)
else:
logger.critical(
"Task %s exists in old contest, but "
"not in the new one. Use -f to force." %
task)
return False
session.commit()
logger.info("Reimport finished (contest id: %s)." %
self.old_contest_id)
return True
class Importer:
"""This script imports a contest from disk using YamlLoader.
The data parsed by YamlLoader is used to create a new Contest in
the database.
"""
def __init__(self, path, drop, test, zero_time, user_number):
self.drop = drop
self.test = test
self.zero_time = zero_time
self.user_number = user_number
self.file_cacher = FileCacher()
self.loader = YamlLoader(os.path.realpath(path), self.file_cacher)
def _prepare_db(self):
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
def do_import(self):
"""Get the contest from the Loader and store it."""
if not self._prepare_db():
return False
# Get the contest
contest, tasks, users = self.loader.get_contest()
# Get the tasks
for task in tasks:
contest.tasks.append(self.loader.get_task(task))
# Get the users or, if asked, generate them
if self.user_number is None:
for user in users:
contest.users.append(self.loader.get_user(user))
else:
logger.info("Generating %s random users." % self.user_number)
contest.users = [User("User %d" % i,
"Last name %d" % i,
"user%03d" % i)
for i in xrange(self.user_number)]
# Apply the modification flags
if self.zero_time:
contest.start = datetime.datetime(1970, 1, 1)
contest.stop = datetime.datetime(1970, 1, 1)
elif self.test:
contest.start = datetime.datetime(1970, 1, 1)
contest.stop = datetime.datetime(2100, 1, 1)
for user in contest.users:
user.password = '******'
user.ip = None
# Store
logger.info("Creating contest on the database.")
with SessionGen() as session:
session.add(contest)
session.commit()
contest_id = contest.id
logger.info("Import finished (new contest id: %s)." % contest_id)
