def do_import(self):
"""Take care of creating the database structure, delegating
the loading of the contest data and putting them on the
database.
"""
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
contest = Contest.import_from_dict(self.loader.import_contest(self.path))
logger.info("Creating contest on the database.")
with SessionGen() as session:
session.add(contest)
logger.info("Analyzing database.")
session.commit()
contest_id = contest.id
analyze_all_tables(session)
logger.info("Import finished (new contest id: %s)." % contest_id)
return True
def do_import(self):
"""Take care of creating the database structure, delegating
the loading of the contest data and putting them on the
database.
"""
logger.info("Creating database structure.")
if self.drop:
try:
with SessionGen() as session:
FSObject.delete_all(session)
session.commit()
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
contest = Contest.import_from_dict(
self.loader.import_contest(self.path))
logger.info("Creating contest on the database.")
with SessionGen() as session:
session.add(contest)
logger.info("Analyzing database.")
session.commit()
contest_id = contest.id
analyze_all_tables(session)
logger.info("Import finished (new contest id: %s)." % contest_id)
return True
def do_reimport(self):
"""Ask the loader to load the contest and actually merge the
two.
"""
# Create the dict corresponding to the new contest.
yaml_contest = self.loader.import_contest(self.path)
yaml_users = dict(((x['username'], x) for x in yaml_contest['users']))
yaml_tasks = dict(((x['name'], x) for x in yaml_contest['tasks']))
with SessionGen(commit=False) as session:
# Create the dict corresponding to the old contest, from
# the database.
contest = Contest.get_from_id(self.contest_id, session)
cms_contest = contest.export_to_dict()
cms_users = dict((x['username'], x) for x in cms_contest['users'])
cms_tasks = dict((x['name'], x) for x in cms_contest['tasks'])
# Delete the old contest from the database.
session.delete(contest)
session.flush()
# Do the actual merge: first of all update all users of
# the old contest with the corresponding ones from the new
# contest; if some user is present in the old contest but
# not in the new one we check if we have to fail or remove
# it and, in the latter case, add it to a list
users_to_remove = []
for user_num, user in enumerate(cms_contest['users']):
try:
user_submissions = \
cms_contest['users'][user_num]['submissions']
cms_contest['users'][user_num] = \
yaml_users[user['username']]
cms_contest['users'][user_num]['submissions'] = \
user_submissions
except KeyError:
if self.force:
logger.warning("User %s exists in old contest, but "
"not in the new one" % user['username'])
users_to_remove.append(user_num)
session.delete(contest.users[user_num])
else:
logger.error("User %s exists in old contest, but "
"not in the new one" % user['username'])
return False
# Delete the users
for user_num in users_to_remove:
del cms_contest['users'][user_num]
# The append the users in the new contest, not present in
# the old one.
for user in yaml_contest['users']:
if user['username'] not in cms_users.keys():
cms_contest['users'].append(user)
# The same for tasks: update old tasks.
tasks_to_remove = []
for task_num, task in enumerate(cms_contest['tasks']):
try:
cms_contest['tasks'][task_num] = yaml_tasks[task['name']]
except KeyError:
if self.force:
logger.warning("Task %s exists in old contest, but "
"not in the new one" % task['name'])
tasks_to_remove.append(task_num)
session.delete(contest.tasks[task_num])
else:
logger.error("Task %s exists in old contest, but "
"not in the new one" % task['name'])
return False
# Delete the tasks
for task_num in tasks_to_remove:
del cms_contest['tasks'][task_num]
# And add new tasks.
for task in yaml_contest['tasks']:
if task['name'] not in cms_tasks.keys():
cms_contest['tasks'].append(task)
# Reimport the contest in the db, with the previous ID.
contest = Contest.import_from_dict(cms_contest)
contest.id = self.contest_id
session.add(contest)
session.flush()
logger.info("Analyzing database.")
analyze_all_tables(session)
session.commit()
logger.info("Reimport of contest %s finished." % self.contest_id)
return True
def do_import(self):
"""Run the actual import code.
"""
logger.operation = "importing contest from %s" % self.import_source
logger.info("Starting import.")
if not os.path.isdir(self.import_source):
if self.import_source.endswith(".zip"):
archive = zipfile.ZipFile(self.import_source, "r")
file_names = archive.infolist()
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
elif self.import_source.endswith(".tar.gz") \
or self.import_source.endswith(".tgz") \
or self.import_source.endswith(".tar.bz2") \
or self.import_source.endswith(".tbz2") \
or self.import_source.endswith(".tar"):
archive = tarfile.open(name=self.import_source)
file_names = archive.getnames()
else:
logger.critical("Unable to import from %s." %
self.import_source)
return False
root = find_root_of_archive(file_names)
if root is None:
logger.critical("Cannot find a root directory in %s." %
self.import_source)
return False
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
self.import_dir = os.path.join(self.import_dir, root)
if self.drop:
logger.info("Dropping and recreating the database.")
try:
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
logger.info("Reading JSON file...")
with open(os.path.join(self.import_dir, "contest.json")) as fin:
contest_json = json.load(fin)
if self.no_submissions:
for user in contest_json["users"]:
user["submissions"] = []
user["user_tests"] = []
if not self.only_files:
with SessionGen(commit=False) as session:
# Import the contest in JSON format.
logger.info("Importing the contest from JSON file.")
contest = Contest.import_from_dict(contest_json)
session.add(contest)
session.flush()
contest_id = contest.id
contest_files = contest.enumerate_files()
session.commit()
if not self.no_files:
logger.info("Importing files.")
files_dir = os.path.join(self.import_dir, "files")
descr_dir = os.path.join(self.import_dir, "descriptions")
for digest in contest_files:
file_ = os.path.join(files_dir, digest)
desc = os.path.join(descr_dir, digest)
if not os.path.exists(file_) or not os.path.exists(desc):
logger.error("Some files needed to the contest "
"are missing in the import directory. "
"The import will continue. Be aware.")
if not self.safe_put_file(file_, desc):
logger.critical("Unable to put file `%s' in the database. "
"Aborting. Please remove the contest "
"from the database." % file_)
# TODO: remove contest from the database.
return False
logger.info("Import finished (contest id: %s)." % contest_id)
logger.operation = ""
# If we extracted an archive, we remove it.
if self.import_dir != self.import_source:
shutil.rmtree(self.import_dir)
return True
def do_reimport(self):
"""Ask the loader to load the contest and actually merge the
two.
"""
# Create the dict corresponding to the new contest.
yaml_contest = self.loader.import_contest(self.path)
yaml_users = dict(((x['username'], x) for x in yaml_contest['users']))
yaml_tasks = dict(((x['name'], x) for x in yaml_contest['tasks']))
with SessionGen(commit=False) as session:
# Create the dict corresponding to the old contest, from
# the database.
contest = Contest.get_from_id(self.contest_id, session)
cms_contest = contest.export_to_dict()
cms_users = dict((x['username'], x) for x in cms_contest['users'])
cms_tasks = dict((x['name'], x) for x in cms_contest['tasks'])
# Delete the old contest from the database.
session.delete(contest)
session.flush()
# Do the actual merge: first of all update all users of
# the old contest with the corresponding ones from the new
# contest; if some user is present in the old contest but
# not in the new one we check if we have to fail or remove
# it and, in the latter case, add it to a list
users_to_remove = []
for user_num, user in enumerate(cms_contest['users']):
if user['username'] in yaml_users:
yaml_user = yaml_users[user['username']]
yaml_user['submissions'] = user['submissions']
yaml_user['user_tests'] = user['user_tests']
yaml_user['questions'] = user['questions']
yaml_user['messages'] = user['messages']
cms_contest['users'][user_num] = yaml_user
else:
if self.force:
logger.warning("User %s exists in old contest, but "
"not in the new one" % user['username'])
users_to_remove.append(user_num)
# FIXME Do we need really to do this, given that
# we already deleted the whole contest?
session.delete(contest.users[user_num])
else:
logger.error("User %s exists in old contest, but "
"not in the new one" % user['username'])
return False
# Delete the users
for user_num in users_to_remove:
del cms_contest['users'][user_num]
# The append the users in the new contest, not present in
# the old one.
for user in yaml_contest['users']:
if user['username'] not in cms_users.keys():
cms_contest['users'].append(user)
# The same for tasks: update old tasks.
tasks_to_remove = []
for task_num, task in enumerate(cms_contest['tasks']):
if task['name'] in yaml_tasks:
yaml_task = yaml_tasks[task['name']]
cms_contest['tasks'][task_num] = yaml_task
else:
if self.force:
logger.warning("Task %s exists in old contest, but "
"not in the new one" % task['name'])
tasks_to_remove.append(task_num)
# FIXME Do we need really to do this, given that
# we already deleted the whole contest?
session.delete(contest.tasks[task_num])
else:
logger.error("Task %s exists in old contest, but "
"not in the new one" % task['name'])
return False
# Delete the tasks
for task_num in tasks_to_remove:
del cms_contest['tasks'][task_num]
# And add new tasks.
for task in yaml_contest['tasks']:
if task['name'] not in cms_tasks.keys():
cms_contest['tasks'].append(task)
# Reimport the contest in the db, with the previous ID.
contest = Contest.import_from_dict(cms_contest)
contest.id = self.contest_id
session.add(contest)
session.flush()
logger.info("Analyzing database.")
analyze_all_tables(session)
session.commit()
logger.info("Reimport of contest %s finished." % self.contest_id)
return True
def do_import(self):
"""Run the actual import code.
"""
logger.operation = "importing contest from %s" % self.import_source
logger.info("Starting import.")
if not os.path.isdir(self.import_source):
if self.import_source.endswith(".zip"):
archive = zipfile.ZipFile(self.import_source, "r")
file_names = archive.infolist()
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
elif self.import_source.endswith(".tar.gz") \
or self.import_source.endswith(".tgz") \
or self.import_source.endswith(".tar.bz2") \
or self.import_source.endswith(".tbz2") \
or self.import_source.endswith(".tar"):
archive = tarfile.open(name=self.import_source)
file_names = archive.getnames()
else:
logger.critical("Unable to import from %s." %
self.import_source)
return False
root = find_root_of_archive(file_names)
if root is None:
logger.critical("Cannot find a root directory in %s." %
self.import_source)
return False
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
self.import_dir = os.path.join(self.import_dir, root)
if self.drop:
logger.info("Dropping and recreating the database.")
try:
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
if not self.no_files:
logger.info("Importing files.")
files_dir = os.path.join(self.import_dir, "files")
descr_dir = os.path.join(self.import_dir, "descriptions")
files = set(os.listdir(files_dir))
for _file in files:
if not self.safe_put_file(os.path.join(files_dir, _file),
os.path.join(descr_dir, _file)):
return False
if not self.only_files:
with SessionGen(commit=False) as session:
# Import the contest in JSON format.
logger.info("Importing the contest from JSON file.")
with open(os.path.join(self.import_dir,
"contest.json")) as fin:
contest = Contest.import_from_dict(json.load(fin))
session.add(contest)
# Check that no files were missing (only if files were
# imported).
if not self.no_files:
contest_files = contest.enumerate_files()
missing_files = contest_files.difference(files)
if len(missing_files) > 0:
logger.warning("Some files needed to the contest "
"are missing in the import directory.")
session.flush()
contest_id = contest.id
session.commit()
logger.info("Import finished (contest id: %s)." % contest_id)
logger.operation = ""
# If we extracted an archive, we remove it.
if self.import_dir != self.import_source:
shutil.rmtree(self.import_dir)
return True
def do_import(self):
"""Run the actual import code.
"""
logger.operation = "importing contest from %s" % self.import_source
logger.info("Starting import.")
if not os.path.isdir(self.import_source):
if self.import_source.endswith(".zip"):
archive = zipfile.ZipFile(self.import_source, "r")
file_names = archive.infolist()
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
elif self.import_source.endswith(".tar.gz") \
or self.import_source.endswith(".tgz") \
or self.import_source.endswith(".tar.bz2") \
or self.import_source.endswith(".tbz2") \
or self.import_source.endswith(".tar"):
archive = tarfile.open(name=self.import_source)
file_names = archive.getnames()
else:
logger.critical("Unable to import from %s." %
self.import_source)
return False
root = find_root_of_archive(file_names)
if root is None:
logger.critical("Cannot find a root directory in %s." %
self.import_source)
return False
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
self.import_dir = os.path.join(self.import_dir, root)
if self.drop:
logger.info("Dropping and recreating the database.")
try:
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
logger.info("Reading JSON file...")
with open(os.path.join(self.import_dir, "contest.json")) as fin:
contest_json = json.load(fin)
if self.no_submissions:
for user in contest_json["users"]:
user["submissions"] = []
user["user_tests"] = []
if not self.only_files:
with SessionGen(commit=False) as session:
# Import the contest in JSON format.
logger.info("Importing the contest from JSON file.")
contest = Contest.import_from_dict(contest_json)
session.add(contest)
session.flush()
contest_id = contest.id
contest_files = contest.enumerate_files()
session.commit()
if not self.no_files:
logger.info("Importing files.")
files_dir = os.path.join(self.import_dir, "files")
descr_dir = os.path.join(self.import_dir, "descriptions")
for digest in contest_files:
file_ = os.path.join(files_dir, digest)
desc = os.path.join(descr_dir, digest)
print open(desc).read()
if not os.path.exists(file_) or not os.path.exists(desc):
logger.error("Some files needed to the contest "
"are missing in the import directory. "
"The import will continue. Be aware.")
if not self.safe_put_file(file_, desc):
logger.critical("Unable to put file `%s' in the database. "
"Aborting. Please remove the contest "
"from the database." % file_)
# TODO: remove contest from the database.
return False
logger.info("Import finished (contest id: %s)." % contest_id)
logger.operation = ""
# If we extracted an archive, we remove it.
if self.import_dir != self.import_source:
shutil.rmtree(self.import_dir)
return True
def do_import(self):
"""Run the actual import code.
"""
logger.operation = "importing contest from %s" % self.import_source
logger.info("Starting import.")
if not os.path.isdir(self.import_source):
if self.import_source.endswith(".zip"):
archive = zipfile.ZipFile(self.import_source, "r")
file_names = archive.infolist()
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
elif self.import_source.endswith(".tar.gz") \
or self.import_source.endswith(".tgz") \
or self.import_source.endswith(".tar.bz2") \
or self.import_source.endswith(".tbz2") \
or self.import_source.endswith(".tar"):
archive = tarfile.open(name=self.import_source)
file_names = archive.getnames()
else:
logger.critical("Unable to import from %s." %
self.import_source)
return False
root = find_root_of_archive(file_names)
if root is None:
logger.critical("Cannot find a root directory in %s." %
self.import_source)
return False
self.import_dir = tempfile.mkdtemp()
archive.extractall(self.import_dir)
self.import_dir = os.path.join(self.import_dir, root)
if self.drop:
logger.info("Dropping and recreating the database.")
try:
metadata.drop_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
try:
metadata.create_all()
except sqlalchemy.exc.OperationalError as error:
logger.critical("Unable to access DB.\n%r" % error)
return False
if not self.no_files:
logger.info("Importing files.")
files_dir = os.path.join(self.import_dir, "files")
descr_dir = os.path.join(self.import_dir, "descriptions")
files = set(os.listdir(files_dir))
for _file in files:
if not self.safe_put_file(os.path.join(files_dir, _file),
os.path.join(descr_dir, _file)):
return False
if not self.only_files:
with SessionGen(commit=False) as session:
# Import the contest in JSON format.
logger.info("Importing the contest from JSON file.")
with open(os.path.join(self.import_dir,
"contest.json")) as fin:
contest = Contest.import_from_dict(json.load(fin))
session.add(contest)
# Check that no files were missing (only if files were
# imported).
if not self.no_files:
contest_files = contest.enumerate_files()
missing_files = contest_files.difference(files)
if len(missing_files) > 0:
logger.warning("Some files needed to the contest "
"are missing in the import directory.")
session.flush()
contest_id = contest.id
session.commit()
logger.info("Import finished (contest id: %s)." % contest_id)
logger.operation = ""
# If we extracted an archive, we remove it.
if self.import_dir != self.import_source:
shutil.rmtree(self.import_dir)
return True
