def main():
parser = argparse.ArgumentParser(
description="Updater of CMS contest dumps.")
parser.add_argument("-V",
"--to-version",
action="store",
type=int,
default=-1,
help="Update to given version number")
parser.add_argument(
"path",
action="store",
type=utf8_decoder,
help="location of the dump or of the 'contest.json' file")
args = parser.parse_args()
path = args.path
to_version = args.to_version
if to_version == -1:
to_version = model_version
if not os.path.exists(path):
logger.critical("The given path doesn't exist")
return
archive = None
if Archive.is_supported(path):
archive = Archive(path)
path = archive.unpack()
file_names = os.listdir(path)
if len(file_names) != 1:
logger.critical("Cannot find a root directory in the " "archive.")
archive.cleanup()
return False
path = os.path.join(path, file_names[0])
if not path.endswith("contest.json"):
path = os.path.join(path, "contest.json")
if not os.path.exists(path):
logger.critical(
"The given path doesn't contain a contest dump in a format "
"CMS is able to understand.")
return
with io.open(path, 'rb') as fin:
data = json.load(fin, encoding="utf-8")
# If no "_version" field is found we assume it's a v1.0
# export (before the new dump format was introduced).
dump_version = data.get("_version", 0)
if dump_version == to_version:
logger.info(
"The dump you're trying to update is already stored using "
"the target format (which is version %d).", dump_version)
return
elif dump_version > model_version:
logger.critical(
"The dump you're trying to update is stored using data model "
"version %d, which is more recent than the one supported by "
"this version of CMS (version %d). You probably need to "
"update CMS to handle it.", dump_version, model_version)
return
elif to_version > model_version:
logger.critical(
"The target data model (version %d) you're trying to update "
"to is too recent for this version of CMS (which supports up "
"to version %d). You probably need to update CMS to handle "
"it.", to_version, model_version)
return
elif dump_version > to_version:
logger.critical(
"Backward updating (from version %d to version %d) is not "
"supported.", dump_version, to_version)
return
for version in range(dump_version, to_version):
# Update from version to version+1
updater = __import__("cmscontrib.updaters.update_%d" % (version + 1),
globals(), locals(), ["Updater"]).Updater(data)
data = updater.run()
data["_version"] = version + 1
assert data["_version"] == to_version
with io.open(path, 'wb') as fout:
json.dump(data, fout, encoding="utf-8", indent=4, sort_keys=True)
if archive is not None:
# Keep the old archive, just rename it
shutil.move(archive.path, archive.path + ".bak")
archive.repack(os.path.abspath(archive.path))
archive.cleanup()
def do_import(self):
"""Run the actual import code."""
logger.info("Starting import.")
archive = None
if Archive.is_supported(self.import_source):
archive = Archive(self.import_source)
self.import_dir = archive.unpack()
file_names = os.listdir(self.import_dir)
if len(file_names) != 1:
logger.critical("Cannot find a root directory in %s.",
self.import_source)
archive.cleanup()
return False
self.import_dir = os.path.join(self.import_dir, file_names[0])
if self.drop:
logger.info("Dropping and recreating the database.")
try:
if not (drop_db() and init_db()):
logger.critical(
"Unexpected error while dropping "
"and recreating the database.",
exc_info=True)
return False
except Exception:
logger.critical("Unable to access DB.", exc_info=True)
return False
with SessionGen() as session:
# Import the contest in JSON format.
if self.load_model:
logger.info("Importing the contest from a JSON file.")
with io.open(os.path.join(self.import_dir, "contest.json"),
"rb") as fin:
# TODO - Throughout all the code we'll assume the
# input is correct without actually doing any
# validations. Thus, for example, we're not
# checking that the decoded object is a dict...
self.datas = json.load(fin)
# If the dump has been exported using a data model
# different than the current one (that is, a previous
# one) we try to update it.
# If no "_version" field is found we assume it's a v1.0
# export (before the new dump format was introduced).
dump_version = self.datas.get("_version", 0)
if dump_version < model_version:
logger.warning(
"The dump you're trying to import has been created "
"by an old version of CMS (it declares data model "
"version %d). It may take a while to adapt it to "
"the current data model (which is version %d). You "
"can use cmsDumpUpdater to update the on-disk dump "
"and speed up future imports.", dump_version,
model_version)
elif dump_version > model_version:
logger.critical(
"The dump you're trying to import has been created "
"by a version of CMS newer than this one (it "
"declares data model version %d) and there is no "
"way to adapt it to the current data model (which "
"is version %d). You probably need to update CMS to "
"handle it. It is impossible to proceed with the "
"importation.", dump_version, model_version)
return False
else:
logger.info("Importing dump with data model version %d.",
dump_version)
for version in range(dump_version, model_version):
# Update from version to version+1
updater = __import__(
"cmscontrib.updaters.update_%d" % (version + 1),
globals(), locals(), ["Updater"]).Updater(self.datas)
self.datas = updater.run()
self.datas["_version"] = version + 1
assert self.datas["_version"] == model_version
self.objs = dict()
for id_, data in iteritems(self.datas):
if not id_.startswith("_"):
self.objs[id_] = self.import_object(data)
for id_, data in iteritems(self.datas):
if not id_.startswith("_"):
self.add_relationships(data, self.objs[id_])
for k, v in list(iteritems(self.objs)):
# Skip submissions if requested
if self.skip_submissions and isinstance(v, Submission):
del self.objs[k]
# Skip user_tests if requested
if self.skip_user_tests and isinstance(v, UserTest):
del self.objs[k]
# Skip print jobs if requested
if self.skip_print_jobs and isinstance(v, PrintJob):
del self.objs[k]
# Skip generated data if requested
if self.skip_generated and \
isinstance(v, (SubmissionResult, UserTestResult)):
del self.objs[k]
contest_id = list()
contest_files = set()
# We add explicitly only the top-level objects:
# contests, and tasks and users not contained in any
# contest. This will add on cascade all dependent
# objects, and not add orphaned objects (like those
# that depended on submissions or user tests that we
# might have removed above).
for id_ in self.datas["_objects"]:
obj = self.objs[id_]
session.add(obj)
session.flush()
if isinstance(obj, Contest):
contest_id += [obj.id]
contest_files |= enumerate_files(
session,
obj,
skip_submissions=self.skip_submissions,
skip_user_tests=self.skip_user_tests,
skip_print_jobs=self.skip_print_jobs,
skip_generated=self.skip_generated)
session.commit()
else:
contest_id = None
contest_files = None
# Import files.
if self.load_files:
logger.info("Importing files.")
files_dir = os.path.join(self.import_dir, "files")
descr_dir = os.path.join(self.import_dir, "descriptions")
files = set(os.listdir(files_dir))
descr = set(os.listdir(descr_dir))
if not descr <= files:
logger.warning("Some files do not have an associated "
"description.")
if not files <= descr:
logger.warning("Some descriptions do not have an "
"associated file.")
if not (contest_files is None or files <= contest_files):
# FIXME Check if it's because this is a light import
# or because we're skipping submissions or user_tests
logger.warning("The dump contains some files that are "
"not needed by the contest.")
if not (contest_files is None or contest_files <= files):
# The reason for this could be that it was a light
# export that's not being reimported as such.
logger.warning("The contest needs some files that are "
"not contained in the dump.")
# Limit import to files we actually need.
if contest_files is not None:
files &= contest_files
for digest in files:
file_ = os.path.join(files_dir, digest)
desc = os.path.join(descr_dir, digest)
if not self.safe_put_file(file_, desc):
logger.critical(
"Unable to put file `%s' in the DB. "
"Aborting. Please remove the contest "
"from the database.", file_)
# TODO: remove contest from the database.
return False
# Clean up, if an archive was used
if archive is not None:
archive.cleanup()
if contest_id is not None:
logger.info("Import finished (contest id: %s).",
", ".join("%d" % id_ for id_ in contest_id))
else:
logger.info("Import finished.")
return True
