def main(): parser = argparse.ArgumentParser( description="Updater of CMS contest dumps.") parser.add_argument("-V", "--to-version", action="store", type=int, default=-1, help="Update to given version number") parser.add_argument( "path", action="store", type=utf8_decoder, help="location of the dump or of the 'contest.json' file") args = parser.parse_args() path = args.path to_version = args.to_version if to_version == -1: to_version = model_version if not os.path.exists(path): logger.critical("The given path doesn't exist") return archive = None if Archive.is_supported(path): archive = Archive(path) path = archive.unpack() file_names = os.listdir(path) if len(file_names) != 1: logger.critical("Cannot find a root directory in the " "archive.") archive.cleanup() return False path = os.path.join(path, file_names[0]) if not path.endswith("contest.json"): path = os.path.join(path, "contest.json") if not os.path.exists(path): logger.critical( "The given path doesn't contain a contest dump in a format " "CMS is able to understand.") return with io.open(path, 'rb') as fin: data = json.load(fin, encoding="utf-8") # If no "_version" field is found we assume it's a v1.0 # export (before the new dump format was introduced). dump_version = data.get("_version", 0) if dump_version == to_version: logger.info( "The dump you're trying to update is already stored using " "the target format (which is version %d).", dump_version) return elif dump_version > model_version: logger.critical( "The dump you're trying to update is stored using data model " "version %d, which is more recent than the one supported by " "this version of CMS (version %d). You probably need to " "update CMS to handle it.", dump_version, model_version) return elif to_version > model_version: logger.critical( "The target data model (version %d) you're trying to update " "to is too recent for this version of CMS (which supports up " "to version %d). You probably need to update CMS to handle " "it.", to_version, model_version) return elif dump_version > to_version: logger.critical( "Backward updating (from version %d to version %d) is not " "supported.", dump_version, to_version) return for version in range(dump_version, to_version): # Update from version to version+1 updater = __import__("cmscontrib.updaters.update_%d" % (version + 1), globals(), locals(), ["Updater"]).Updater(data) data = updater.run() data["_version"] = version + 1 assert data["_version"] == to_version with io.open(path, 'wb') as fout: json.dump(data, fout, encoding="utf-8", indent=4, sort_keys=True) if archive is not None: # Keep the old archive, just rename it shutil.move(archive.path, archive.path + ".bak") archive.repack(os.path.abspath(archive.path)) archive.cleanup()
def do_import(self): """Run the actual import code.""" logger.info("Starting import.") archive = None if Archive.is_supported(self.import_source): archive = Archive(self.import_source) self.import_dir = archive.unpack() file_names = os.listdir(self.import_dir) if len(file_names) != 1: logger.critical("Cannot find a root directory in %s.", self.import_source) archive.cleanup() return False self.import_dir = os.path.join(self.import_dir, file_names[0]) if self.drop: logger.info("Dropping and recreating the database.") try: if not (drop_db() and init_db()): logger.critical( "Unexpected error while dropping " "and recreating the database.", exc_info=True) return False except Exception: logger.critical("Unable to access DB.", exc_info=True) return False with SessionGen() as session: # Import the contest in JSON format. if self.load_model: logger.info("Importing the contest from a JSON file.") with io.open(os.path.join(self.import_dir, "contest.json"), "rb") as fin: # TODO - Throughout all the code we'll assume the # input is correct without actually doing any # validations. Thus, for example, we're not # checking that the decoded object is a dict... self.datas = json.load(fin) # If the dump has been exported using a data model # different than the current one (that is, a previous # one) we try to update it. # If no "_version" field is found we assume it's a v1.0 # export (before the new dump format was introduced). dump_version = self.datas.get("_version", 0) if dump_version < model_version: logger.warning( "The dump you're trying to import has been created " "by an old version of CMS (it declares data model " "version %d). It may take a while to adapt it to " "the current data model (which is version %d). You " "can use cmsDumpUpdater to update the on-disk dump " "and speed up future imports.", dump_version, model_version) elif dump_version > model_version: logger.critical( "The dump you're trying to import has been created " "by a version of CMS newer than this one (it " "declares data model version %d) and there is no " "way to adapt it to the current data model (which " "is version %d). You probably need to update CMS to " "handle it. It is impossible to proceed with the " "importation.", dump_version, model_version) return False else: logger.info("Importing dump with data model version %d.", dump_version) for version in range(dump_version, model_version): # Update from version to version+1 updater = __import__( "cmscontrib.updaters.update_%d" % (version + 1), globals(), locals(), ["Updater"]).Updater(self.datas) self.datas = updater.run() self.datas["_version"] = version + 1 assert self.datas["_version"] == model_version self.objs = dict() for id_, data in iteritems(self.datas): if not id_.startswith("_"): self.objs[id_] = self.import_object(data) for id_, data in iteritems(self.datas): if not id_.startswith("_"): self.add_relationships(data, self.objs[id_]) for k, v in list(iteritems(self.objs)): # Skip submissions if requested if self.skip_submissions and isinstance(v, Submission): del self.objs[k] # Skip user_tests if requested if self.skip_user_tests and isinstance(v, UserTest): del self.objs[k] # Skip print jobs if requested if self.skip_print_jobs and isinstance(v, PrintJob): del self.objs[k] # Skip generated data if requested if self.skip_generated and \ isinstance(v, (SubmissionResult, UserTestResult)): del self.objs[k] contest_id = list() contest_files = set() # We add explicitly only the top-level objects: # contests, and tasks and users not contained in any # contest. This will add on cascade all dependent # objects, and not add orphaned objects (like those # that depended on submissions or user tests that we # might have removed above). for id_ in self.datas["_objects"]: obj = self.objs[id_] session.add(obj) session.flush() if isinstance(obj, Contest): contest_id += [obj.id] contest_files |= enumerate_files( session, obj, skip_submissions=self.skip_submissions, skip_user_tests=self.skip_user_tests, skip_print_jobs=self.skip_print_jobs, skip_generated=self.skip_generated) session.commit() else: contest_id = None contest_files = None # Import files. if self.load_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") files = set(os.listdir(files_dir)) descr = set(os.listdir(descr_dir)) if not descr <= files: logger.warning("Some files do not have an associated " "description.") if not files <= descr: logger.warning("Some descriptions do not have an " "associated file.") if not (contest_files is None or files <= contest_files): # FIXME Check if it's because this is a light import # or because we're skipping submissions or user_tests logger.warning("The dump contains some files that are " "not needed by the contest.") if not (contest_files is None or contest_files <= files): # The reason for this could be that it was a light # export that's not being reimported as such. logger.warning("The contest needs some files that are " "not contained in the dump.") # Limit import to files we actually need. if contest_files is not None: files &= contest_files for digest in files: file_ = os.path.join(files_dir, digest) desc = os.path.join(descr_dir, digest) if not self.safe_put_file(file_, desc): logger.critical( "Unable to put file `%s' in the DB. " "Aborting. Please remove the contest " "from the database.", file_) # TODO: remove contest from the database. return False # Clean up, if an archive was used if archive is not None: archive.cleanup() if contest_id is not None: logger.info("Import finished (contest id: %s).", ", ".join("%d" % id_ for id_ in contest_id)) else: logger.info("Import finished.") return True
def main(): parser = argparse.ArgumentParser( description="Updater of CMS contest dumps.") parser.add_argument( "-V", "--to-version", action="store", type=int, default=-1, help="Update to given version number") parser.add_argument( "path", action="store", type=utf8_decoder, help="location of the dump or of the 'contest.json' file") args = parser.parse_args() path = args.path to_version = args.to_version if to_version == -1: to_version = model_version if not os.path.exists(path): logger.critical("The given path doesn't exist") return archive = None if Archive.is_supported(path): archive = Archive(path) path = archive.unpack() file_names = os.listdir(path) if len(file_names) != 1: logger.critical("Cannot find a root directory in the " "archive.") archive.cleanup() return False path = os.path.join(path, file_names[0]) if not path.endswith("contest.json"): path = os.path.join(path, "contest.json") if not os.path.exists(path): logger.critical( "The given path doesn't contain a contest dump in a format " "CMS is able to understand.") return with io.open(path, 'rb') as fin: data = json.load(fin, encoding="utf-8") # If no "_version" field is found we assume it's a v1.0 # export (before the new dump format was introduced). dump_version = data.get("_version", 0) if dump_version == to_version: logger.info( "The dump you're trying to update is already stored using " "the target format (which is version %d).", dump_version) return elif dump_version > model_version: logger.critical( "The dump you're trying to update is stored using data model " "version %d, which is more recent than the one supported by " "this version of CMS (version %d). You probably need to " "update CMS to handle it.", dump_version, model_version) return elif to_version > model_version: logger.critical( "The target data model (version %d) you're trying to update " "to is too recent for this version of CMS (which supports up " "to version %d). You probably need to update CMS to handle " "it.", to_version, model_version) return elif dump_version > to_version: logger.critical( "Backward updating (from version %d to version %d) is not " "supported.", dump_version, to_version) return for version in range(dump_version, to_version): # Update from version to version+1 updater = __import__( "cmscontrib.updaters.update_%d" % (version + 1), globals(), locals(), ["Updater"]).Updater(data) data = updater.run() data["_version"] = version + 1 assert data["_version"] == to_version with io.open(path, 'wb') as fout: json.dump(data, fout, encoding="utf-8", indent=4, sort_keys=True) if archive is not None: # Keep the old archive, just rename it shutil.move(archive.path, archive.path + ".bak") archive.repack(os.path.abspath(archive.path)) archive.cleanup()
def do_import(self): """Run the actual import code.""" logger.info("Starting import.") archive = None if Archive.is_supported(self.import_source): archive = Archive(self.import_source) self.import_dir = archive.unpack() file_names = os.listdir(self.import_dir) if len(file_names) != 1: logger.critical("Cannot find a root directory in %s.", self.import_source) archive.cleanup() return False self.import_dir = os.path.join(self.import_dir, file_names[0]) if self.drop: logger.info("Dropping and recreating the database.") try: if not (drop_db() and init_db()): logger.critical("Unexpected error while dropping " "and recreating the database.", exc_info=True) return False except Exception: logger.critical("Unable to access DB.", exc_info=True) return False with SessionGen() as session: # Import the contest in JSON format. if self.load_model: logger.info("Importing the contest from a JSON file.") with io.open(os.path.join(self.import_dir, "contest.json"), "rb") as fin: # TODO - Throughout all the code we'll assume the # input is correct without actually doing any # validations. Thus, for example, we're not # checking that the decoded object is a dict... self.datas = json.load(fin) # If the dump has been exported using a data model # different than the current one (that is, a previous # one) we try to update it. # If no "_version" field is found we assume it's a v1.0 # export (before the new dump format was introduced). dump_version = self.datas.get("_version", 0) if dump_version < model_version: logger.warning( "The dump you're trying to import has been created " "by an old version of CMS (it declares data model " "version %d). It may take a while to adapt it to " "the current data model (which is version %d). You " "can use cmsDumpUpdater to update the on-disk dump " "and speed up future imports.", dump_version, model_version) elif dump_version > model_version: logger.critical( "The dump you're trying to import has been created " "by a version of CMS newer than this one (it " "declares data model version %d) and there is no " "way to adapt it to the current data model (which " "is version %d). You probably need to update CMS to " "handle it. It is impossible to proceed with the " "importation.", dump_version, model_version) return False else: logger.info( "Importing dump with data model version %d.", dump_version) for version in range(dump_version, model_version): # Update from version to version+1 updater = __import__( "cmscontrib.updaters.update_%d" % (version + 1), globals(), locals(), ["Updater"]).Updater(self.datas) self.datas = updater.run() self.datas["_version"] = version + 1 assert self.datas["_version"] == model_version self.objs = dict() for id_, data in iteritems(self.datas): if not id_.startswith("_"): self.objs[id_] = self.import_object(data) for id_, data in iteritems(self.datas): if not id_.startswith("_"): self.add_relationships(data, self.objs[id_]) for k, v in list(iteritems(self.objs)): # Skip submissions if requested if self.skip_submissions and isinstance(v, Submission): del self.objs[k] # Skip user_tests if requested if self.skip_user_tests and isinstance(v, UserTest): del self.objs[k] # Skip print jobs if requested if self.skip_print_jobs and isinstance(v, PrintJob): del self.objs[k] # Skip generated data if requested if self.skip_generated and \ isinstance(v, (SubmissionResult, UserTestResult)): del self.objs[k] contest_id = list() contest_files = set() # We add explicitly only the top-level objects: # contests, and tasks and users not contained in any # contest. This will add on cascade all dependent # objects, and not add orphaned objects (like those # that depended on submissions or user tests that we # might have removed above). for id_ in self.datas["_objects"]: obj = self.objs[id_] session.add(obj) session.flush() if isinstance(obj, Contest): contest_id += [obj.id] contest_files |= enumerate_files( session, obj, skip_submissions=self.skip_submissions, skip_user_tests=self.skip_user_tests, skip_print_jobs=self.skip_print_jobs, skip_generated=self.skip_generated) session.commit() else: contest_id = None contest_files = None # Import files. if self.load_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") files = set(os.listdir(files_dir)) descr = set(os.listdir(descr_dir)) if not descr <= files: logger.warning("Some files do not have an associated " "description.") if not files <= descr: logger.warning("Some descriptions do not have an " "associated file.") if not (contest_files is None or files <= contest_files): # FIXME Check if it's because this is a light import # or because we're skipping submissions or user_tests logger.warning("The dump contains some files that are " "not needed by the contest.") if not (contest_files is None or contest_files <= files): # The reason for this could be that it was a light # export that's not being reimported as such. logger.warning("The contest needs some files that are " "not contained in the dump.") # Limit import to files we actually need. if contest_files is not None: files &= contest_files for digest in files: file_ = os.path.join(files_dir, digest) desc = os.path.join(descr_dir, digest) if not self.safe_put_file(file_, desc): logger.critical("Unable to put file `%s' in the DB. " "Aborting. Please remove the contest " "from the database.", file_) # TODO: remove contest from the database. return False # Clean up, if an archive was used if archive is not None: archive.cleanup() if contest_id is not None: logger.info("Import finished (contest id: %s).", ", ".join("%d" % id_ for id_ in contest_id)) else: logger.info("Import finished.") return True