def do_import(self): """Take care of creating the database structure, delegating the loading of the contest data and putting them on the database. """ logger.info("Creating database structure.") if self.drop: try: with SessionGen() as session: FSObject.delete_all(session) session.commit() metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False contest = Contest.import_from_dict(self.loader.import_contest(self.path)) logger.info("Creating contest on the database.") with SessionGen() as session: session.add(contest) logger.info("Analyzing database.") session.commit() contest_id = contest.id analyze_all_tables(session) logger.info("Import finished (new contest id: %s)." % contest_id) return True
def do_import(self): """Take care of creating the database structure, delegating the loading of the contest data and putting them on the database. """ logger.info("Creating database structure.") if self.drop: try: with SessionGen() as session: FSObject.delete_all(session) session.commit() metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False contest = Contest.import_from_dict( self.loader.import_contest(self.path)) logger.info("Creating contest on the database.") with SessionGen() as session: session.add(contest) logger.info("Analyzing database.") session.commit() contest_id = contest.id analyze_all_tables(session) logger.info("Import finished (new contest id: %s)." % contest_id) return True
def do_reimport(self): """Ask the loader to load the contest and actually merge the two. """ # Create the dict corresponding to the new contest. yaml_contest = self.loader.import_contest(self.path) yaml_users = dict(((x['username'], x) for x in yaml_contest['users'])) yaml_tasks = dict(((x['name'], x) for x in yaml_contest['tasks'])) with SessionGen(commit=False) as session: # Create the dict corresponding to the old contest, from # the database. contest = Contest.get_from_id(self.contest_id, session) cms_contest = contest.export_to_dict() cms_users = dict((x['username'], x) for x in cms_contest['users']) cms_tasks = dict((x['name'], x) for x in cms_contest['tasks']) # Delete the old contest from the database. session.delete(contest) session.flush() # Do the actual merge: first of all update all users of # the old contest with the corresponding ones from the new # contest; if some user is present in the old contest but # not in the new one we check if we have to fail or remove # it and, in the latter case, add it to a list users_to_remove = [] for user_num, user in enumerate(cms_contest['users']): try: user_submissions = \ cms_contest['users'][user_num]['submissions'] cms_contest['users'][user_num] = \ yaml_users[user['username']] cms_contest['users'][user_num]['submissions'] = \ user_submissions except KeyError: if self.force: logger.warning("User %s exists in old contest, but " "not in the new one" % user['username']) users_to_remove.append(user_num) session.delete(contest.users[user_num]) else: logger.error("User %s exists in old contest, but " "not in the new one" % user['username']) return False # Delete the users for user_num in users_to_remove: del cms_contest['users'][user_num] # The append the users in the new contest, not present in # the old one. for user in yaml_contest['users']: if user['username'] not in cms_users.keys(): cms_contest['users'].append(user) # The same for tasks: update old tasks. tasks_to_remove = [] for task_num, task in enumerate(cms_contest['tasks']): try: cms_contest['tasks'][task_num] = yaml_tasks[task['name']] except KeyError: if self.force: logger.warning("Task %s exists in old contest, but " "not in the new one" % task['name']) tasks_to_remove.append(task_num) session.delete(contest.tasks[task_num]) else: logger.error("Task %s exists in old contest, but " "not in the new one" % task['name']) return False # Delete the tasks for task_num in tasks_to_remove: del cms_contest['tasks'][task_num] # And add new tasks. for task in yaml_contest['tasks']: if task['name'] not in cms_tasks.keys(): cms_contest['tasks'].append(task) # Reimport the contest in the db, with the previous ID. contest = Contest.import_from_dict(cms_contest) contest.id = self.contest_id session.add(contest) session.flush() logger.info("Analyzing database.") analyze_all_tables(session) session.commit() logger.info("Reimport of contest %s finished." % self.contest_id) return True
def do_import(self): """Run the actual import code. """ logger.operation = "importing contest from %s" % self.import_source logger.info("Starting import.") if not os.path.isdir(self.import_source): if self.import_source.endswith(".zip"): archive = zipfile.ZipFile(self.import_source, "r") file_names = archive.infolist() self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) elif self.import_source.endswith(".tar.gz") \ or self.import_source.endswith(".tgz") \ or self.import_source.endswith(".tar.bz2") \ or self.import_source.endswith(".tbz2") \ or self.import_source.endswith(".tar"): archive = tarfile.open(name=self.import_source) file_names = archive.getnames() else: logger.critical("Unable to import from %s." % self.import_source) return False root = find_root_of_archive(file_names) if root is None: logger.critical("Cannot find a root directory in %s." % self.import_source) return False self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) self.import_dir = os.path.join(self.import_dir, root) if self.drop: logger.info("Dropping and recreating the database.") try: metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False logger.info("Reading JSON file...") with open(os.path.join(self.import_dir, "contest.json")) as fin: contest_json = json.load(fin) if self.no_submissions: for user in contest_json["users"]: user["submissions"] = [] user["user_tests"] = [] if not self.only_files: with SessionGen(commit=False) as session: # Import the contest in JSON format. logger.info("Importing the contest from JSON file.") contest = Contest.import_from_dict(contest_json) session.add(contest) session.flush() contest_id = contest.id contest_files = contest.enumerate_files() session.commit() if not self.no_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") for digest in contest_files: file_ = os.path.join(files_dir, digest) desc = os.path.join(descr_dir, digest) if not os.path.exists(file_) or not os.path.exists(desc): logger.error("Some files needed to the contest " "are missing in the import directory. " "The import will continue. Be aware.") if not self.safe_put_file(file_, desc): logger.critical("Unable to put file `%s' in the database. " "Aborting. Please remove the contest " "from the database." % file_) # TODO: remove contest from the database. return False logger.info("Import finished (contest id: %s)." % contest_id) logger.operation = "" # If we extracted an archive, we remove it. if self.import_dir != self.import_source: shutil.rmtree(self.import_dir) return True
def do_reimport(self): """Ask the loader to load the contest and actually merge the two. """ # Create the dict corresponding to the new contest. yaml_contest = self.loader.import_contest(self.path) yaml_users = dict(((x['username'], x) for x in yaml_contest['users'])) yaml_tasks = dict(((x['name'], x) for x in yaml_contest['tasks'])) with SessionGen(commit=False) as session: # Create the dict corresponding to the old contest, from # the database. contest = Contest.get_from_id(self.contest_id, session) cms_contest = contest.export_to_dict() cms_users = dict((x['username'], x) for x in cms_contest['users']) cms_tasks = dict((x['name'], x) for x in cms_contest['tasks']) # Delete the old contest from the database. session.delete(contest) session.flush() # Do the actual merge: first of all update all users of # the old contest with the corresponding ones from the new # contest; if some user is present in the old contest but # not in the new one we check if we have to fail or remove # it and, in the latter case, add it to a list users_to_remove = [] for user_num, user in enumerate(cms_contest['users']): if user['username'] in yaml_users: yaml_user = yaml_users[user['username']] yaml_user['submissions'] = user['submissions'] yaml_user['user_tests'] = user['user_tests'] yaml_user['questions'] = user['questions'] yaml_user['messages'] = user['messages'] cms_contest['users'][user_num] = yaml_user else: if self.force: logger.warning("User %s exists in old contest, but " "not in the new one" % user['username']) users_to_remove.append(user_num) # FIXME Do we need really to do this, given that # we already deleted the whole contest? session.delete(contest.users[user_num]) else: logger.error("User %s exists in old contest, but " "not in the new one" % user['username']) return False # Delete the users for user_num in users_to_remove: del cms_contest['users'][user_num] # The append the users in the new contest, not present in # the old one. for user in yaml_contest['users']: if user['username'] not in cms_users.keys(): cms_contest['users'].append(user) # The same for tasks: update old tasks. tasks_to_remove = [] for task_num, task in enumerate(cms_contest['tasks']): if task['name'] in yaml_tasks: yaml_task = yaml_tasks[task['name']] cms_contest['tasks'][task_num] = yaml_task else: if self.force: logger.warning("Task %s exists in old contest, but " "not in the new one" % task['name']) tasks_to_remove.append(task_num) # FIXME Do we need really to do this, given that # we already deleted the whole contest? session.delete(contest.tasks[task_num]) else: logger.error("Task %s exists in old contest, but " "not in the new one" % task['name']) return False # Delete the tasks for task_num in tasks_to_remove: del cms_contest['tasks'][task_num] # And add new tasks. for task in yaml_contest['tasks']: if task['name'] not in cms_tasks.keys(): cms_contest['tasks'].append(task) # Reimport the contest in the db, with the previous ID. contest = Contest.import_from_dict(cms_contest) contest.id = self.contest_id session.add(contest) session.flush() logger.info("Analyzing database.") analyze_all_tables(session) session.commit() logger.info("Reimport of contest %s finished." % self.contest_id) return True
def do_import(self): """Run the actual import code. """ logger.operation = "importing contest from %s" % self.import_source logger.info("Starting import.") if not os.path.isdir(self.import_source): if self.import_source.endswith(".zip"): archive = zipfile.ZipFile(self.import_source, "r") file_names = archive.infolist() self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) elif self.import_source.endswith(".tar.gz") \ or self.import_source.endswith(".tgz") \ or self.import_source.endswith(".tar.bz2") \ or self.import_source.endswith(".tbz2") \ or self.import_source.endswith(".tar"): archive = tarfile.open(name=self.import_source) file_names = archive.getnames() else: logger.critical("Unable to import from %s." % self.import_source) return False root = find_root_of_archive(file_names) if root is None: logger.critical("Cannot find a root directory in %s." % self.import_source) return False self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) self.import_dir = os.path.join(self.import_dir, root) if self.drop: logger.info("Dropping and recreating the database.") try: metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False if not self.no_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") files = set(os.listdir(files_dir)) for _file in files: if not self.safe_put_file(os.path.join(files_dir, _file), os.path.join(descr_dir, _file)): return False if not self.only_files: with SessionGen(commit=False) as session: # Import the contest in JSON format. logger.info("Importing the contest from JSON file.") with open(os.path.join(self.import_dir, "contest.json")) as fin: contest = Contest.import_from_dict(json.load(fin)) session.add(contest) # Check that no files were missing (only if files were # imported). if not self.no_files: contest_files = contest.enumerate_files() missing_files = contest_files.difference(files) if len(missing_files) > 0: logger.warning("Some files needed to the contest " "are missing in the import directory.") session.flush() contest_id = contest.id session.commit() logger.info("Import finished (contest id: %s)." % contest_id) logger.operation = "" # If we extracted an archive, we remove it. if self.import_dir != self.import_source: shutil.rmtree(self.import_dir) return True
def do_import(self): """Run the actual import code. """ logger.operation = "importing contest from %s" % self.import_source logger.info("Starting import.") if not os.path.isdir(self.import_source): if self.import_source.endswith(".zip"): archive = zipfile.ZipFile(self.import_source, "r") file_names = archive.infolist() self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) elif self.import_source.endswith(".tar.gz") \ or self.import_source.endswith(".tgz") \ or self.import_source.endswith(".tar.bz2") \ or self.import_source.endswith(".tbz2") \ or self.import_source.endswith(".tar"): archive = tarfile.open(name=self.import_source) file_names = archive.getnames() else: logger.critical("Unable to import from %s." % self.import_source) return False root = find_root_of_archive(file_names) if root is None: logger.critical("Cannot find a root directory in %s." % self.import_source) return False self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) self.import_dir = os.path.join(self.import_dir, root) if self.drop: logger.info("Dropping and recreating the database.") try: metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False logger.info("Reading JSON file...") with open(os.path.join(self.import_dir, "contest.json")) as fin: contest_json = json.load(fin) if self.no_submissions: for user in contest_json["users"]: user["submissions"] = [] user["user_tests"] = [] if not self.only_files: with SessionGen(commit=False) as session: # Import the contest in JSON format. logger.info("Importing the contest from JSON file.") contest = Contest.import_from_dict(contest_json) session.add(contest) session.flush() contest_id = contest.id contest_files = contest.enumerate_files() session.commit() if not self.no_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") for digest in contest_files: file_ = os.path.join(files_dir, digest) desc = os.path.join(descr_dir, digest) print open(desc).read() if not os.path.exists(file_) or not os.path.exists(desc): logger.error("Some files needed to the contest " "are missing in the import directory. " "The import will continue. Be aware.") if not self.safe_put_file(file_, desc): logger.critical("Unable to put file `%s' in the database. " "Aborting. Please remove the contest " "from the database." % file_) # TODO: remove contest from the database. return False logger.info("Import finished (contest id: %s)." % contest_id) logger.operation = "" # If we extracted an archive, we remove it. if self.import_dir != self.import_source: shutil.rmtree(self.import_dir) return True
def do_import(self): """Run the actual import code. """ logger.operation = "importing contest from %s" % self.import_source logger.info("Starting import.") if not os.path.isdir(self.import_source): if self.import_source.endswith(".zip"): archive = zipfile.ZipFile(self.import_source, "r") file_names = archive.infolist() self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) elif self.import_source.endswith(".tar.gz") \ or self.import_source.endswith(".tgz") \ or self.import_source.endswith(".tar.bz2") \ or self.import_source.endswith(".tbz2") \ or self.import_source.endswith(".tar"): archive = tarfile.open(name=self.import_source) file_names = archive.getnames() else: logger.critical("Unable to import from %s." % self.import_source) return False root = find_root_of_archive(file_names) if root is None: logger.critical("Cannot find a root directory in %s." % self.import_source) return False self.import_dir = tempfile.mkdtemp() archive.extractall(self.import_dir) self.import_dir = os.path.join(self.import_dir, root) if self.drop: logger.info("Dropping and recreating the database.") try: metadata.drop_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False try: metadata.create_all() except sqlalchemy.exc.OperationalError as error: logger.critical("Unable to access DB.\n%r" % error) return False if not self.no_files: logger.info("Importing files.") files_dir = os.path.join(self.import_dir, "files") descr_dir = os.path.join(self.import_dir, "descriptions") files = set(os.listdir(files_dir)) for _file in files: if not self.safe_put_file(os.path.join(files_dir, _file), os.path.join(descr_dir, _file)): return False if not self.only_files: with SessionGen(commit=False) as session: # Import the contest in JSON format. logger.info("Importing the contest from JSON file.") with open(os.path.join(self.import_dir, "contest.json")) as fin: contest = Contest.import_from_dict(json.load(fin)) session.add(contest) # Check that no files were missing (only if files were # imported). if not self.no_files: contest_files = contest.enumerate_files() missing_files = contest_files.difference(files) if len(missing_files) > 0: logger.warning("Some files needed to the contest " "are missing in the import directory.") session.flush() contest_id = contest.id session.commit() logger.info("Import finished (contest id: %s)." % contest_id) logger.operation = "" # If we extracted an archive, we remove it. if self.import_dir != self.import_source: shutil.rmtree(self.import_dir) return True