def test_print_progress(self): old_stdout = sys.stdout sys.stdout = StringIO() for cur in range(0, 3): print_progress(cur, 3, "text") self.assertEqual( "\r1/3 text\r2/3 text\r3/3 text\n", sys.stdout.getvalue(), "should print out all the stages of progress separated by carriage returns without line feed." ) sys.stdout = old_stdout # Reset redirect.
def convert_classes(self): """ Convert the eFiction classes table to Open Doors tags. :return: Open Doors tags with the original type "class" """ old_classes, current, total = self.sql.read_table_with_total( self.working_original, "classes") old_class_types, _, _ = self.sql.read_table_with_total( self.working_original, "classtypes") for old_class in old_classes: parent = [ ct['classtype_title'] for ct in old_class_types if ct['classtype_id'] == old_class['class_type'] ] new_tag = { 'id': old_class['class_id'], 'name': old_class['class_name'], 'parent': ", ".join(parent) if parent != [] else "" } query = f""" INSERT INTO tags (`original_tagid`, `original_tag`, `original_type`, `original_parent`) VALUES {new_tag['id'], new_tag['name'], 'class', new_tag['parent']}; """ self.sql.execute(self.working_open_doors, query) current = print_progress(current, total, "classes converted") return self.sql.execute_and_fetchall( self.working_open_doors, "SELECT * FROM tags WHERE `original_type` = 'class'")
def convert_categories(self): """ Convert the eFiction categories table to Open Doors tags. :return: Open Doors tags with the original type "categories" """ old_categories, current, total = self.sql.read_table_with_total( self.working_original, "categories") for old_category in old_categories: parent = [ cat['category'] for cat in old_categories if cat['catid'] == old_category['parentcatid'] ] new_tag = { 'id': old_category['catid'], 'parent': ", ".join(parent) if parent != [] else "", 'name': old_category['category'], 'description': old_category['description'] } query = f""" INSERT INTO tags (`original_tagid`, `original_tag`, `original_type`, `original_description`, `original_parent`) VALUES {new_tag['id'], new_tag['name'], 'category', new_tag['description'], new_tag['parent']}; """ self.sql.execute(self.working_open_doors, query) current = print_progress(current, total, "categories converted") return self.sql.execute_and_fetchall( self.working_open_doors, "SELECT * FROM tags WHERE `original_type` = 'category'")
def __load_chapter_text_into_db(self, chapter_paths: List[dict]): """ Load chapters text from the `stories` files into the chapters table. Uses Windows 1252 if UTF-8 fails. :param chapter_paths: List of chapter metadata including path, author id and chapter id :return: """ warnings = [] self.logger.info("...loading data from chapters table...") old_chapters, current, total = self.sql.read_table_with_total( self.working_original, "chapters") self.logger.info("...removing rows from existing chapters table...") self.sql.execute(self.working_open_doors, "TRUNCATE TABLE chapters;") self.logger.info("...loading text from chapter files...") for old_chapter in old_chapters: chapid = old_chapter['chapid'] chapter = [ chapter_path for chapter_path in chapter_paths if chapter_path['chap_id'] == str(chapid) ] if chapter: file = chapter[0]['path'] try: with open(file, 'r') as f: raw = f.read() except UnicodeDecodeError as err: warnings.append( f"Chapter with id {chapid} contains non-ASCII characters which are not valid " f"UTF-8. Trying Windows 1252...") with open(file, 'r', encoding='cp1252') as f: raw = f.read() text = normalize(raw) if old_chapter['endnotes']: text = text + f"\n\n\n<hr>\n{old_chapter['endnotes']}" query = """ INSERT INTO chapters (id, position, title, text, story_id, notes) VALUES (%s, %s, %s, %s, %s, %s); """ self.sql.execute( self.working_open_doors, query, (chapid, old_chapter['inorder'], old_chapter['title'], text, old_chapter['sid'], old_chapter['notes'])) current = print_progress(current, total, "chapters converted") # If there were any errors, display a warning for the user to check the affected chapters if warnings: self.logger.warning("\n".join(warnings)) self.logger.error( make_banner( '-', "There were warnings; check the affected chapters listed above to make sure curly quotes " "and accented characters are correctly displayed.")) return self.sql.execute_and_fetchall(self.working_open_doors, "SELECT * FROM chapters;")
def _convert_authors(self, old_authors=None): self.logger.info("Converting authors...") current = 0 total = len(old_authors) for old_author in old_authors: new_author = { 'id': old_author['uid'], 'name': old_author['penname'], 'email': self.generate_email(old_author['penname'], old_author['email'], self.config['Archive']['archive_name']) } query = f"INSERT INTO authors (`id`, `name`, `email`) " \ f"VALUES {new_author['id'], new_author['name'], new_author['email']}" self.sql.execute(self.working_open_doors, query) current = print_progress(current, total, "authors converted") return self.sql.read_table_to_dict(self.working_open_doors, "authors")
def _convert_characters(self, old_characters): current, total = (0, len(old_characters)) for old_character in old_characters: parent = [ct['original_tag'] for ct in self.categories if ct['original_tagid'] == old_character['catid']] new_tag = { 'id': old_character['charid'], 'name': old_character['charname'], 'parent': ", ".join(parent) if parent != [] else "" } query = f""" INSERT INTO tags (`original_tagid`, `original_tag`, `original_type`, `original_parent`) VALUES {new_tag['id'], new_tag['name'], 'character', new_tag['parent']}; """ self.sql.execute(self.working_open_doors, query) current = print_progress(current, total, "characters converted") return self.sql.execute_and_fetchall(self.working_open_doors, "SELECT * FROM tags WHERE `original_type` = 'character'")
def convert_stories(self, language_code): """ Convert eFiction stories to the Open Doors format. Note that we leave all the tag columns (rating, relationships, tags, categories etc) empty because they are all in the `tags` table and will be populated with AO3 tags when this archive is processed in the ODAP. :return: The Open Doors stories table as a dict. """ self.logger.info("Converting stories...") old_stories, current, total = self.sql.read_table_with_total( self.working_original, "stories") for old_story in old_stories: new_story = { 'id': old_story['sid'], 'title': key_find('title', old_story, '').strip(), 'summary': normalize(old_story['summary']), 'notes': key_find('storynotes', old_story, '').strip(), 'date': str(old_story['date']), 'updated': str(old_story['updated']), 'language_code': language_code } self.logger.debug( f"Converting story metadata for '{new_story['title']}'") query = f""" INSERT INTO stories (id, title, summary, notes, date, updated, language_code) VALUES {new_story['id'], new_story['title'], new_story['summary'], new_story['notes'], new_story['date'], new_story['updated'], new_story['language_code']}; """ self.sql.execute(self.working_open_doors, query) self.logger.debug(f" tags...") tags = self._convert_story_tags(old_story) self._convert_tags_join(new_story, tags) self.logger.debug(f" authors...") self._convert_author_join(new_story, old_story['uid']) # Find if there are any coauthors for the work coauthors = self.fetch_coauthors(new_story) for coauthor in coauthors: self._convert_author_join(new_story, coauthor) current = print_progress(current, total, "stories converted") return self.sql.execute_and_fetchall(self.working_open_doors, "SELECT * FROM stories")
def convert_stories(self): """ Convert eFiction stories to the Open Doors format. :return: The Open Doors stories table as a dict. """ self.logger.info("Converting stories...") old_stories, current, total = self.sql.read_table_with_total( self.working_original, "stories") for old_story in old_stories: new_story = { 'id': old_story['sid'], 'title': (old_story['title'] or '').strip(), 'summary': normalize(old_story['summary']), 'notes': (old_story['storynotes'] or '').strip(), 'date': str(old_story['date']), 'updated': str(old_story['updated']), } self.logger.debug( f"Converting story metadata for '{new_story['title']}'") query = f""" INSERT INTO stories (id, title, summary, notes, date, updated) VALUES {new_story['id'], new_story['title'], new_story['summary'], new_story['notes'], new_story['date'], new_story['updated']}; """ self.sql.execute(self.working_open_doors, query) self.logger.debug(f" tags...") tags = self._convert_story_tags(old_story) self._convert_tags_join(new_story, tags) self.logger.debug(f" authors...") self._convert_author_join(new_story, old_story['uid']) coauthors = [] if old_story[ 'coauthors'] is not None and old_story['coauthors'] != "": for authorid in old_story['coauthors'].split(","): coauthors.append(authorid.strip()) for coauthor in coauthors: self._convert_author_join(new_story, coauthor) current = print_progress(current, total, "stories converted") return self.sql.execute_and_fetchall(self.working_open_doors, "SELECT * FROM stories")
def convert_ratings(self): """ Convert the eFiction ratings table to Open Doors tags. :return: Open Doors tags with the original type "rating" """ old_ratings, current, total = self.sql.read_table_with_total(self.working_original, "ratings") for old_rating in old_ratings: new_rating = { 'id': old_rating['rid'], 'name': old_rating['rating'], 'description': old_rating['warningtext'] } query = f""" INSERT INTO tags (`original_tagid`, `original_tag`, `original_type`, `original_description`) VALUES {new_rating['id'], new_rating['name'], 'rating', new_rating['description']}; """ self.sql.execute(self.working_open_doors, query) current = print_progress(current, total, "ratings converted") return self.sql.execute_and_fetchall(self.working_open_doors, "SELECT * FROM tags WHERE `original_type` = 'rating';")