sql.run_script_from_file('shared_python/create-open-doors-tables.sql',
database=args.output_database,
prefix=args.db_table_prefix + '_')
story_exclusion_filter = ''
# Filter out DNI stories - story_ids_to_remove must be comma-separated list of DNI ids
if os.path.exists(args.story_ids_to_remove):
with open(args.story_ids_to_remove, "rt") as f:
print "Removing {0} Do Not Import stories...".format(
sum(line.count(",") for line in f))
f.seek(0)
for line in f:
story_exclusion_filter = filter + '(' + line + ')'
# Export tables
stories_without_tags = final.original_table(table_names['stories'],
story_exclusion_filter)
print "Stories without tags: {0}".format(len(stories_without_tags))
chapters = final.original_table(table_names['chapters'], '')
bookmarks_without_tags = final.original_table(table_names['bookmarks'],
story_exclusion_filter)
# ----------------------
# AA and custom archives
# ----------------------
if args.archive_type == 'AA':
# STORIES
print "Copying stories to final table {0}.{1}_stories...".format(
args.output_database, args.db_table_prefix)
final_stories = []
for story in stories_without_tags:
# Add additional story processing here
def copy_to_temp_db(self, has_coauthors=None):
efiction_db = self.efiction_original_database
temp_db = self.args.temp_db_database
coauthors = {}
table_names = self.table_names()
if has_coauthors is None:
has_coauthor_table = raw_input(
"\nDoes this archive have a coauthors table? Y/N\n")
has_coauthors = True if str.lower(
has_coauthor_table) == 'y' else False
else:
has_coauthors = has_coauthors
if has_coauthors:
coauthors_dict = self.sql.execute_dict(
"SELECT * FROM fanfiction_coauthors")
for coauthor in coauthors_dict:
coauthors[coauthor['sid']] = coauthor['uid']
# Create Open Doors tables in temp db
self.sql.run_script_from_file(os.path.join(
os.path.dirname(__file__),
'../shared_python/create-open-doors-tables.sql'),
database=temp_db)
# Export data to Open Doors tables
final = FinalTables(self.args, self.sql.db, self.log)
stories_without_tags = final.original_table(table_names['stories'],
database_name=efiction_db)
self.log.info("Stories without tags in original eFiction: {0}".format(
len(stories_without_tags)))
chapters = final.original_table(table_names['chapters'],
database_name=efiction_db)
self.log.info("Chapters in original eFiction: {0}".format(
len(chapters)))
# STORIES
self.log.info(
"Copying stories to temporary table {0}.stories...".format(
temp_db))
final_stories = []
for story in stories_without_tags:
if coauthors is not None and coauthors.has_key(story['sid']):
story['coauthors'] = coauthors[story['sid']]
else:
story['coauthors'] = None
final_stories.append(self.story_to_final_without_tags(story))
final.insert_into_final('stories', final_stories, temp_db)
# AUTHORS
self.log.info("Copying authors from original eFiction source...")
final_authors = []
authors = final.original_table(
table_names['authors'],
database_name=self.efiction_original_database)
for author in authors:
final_author = self.author_to_final(author)
final_authors.append(final_author)
final.insert_into_final('authors', final_authors, temp_db)
# CHAPTERS
self.log.info("Copying chapters from original eFiction source...")
final_chapters = [
self.chapter_to_final(chapter) for chapter in chapters
]
final.insert_into_final('chapters', final_chapters, temp_db)
# TAGS
self.log.info("Copying tags from original eFiction source...")
final_tags = final.original_table(
'tags', database_name=self.efiction_original_database)
self.tags.create_tags_table(temp_db)
final.insert_into_final('tags', final_tags, temp_db)