def main():
parser = argparse.ArgumentParser()
parser.add_argument('source', help="Path to old HP database")
parser.add_argument('destination', help="Desired path new HPX database")
parser.add_argument('-r', '--rar', help="Path to unrar tool")
parser.add_argument('-p', '--process', type=int, default=3, help="Amount of processes allowed to spawn")
parser.add_argument('-a', '--skip-archive', action='store_true', help="Skip generating pages for galleries in archive files (it might take too long)")
args = parser.parse_args()
AMOUNT_OF_TASKS = args.process if args.process > 0 else 1
rarfile.UNRAR_TOOL = args.rar
src = args.source
dst = args.destination
print("Source:", src)
print("Destination:", dst)
if os.path.exists(dst):
print("Warning: destination file already exists, you might want to delete")
if args.skip_archive:
print("Warning: pages for galleries in archives will not be generated")
print("Connecting to Happypanda database..")
conn_src = sqlite3.connect(src)
conn_src.row_factory = sqlite3.Row
DBBase._DB_CONN = conn_src
ListDB.init_lists()
print("Fetching gallery lists..")
src_galleries = GalleryDB.get_all_gallery()
print("Fetching all galleries, chapters, tags and hashes..")
print("Fetched galleries count:", len(src_galleries))
print("Creating new Happypanda X database")
engine = db.create_engine(os.path.join("sqlite:///", dst))
db.Base.metadata.create_all(engine)
sess = db.scoped_session(db.sessionmaker())
sess.configure(bind=engine)
constants.db_session = sess
db.initEvents(sess)
s = sess()
db.init_defaults(s)
print("Converting to Happypanda X Gallery.. ")
gallery_mixmap = {}
dst_galleries = []
dst_profiles = []
en_lang = db.Language()
en_lang.name = "English"
dst_languages = {"english": en_lang}
dst_artists = {}
dst_gtype = {}
dst_status = {}
dst_namespace = {}
dst_tag = {}
dst_nstagmapping = {}
dst_collections = {}
dst_grouping = {}
dst_pages = {}
pages_to_send = []
pages_to_send2 = []
try:
pages_count = 0
unique_paths = []
for numb, g in enumerate(src_galleries):
galleries = []
gallery_ns = None
for ch in g.chapters:
path = g.path if ch.in_archive else ch.path
if not os.path.exists(path):
try:
print("\nSkipping '{}' because path doesn't exists.".format(ch.title))
except UnicodeError:
print("\nSkipping '{}' because path doesn't exists.".format(ch.title.encode(errors='ignore')))
continue
if not args.skip_archive:
if path.endswith(('.rar', 'cbr')) and not args.rar:
try:
print("\nSkipping '{}' because path to unrar tool has not been supplied.".format(ch.title))
except UnicodeError:
print("\nSkipping '{}' because path to unrar tool has not been supplied.".format(ch.title.encode(errors='ignore')))
continue
path_in_archive = ch.path
gallery = db.Gallery()
if ch.in_archive:
h = hash((g.path, ch.path))
if h in unique_paths:
continue
unique_paths.append(h)
if not args.skip_archive:
dst_pages[pages_count] = gallery
pages_to_send.append((pages_count, ch.in_archive, ch.path, path, g.path))
pages_count += 1
else:
h = hash((ch.path,))
if h in unique_paths:
continue
unique_paths.append(h)
dst_pages[pages_count] = gallery
pages_to_send2.append((pages_count, ch.in_archive, ch.path, path, g.path))
pages_count += 1
for col in copy.copy(gallery.collections):
if col.name in dst_collections:
gallery.collections.remove(col)
gallery.collections.append(dst_collections[col.name])
else:
dst_collections[col.name] = col
if gallery_ns:
gallery.grouping = gallery_ns
else:
gallery_ns = db.Grouping()
gallery_ns.name = ch.title if ch.title else g.title
gallery_ns = dst_grouping.get(gallery_ns.name, gallery_ns)
dst_grouping[gallery_ns.name] = gallery_ns
gallery_ns.galleries.append(gallery)
if g.status:
gstatus = db.Status()
gstatus.name
gstatus = dst_status.get(gstatus.name, gstatus)
gallery_ns.status = gstatus
dst_status[gstatus.name] = gstatus
gallery.number = ch.number
lang = g.language.lower() if g.language else None
if lang and not lang in dst_languages:
db_lang = db.Language()
db_lang.name = g.language
dst_languages[lang] = db_lang
else:
db_lang = dst_languages['english']
gallery.language = db_lang
title = db.Title()
title.name = ch.title if ch.title else g.title
title.language = db_lang
gallery.titles.clear()
gallery.titles.append(title)
if g.artist:
artist = db.Artist()
artist_name = db.AliasName()
artist_name.name = g.artist
artist_name.language = db_lang
d_artist = dst_artists.get(artist_name.name)
if not d_artist:
d_artist = artist
d_artist.names.append(artist_name)
gallery.artists.append(artist)
dst_artists[artist_name.name] = artist
gallery.info = g.info
if g.fav:
gallery.metatags.append(db.MetaTag.tags[db.MetaTag.names.favorite])
if g.view == 2:
gallery.metatags.append(db.MetaTag.tags[db.MetaTag.names.inbox])
if g.rating is not None:
gallery.rating = g.rating * 2
if g.type:
gtype = db.Category()
gtype.name = g.type
gtype = dst_gtype.get(gtype.name, gtype)
gallery.type = gtype
dst_gtype[gtype.name] = gtype
if g.link:
gurl = db.Url()
gurl.name = g.link
gallery.urls.append(gurl)
gallery.pub_date = g.pub_date
gallery.timestamp = g.date_added
gallery.last_read = g.last_read
gallery.times_read = g.times_read
galleries.append(gallery)
if not g.id in gallery_mixmap:
gallery_mixmap[g.id] = []
gallery_mixmap[g.id].append(gallery)
# tags
for ns in g.tags:
n = db.Namespace(name=constants.special_namespace if ns == 'default' else ns)
n = dst_namespace.get(ns, n)
dst_namespace[ns] = n
for tag in g.tags[ns]:
t = db.Tag(name=tag)
t = dst_tag.get(tag, t)
dst_tag[t.name] = t
nstagname = ns + tag
nstag = db.NamespaceTags(n, t)
nstag = dst_nstagmapping.get(nstagname, nstag)
dst_nstagmapping[nstagname] = nstag
for ch_g in galleries:
ch_g.tags.append(nstag)
dst_galleries.extend(galleries)
try:
print_progress(numb, len(src_galleries), "Progress:", bar_length=50)
except UnicodeEncodeError:
print("\nStill in progress... please wait...")
if not pages_to_send:
AMOUNT_OF_TASKS = 1
page_pool = []
for x in range(AMOUNT_OF_TASKS):
pipe1, pipe2 = Pipe(False)
p = Process(target=page_generate, args=(args.rar, pages_in, pipe2), daemon=True)
p.start()
page_pool.append(p)
threading.Thread(target=process_pipes, args=(pages_out, pipe1), daemon=True).start()
pages_to_send2 = list(split(pages_to_send2, AMOUNT_OF_TASKS))
pages_to_send = list(split(pages_to_send, AMOUNT_OF_TASKS))
if len(pages_to_send) < len(pages_to_send):
for n, x in enumerate(pages_to_send):
try:
x.extend(pages_to_send2[n])
except IndexError:
pass
pages_to_sendx = pages_to_send
else:
for n, x in enumerate(pages_to_send2):
try:
x.extend(pages_to_send[n])
except IndexError:
pass
pages_to_sendx = pages_to_send2
pages_map = {}
pages_finish = {}
pages_finished = []
for n, x in enumerate(pages_to_sendx):
pages_map[n] = x
pages_finish[n] = False
pages_in.put((n, x))
print("\nResolving gallery pages...")
unique_pages = []
current_p_count = 0
while current_p_count < pages_count:
items = pages_out.get()
# need to make sure process helps with remaining if it finishes its own
for dead_p in page_pool:
if not dead_p.is_alive():
dead_p.terminate()
if pages_finished:
pages_id = pages_finished.pop()
for p_id in pages_finish:
if not pages_finish[p_id]:
Process(target=page_generate, args=(args.rar, pages_in, pipe2), daemon=True).start()
pages_in.put((p_id, list(reversed(pages_map[p_id]))))
if isinstance(items, int):
pages_finished.append(items)
pages_finish[items] = True
continue
for item in items:
p_hash, g, pages = item
p_hash = hash(p_hash)
if p_hash in unique_pages:
continue
unique_pages.append(p_hash)
current_p_count += 1
for pp in pages:
p = db.Page()
p.name = pp[0]
p.path = pp[1]
p.number = pp[2]
p.in_archive = pp[3]
dst_pages[g].pages.append(p)
try:
print_progress(current_p_count, pages_count, "Progress:", bar_length=50)
except UnicodeEncodeError:
print("\nStill in progress... please wait...")
finally:
pass
print("\nCreating gallery lists")
dst_lists = []
for l in GALLERY_LISTS:
glist = db.GalleryFilter()
glist.name = l.name
glist.filter = l.filter
glist.enforce = l.enforce
glist.regex = l.regex
glist.l_case = l.case
glist.strict = l.strict
for g in l.galleries():
if g.id in gallery_mixmap:
glist.galleries.extend(gallery_mixmap[g.id])
dst_lists.append(glist)
print("Adding languages...")
s.add_all(dst_languages.values())
print("Adding artists...")
s.add_all(dst_artists.values())
print("Adding gallery types...")
s.add_all(dst_gtype.values())
print("Adding gallery status...")
s.add_all(dst_status.values())
print("Adding gallery namespaces...")
s.add_all(dst_namespace.values())
print("Adding gallery tags...")
s.add_all(dst_tag.values())
s.add_all(dst_nstagmapping.values())
print("Adding galleries...")
s.add_all(dst_galleries)
print("Adding gallery lists...")
s.add_all(dst_lists)
print("Committing...")
s.commit()
print("Done!")
pages_finish[items] = True
continue
for item in items:
p_hash, g, pages = item
p_hash = hash(p_hash)
if p_hash in unique_pages:
continue
unique_pages.append(p_hash)
current_p_count += 1
for pp in pages:
p = db.Page()
p.name = pp[0]
p.path = pp[1]
p.number = pp[2]
p.in_archive = pp[3]
dst_pages[g].pages.append(p)
try:
print_progress(current_p_count, pages_count, "Progress:", bar_length=50)
except UnicodeEncodeError:
print("\nStill in progress... please wait...")
finally:
pass
print("\nCreating gallery lists")
dst_lists = []
for l in GALLERY_LISTS:
glist = db.GalleryFilter()
glist.name = l.name
glist.filter = l.filter
glist.enforce = l.enforce
