def deduplicateMangaFolders(): dirDictDict = nt.dirNameProxy.getDirDicts() keys = list(dirDictDict.keys()) keys.sort() pc = PathCleaner() pc.openDB() # dm = DedupManager() for offset in range(len(keys)): curDict = dirDictDict[keys[offset]] curKeys = curDict.keys() for curKey in curKeys: if not curKey: print("Invalid key!", curKey) continue for subKey in keys[offset+1:]: if curKey in dirDictDict[subKey]: print("Duplicate Directory for key '%s'" % curKey) print(" Preferred:", curDict[curKey]) print(" Duplicate:", dirDictDict[subKey][curKey]) fromDir = dirDictDict[subKey][curKey] toDir = curDict[curKey] items = os.listdir(fromDir) for item in items: fromPath = os.path.join(fromDir, item) toPath = os.path.join(toDir, item) loop = 2 while os.path.exists(toPath): pathBase, ext = os.path.splitext(toPath) print("Duplicate file!") toPath = "{start} ({loop}){ext}".format(start=pathBase, loop=loop, ext=ext) print("Moving: ", item) print(" From: ", fromPath) print(" To: ", toPath) pc.moveFile(fromPath, toPath) shutil.move(fromPath, toPath)
def deduplicateMangaFolders(): dirDictDict = nt.dirNameProxy.getDirDicts() keys = list(dirDictDict.keys()) keys.sort() pc = PathCleaner() pc.openDB() # dm = DedupManager() for offset in range(len(keys)): curDict = dirDictDict[keys[offset]] curKeys = curDict.keys() for curKey in curKeys: if not curKey: print("Invalid key!", curKey) continue for subKey in keys[offset + 1:]: if curKey in dirDictDict[subKey]: print("Duplicate Directory for key '%s'" % curKey) print(" Preferred:", curDict[curKey]) print(" Duplicate:", dirDictDict[subKey][curKey]) fromDir = dirDictDict[subKey][curKey] toDir = curDict[curKey] items = os.listdir(fromDir) for item in items: fromPath = os.path.join(fromDir, item) toPath = os.path.join(toDir, item) loop = 2 while os.path.exists(toPath): pathBase, ext = os.path.splitext(toPath) print("Duplicate file!") toPath = "{start} ({loop}){ext}".format( start=pathBase, loop=loop, ext=ext) print("Moving: ", item) print(" From: ", fromPath) print(" To: ", toPath) pc.moveFile(fromPath, toPath) shutil.move(fromPath, toPath)
def consolidateMangaFolders(dirPath, smartMode=True): idLut = nt.MtNamesMapWrapper("fsName->buId") pc = PathCleaner() pc.openDB() count = 0 print("Dir", dirPath) items = os.listdir(dirPath) items.sort() for item in items: item = os.path.join(dirPath, item) if os.path.isdir(item): fPath, dirName = os.path.split(item) lookup = nt.dirNameProxy[dirName] if lookup["fqPath"] != item: print() print() print("------------------------------------------------------") canonName = nt.getCanonicalMangaUpdatesName(dirName) print("Duplicate Directory '%s' - Canon = '%s'" % (dirName, canonName)) count += 1 mtId = idLut[nt.prepFilenameForMatching(dirName)] for num in mtId: print( " URL: https://www.mangaupdates.com/series.html?id=%s" % (num, )) fPath, dir2Name = os.path.split(lookup["fqPath"]) if not os.path.exists(item): print("'%s' has been removed. Skipping" % item) continue if not os.path.exists(lookup["fqPath"]): print("'%s' has been removed. Skipping" % lookup["fqPath"]) continue n1 = lv.distance(dirName, canonName) n2 = lv.distance(dir2Name, canonName) r1 = abs(nt.extractRatingToFloat(dirName)) r2 = abs(nt.extractRatingToFloat(dir2Name)) if "[complete]" in dirName.lower(): r1 += 0.1 if "[complete]" in dir2Name.lower(): r2 += 0.1 if "[wtf]" in dirName.lower(): r1 += 0.2 if "[wtf]" in dir2Name.lower(): r2 += 0.2 print(" 1: ", item) print(" 2: ", lookup["fqPath"]) print(" 1: ", dirName, ' ->', nt.getCanonicalMangaUpdatesName(dirName)) print(" 2: ", dir2Name, ' ->', nt.getCanonicalMangaUpdatesName(dir2Name)) print( " 1: ({num} items)(distance {dist})(rating {rat})".format( num=len(os.listdir(item)), dist=n1, rat=r1)) print( " 2: ({num} items)(distance {dist})(rating {rat})".format( num=len(os.listdir(lookup["fqPath"])), dist=n2, rat=r2)) mtId2 = idLut[nt.prepFilenameForMatching(dir2Name)] if mtId != mtId2: print("DISCORDANT ID NUMBERS - {num1}, {num2}!".format( num1=mtId, num2=mtId2)) for num in mtId2: print( " URL: https://www.mangaupdates.com/series.html?id=%s" % (num, )) continue if r1 > r2: doMove = "reverse" elif r2 > r1: doMove = "forward" else: doMove = '' if not doMove or not smartMode: doMove = query_response( "move files ('f' dir 1 -> dir 2. 'r' dir 1 <- dir 2. 'l' use levenshtein distance. 'n' do not move)?" ) if doMove == "forward": print("Forward move") fromDir = item toDir = lookup["fqPath"] elif doMove == "reverse": print("Reverse move") fromDir = lookup["fqPath"] toDir = item elif doMove == "levenshtein": print("Levenshtein distance chooser") # I'm using less then or equal, so situations where # both names are equadistant get aggregated anyways. if n1 <= n2: fromDir = lookup["fqPath"] toDir = item else: fromDir = item toDir = lookup["fqPath"] else: print("Skipping") continue print("moving from: '%s' " % fromDir) print(" to: '%s' " % toDir) items = os.listdir(fromDir) for item in items: fromPath = os.path.join(fromDir, item) toPath = os.path.join(toDir, item) loop = 2 while os.path.exists(toPath): pathBase, ext = os.path.splitext(toPath) print(" Duplicate file!") toPath = "{start} ({loop}){ext}".format(start=pathBase, loop=loop, ext=ext) print(" Moving: ", item) print(" From: ", fromPath) print(" To: ", toPath) pc.moveFile(fromPath, toPath) try: pc.moveFile(fromPath, toPath) except psycopg2.IntegrityError: print("Error moving item in dedup database") # pc.deletePath(toPath) shutil.move(fromPath, toPath) print("Deleting directory") os.rmdir(fromDir) print("total items", count)
def consolidateMangaFolders(dirPath, smartMode=True): idLut = nt.MtNamesMapWrapper("fsName->buId") pc = PathCleaner() pc.openDB() count = 0 print("Dir", dirPath) items = os.listdir(dirPath) items.sort() for item in items: item = os.path.join(dirPath, item) if os.path.isdir(item): fPath, dirName = os.path.split(item) lookup = nt.dirNameProxy[dirName] if lookup["fqPath"] != item: print() print() print("------------------------------------------------------") canonName = nt.getCanonicalMangaUpdatesName(dirName) print("Duplicate Directory '%s' - Canon = '%s'" % (dirName, canonName)) count += 1 mtId = idLut[nt.prepFilenameForMatching(dirName)] for num in mtId: print(" URL: https://www.mangaupdates.com/series.html?id=%s" % (num, )) fPath, dir2Name = os.path.split(lookup["fqPath"]) if not os.path.exists(item): print("'%s' has been removed. Skipping" % item) continue if not os.path.exists(lookup["fqPath"]): print("'%s' has been removed. Skipping" % lookup["fqPath"]) continue n1 = lv.distance(dirName, canonName) n2 = lv.distance(dir2Name, canonName) r1 = abs(nt.extractRatingToFloat(dirName)) r2 = abs(nt.extractRatingToFloat(dir2Name)) if "[complete]" in dirName.lower(): r1 += 0.1 if "[complete]" in dir2Name.lower(): r2 += 0.1 if "[wtf]" in dirName.lower(): r1 += 0.2 if "[wtf]" in dir2Name.lower(): r2 += 0.2 print(" 1: ", item) print(" 2: ", lookup["fqPath"]) print(" 1: ", dirName, ' ->', nt.getCanonicalMangaUpdatesName(dirName)) print(" 2: ", dir2Name, ' ->', nt.getCanonicalMangaUpdatesName(dir2Name)) print(" 1: ({num} items)(distance {dist})(rating {rat})".format(num=len(os.listdir(item)), dist=n1, rat=r1)) print(" 2: ({num} items)(distance {dist})(rating {rat})".format(num=len(os.listdir(lookup["fqPath"])), dist=n2, rat=r2)) mtId2 = idLut[nt.prepFilenameForMatching(dir2Name)] if mtId != mtId2: print("DISCORDANT ID NUMBERS - {num1}, {num2}!".format(num1=mtId, num2=mtId2)) for num in mtId2: print(" URL: https://www.mangaupdates.com/series.html?id=%s" % (num, )) continue if r1 > r2: doMove = "reverse" elif r2 > r1: doMove = "forward" else: doMove = '' if not doMove or not smartMode: doMove = query_response("move files ('f' dir 1 -> dir 2. 'r' dir 1 <- dir 2. 'l' use levenshtein distance. 'n' do not move)?") if doMove == "forward": print("Forward move") fromDir = item toDir = lookup["fqPath"] elif doMove == "reverse": print("Reverse move") fromDir = lookup["fqPath"] toDir = item elif doMove == "levenshtein": print("Levenshtein distance chooser") # I'm using less then or equal, so situations where # both names are equadistant get aggregated anyways. if n1 <= n2: fromDir = lookup["fqPath"] toDir = item else: fromDir = item toDir = lookup["fqPath"] else: print("Skipping") continue print("moving from: '%s' " % fromDir) print(" to: '%s' " % toDir) items = os.listdir(fromDir) for item in items: fromPath = os.path.join(fromDir, item) toPath = os.path.join(toDir, item) loop = 2 while os.path.exists(toPath): pathBase, ext = os.path.splitext(toPath) print(" Duplicate file!") toPath = "{start} ({loop}){ext}".format(start=pathBase, loop=loop, ext=ext) print(" Moving: ", item) print(" From: ", fromPath) print(" To: ", toPath) pc.moveFile(fromPath, toPath) try: pc.moveFile(fromPath, toPath) except psycopg2.IntegrityError: print("Error moving item in dedup database") # pc.deletePath(toPath) shutil.move(fromPath, toPath) print("Deleting directory") os.rmdir(fromDir) print("total items", count)