def crop(*args, **kwargs): psutil.Process(os.getpid()).nice( psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) subname = os.path.join(*args[1:4]) toppath = os.path.join( (args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) basepath = os.path.join(toppath, "Images", subname) while not os.path.isdir(basepath) or len( os.walk(basepath).__next__()[1]) == 0: time.sleep(0.4) folder = os.path.join(basepath, os.walk(basepath).__next__()[1][0]) datapath = os.path.join(basepath, "crop.txt") maxthreads = int(kwargs["cropthreads"]) if "cropthreads" in kwargs else ( int(kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count()) if not os.path.exists(datapath): #print("waiting for game") while not os.path.exists(datapath): time.sleep(1) print("crop {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="") files = [] with open(datapath, "r") as data: imgsize = int(data.readline().rstrip('\n')) for line in data: files.append(line) pool = mp.Pool(processes=maxthreads) m = mp.Manager() progressQueue = m.Queue() originalSize = len(files) doneSize = 0 while len(files) > 0: workers = pool.map_async( partial(work, imgsize=imgsize, folder=folder, progressQueue=progressQueue), files, 128) for _ in range(len(files)): if progressQueue.get(True): doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0] - 15 print("\rcrop {:5.1f}% [{}{}]".format( round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") workers.wait() files = [x for x in workers.get() if x] if len(files) > 0: time.sleep(10 if len(files) > 1000 else 1) print("\rcrop {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
def printErase(arg): try: tsiz = tsize()[0] print("\r{}{}\n".format(arg, " " * (tsiz*math.ceil(len(arg)/tsiz)-len(arg) - 1)), end="", flush=True) except: #raise pass
def crop(outFolder, timestamp, surface, daytime, basePath=None, args: Namespace = Namespace()): psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10) subname = Path(timestamp, surface, daytime) toppath = Path( basePath if basePath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps"), outFolder, ) imagePath = Path(toppath, "Images") datapath = Path(imagePath, subname, "crop.txt") maxthreads = args.cropthreads if args.cropthreads else args.maxthreads while not datapath.exists(): time.sleep(1) print(f"crop {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="") files = [] with datapath.open("r", encoding="utf-8") as data: assert data.readline().rstrip("\n") == "v2" for line in data: files.append(line) pool = mp.Pool(processes=maxthreads) m = mp.Manager() progressQueue = m.Queue() originalSize = len(files) doneSize = 0 try: while len(files) > 0: workers = pool.map_async( partial(work, folder=imagePath, progressQueue=progressQueue), files, 128, ) for _ in range(len(files)): if progressQueue.get(True): doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0] - 15 print(f"\rcrop {round(progress * 100, 1):5.1f}% [{'=' * int(progress * tsiz)}{' ' * (tsiz - int(progress * tsiz))}]",end="",) workers.wait() files = [x for x in workers.get() if x] if len(files) > 0: time.sleep(10 if len(files) > 1000 else 1) print(f"\rcrop {100:5.1f}% [{'=' * (tsize()[0]-15)}]") except KeyboardInterrupt: time.sleep(0.2) print(f"Keyboardinterrupt caught with {len(files)} files left.") if len(files) < 40: for line in files: print(line) raise
def image_index(directory): '''function that uses os.walk(directory) - to output an index of all images in directory and subdirectories of extensions: ("jpg", "JPG", "jpeg", "JPEG", "png", "PNG")''' from os import walk from shutil import get_terminal_size as tsize l = list(walk(directory)) width = tsize()[0] imgcount = 0 counter = 0 filenum = 0 while counter < len(l): for i in range(len(l[counter][2])): filenum += 1 if len(l[counter][2]) >= 1: imagelist = [ i for i in l[counter][2] if i.endswith(("jpg", "JPG", "jpeg", "JPEG", "png", "PNG")) ] if len(imagelist) > 0: print('\n') print("FOLDER:".center(width)) print(str(l[counter][0]).center(width)) print("#", counter) hr = print("-" * width) hr print("FILES: {}".format(len(l[counter][2])).center(width)) print("IMAGES: {}".format(len(imagelist)).center(width)) for x in imagelist: print("[!]\t{}".format(x)) imgcount += 1 counter += 1 else: pass counter += 1 else: print("\n{} subdirectories,\n{} files scanned\n{} images".format( len(l), filenum, imgcount))
def maine(): from shutil import get_terminal_size as tsize from os import listdir, getcwd import sys global sudo global yes print("Accio\n".center(tsize()[0])) print( 'Program that finds images (jpg, jpeg, png, gif, bmp) in a given directory \nand subdirectories, and optionally transfers them into another one.\n' ) print("Enter path/to/parent/directory to be scanned: \n") loc = str(input(">>> ").strip(" ")) wd = lambda: getcwd() try: tl = listdir(loc) assert len(tl) >= 1 image_index(loc) except FileNotFoundError: print("Not found") except AssertionError: print("Empty") yes = ['y', 'Y', 'yes', 'YES'] while 1: try: print("Directory to be extracted: ") origin = loc + "/" + str(input(">>> {}/".format(loc)).strip(" ")) print("Destination: ") destination = str(input(">>> ").strip(" ")) print("Run as superuser? (Better chance of success):") sudo = input(">>> ") move_contents(origin, destination) except KeyboardInterrupt: print("Good Bye!") sys.exit()
def ref(*args, **kwargs): psutil.Process(os.getpid()).nice( psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) toppath = os.path.join( (args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) datapath = os.path.join(toppath, "mapInfo.json") maxthreads = int(kwargs["refthreads"]) if "refthreads" in kwargs else (int( kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count()) pool = mp.Pool(processes=maxthreads) with open(datapath, "r") as f: data = json.load(f) if os.path.isfile(datapath[:-5] + ".out.json"): with open(datapath[:-5] + ".out.json", "r") as f: outdata = json.load(f) else: outdata = {} if len(args) > 1: for i, mapObj in enumerate(data["maps"]): if mapObj["path"] == args[1]: new = i break else: new = len(data["maps"]) - 1 newMap = data["maps"][new] allImageIndex = {} allDayImages = {} for daytime in ("day", "night"): newComparedSurfaces = [] compareList = [] keepList = [] firstRemoveList = [] cropList = {} didAnything = False if len(args) <= 3 or daytime == args[3]: for surfaceName, surface in newMap["surfaces"].items(): if (len(args) <= 2 or surfaceName == args[2]) and daytime in surface and str( surface[daytime]) == "true" and ( len(args) <= 3 or daytime == args[3]): didAnything = True z = surface["zoom"]["max"] dayImages = [] newComparedSurfaces.append((surfaceName, daytime)) oldMapsList = [] for old in range(new): if surfaceName in data["maps"][old]["surfaces"]: oldMapsList.append(old) for old in oldMapsList: with open( os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, "crop.txt"), "r") as f: next(f) for line in f: split = line.rstrip("\n").split(" ", 5) cropList[(surfaceName, daytime, str(z), int(split[0]), int(os.path.splitext( split[1])[0]))] = int( split[4], 16) with open( os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, "crop.txt"), "r") as f: next(f) for line in f: split = line.rstrip("\n").split(" ", 5) cropList[( surfaceName, daytime, str(z), int(split[0]), int(os.path.splitext(split[1])[0]) )] = int(split[4], 16) | cropList.get( (surfaceName, daytime, str(z), int(split[0]), int(os.path.splitext(split[1])[0])), 0) oldImages = {} for old in oldMapsList: if surfaceName in data["maps"][old][ "surfaces"] and daytime in surface and z == surface[ "zoom"]["max"]: if surfaceName not in allImageIndex: allImageIndex[surfaceName] = {} path = os.path.join(toppath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): oldImages[(x, y.replace( ext, outext))] = data["maps"][old]["path"] if daytime != "day": if not os.path.isfile( os.path.join(toppath, "Images", newMap["path"], surfaceName, "day", "ref.txt")): print( "WARNING: cannot find day surface to copy non-day surface from. running ref.py on night surfaces is not very accurate." ) else: if DEBUG: print( "found day surface, reuse results from ref.py from there" ) with open( os.path.join(toppath, "Images", newMap["path"], surfaceName, "day", "ref.txt"), "r") as f: for line in f: #if (line.rstrip("\n").split(" ", 2)[1] == "6"): print("YUP", line.rstrip("\n").split(" ", 2)[0]) dayImages.append( tuple(line.rstrip("\n").split(" ", 2))) allDayImages[surfaceName] = dayImages path = os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): #if (y == "6.png"): print("hoi", x) if (x, os.path.splitext(y)[0]) in dayImages or ( x, y.replace(ext, outext)) not in oldImages: keepList.append( (surfaceName, daytime, str(z), x, y)) elif (x, y.replace(ext, outext)) in oldImages: compareList.append( (oldImages[(x, y.replace(ext, outext))], surfaceName, daytime, str(z), x, y)) if not didAnything: continue if DEBUG: print("found %s new images" % len(keepList)) if len(compareList) > 0: if DEBUG: print("comparing %s existing images" % len(compareList)) treshold = .3 * Image.open( os.path.join(toppath, "Images", *compareList[0]).replace( ext, outext)).size[0]**2 #print(treshold) #compare(compareList[0], treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"])) m = mp.Manager() progressQueue = m.Queue() workers = pool.map_async( partial(compare, treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue), compareList, 128) doneSize = 0 print("ref {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="") for i in range(len(compareList)): progressQueue.get(True) doneSize += 1 progress = float(doneSize) / len(compareList) tsiz = tsize()[0] - 15 print("\rref {:5.1f}% [{}{}]".format( round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") workers.wait() resultList = workers.get() newList = [x[1] for x in [x for x in resultList if x[0]]] firstRemoveList += [ x[1] for x in [x for x in resultList if not x[0]] ] if DEBUG: print("found %s changed in %s images" % (len(newList), len(compareList))) keepList += newList print("\rref {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15))) if DEBUG: print("scanning %s chunks for neighbour cropping" % len(firstRemoveList)) resultList = pool.map( partial(neighbourScan, keepList=keepList, cropList=cropList), firstRemoveList, 64) neighbourList = [x[1] for x in [x for x in resultList if x[0]]] removeList = [x[1] for x in [x for x in resultList if not x[0]]] if DEBUG: print("keeping %s neighbouring images" % len(neighbourList)) if DEBUG: print("deleting %s, keeping %s of %s existing images" % (len(removeList), len(keepList) + len(neighbourList), len(keepList) + len(neighbourList) + len(removeList))) if DEBUG: print("removing identical images") for x in removeList: os.remove(os.path.join(toppath, "Images", newMap["path"], *x)) if DEBUG: print("creating render index") for surfaceName, daytime in newComparedSurfaces: z = surface["zoom"]["max"] with open( os.path.join(toppath, "Images", newMap["path"], surfaceName, daytime, "ref.txt"), "w") as f: for aList in (keepList, neighbourList): for coord in aList: if coord[0] == surfaceName and coord[ 1] == daytime and coord[2] == str(z): f.write("%s %s\n" % (coord[3], os.path.splitext(coord[4])[0])) if DEBUG: print("creating client index") for aList in (keepList, neighbourList): for coord in aList: x = int(coord[3]) y = int(os.path.splitext(coord[4])[0]) if coord[0] not in allImageIndex: allImageIndex[coord[0]] = {} if coord[1] not in allImageIndex[coord[0]]: allImageIndex[coord[0]][coord[1]] = {} if y not in allImageIndex[coord[0]][coord[1]]: allImageIndex[coord[0]][coord[1]][y] = [x] elif x not in allImageIndex[coord[0]][coord[1]][y]: allImageIndex[coord[0]][coord[1]][y].append(x) # compress and build string changed = False if "maps" not in outdata: outdata["maps"] = {} if str(new) not in outdata["maps"]: outdata["maps"][str(new)] = {"surfaces": {}} for surfaceName, daytimeImageIndex in allImageIndex.items(): indexList = [] daytime = "night" if "night" in daytimeImageIndex and data["maps"][ new]["surfaces"][surfaceName] and str( data["maps"][new]["surfaces"][surfaceName] ["night"]) == "true" else "day" surfaceImageIndex = daytimeImageIndex[daytime] for y, xList in surfaceImageIndex.items(): string = getBase64(y, False) isLastChangedImage = False isLastNightImage = False for x in range(min(xList), max(xList) + 2): isChangedImage = x in xList #does the image exist at all? isNightImage = daytime == "night" and ( str(x), str(y)) not in allDayImages[ surfaceName] #is this image only in night? if isLastChangedImage != isChangedImage or ( isChangedImage and isLastNightImage != isNightImage ): #differential encoding string += getBase64( x, isNightImage if isChangedImage else isLastNightImage) isLastChangedImage = isChangedImage isLastNightImage = isNightImage indexList.append(string) if surfaceName not in outdata["maps"][str(new)]["surfaces"]: outdata["maps"][str(new)]["surfaces"][surfaceName] = {} outdata["maps"][str( new)]["surfaces"][surfaceName]["chunks"] = '='.join(indexList) if len(indexList) > 0: changed = True if changed: if DEBUG: print("writing mapInfo.out.json") with open(datapath[:-5] + ".out.json", "w+") as f: json.dump(outdata, f) if DEBUG: print("deleting empty folders") for curdir, subdirs, files in os.walk(toppath, *args[1:4]): if len(subdirs) == 0 and len(files) == 0: os.rmdir(curdir)
def zoom(*args, **kwargs): psutil.Process(os.getpid()).nice( psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) needsThumbnail = ( str(args[5]).lower() != "false") if len(args) > 5 else True toppath = os.path.join( (args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) datapath = os.path.join(toppath, "mapInfo.json") basepath = os.path.join(toppath, "Images") maxthreads = int( kwargs["zoomthreads" if kwargs["zoomthreads"] else "maxthreads"]) #print(basepath) with open(datapath, "r") as f: data = json.load(f) for mapIndex, map in enumerate(data["maps"]): if len(args) <= 1 or map["path"] == args[1]: for surfaceName, surface in map["surfaces"].items(): if len(args) <= 2 or surfaceName == args[2]: maxzoom = surface["zoom"]["max"] minzoom = surface["zoom"]["min"] daytimes = [] try: if surface["day"]: daytimes.append("day") except KeyError: pass try: if surface["night"]: daytimes.append("night") except KeyError: pass for daytime in daytimes: if len(args) <= 3 or daytime == args[3]: if not os.path.isdir( os.path.join(toppath, "Images", str(map["path"]), surfaceName, daytime, str(maxzoom - 1))): print("zoom {:5.1f}% [{}]".format( 0, " " * (tsize()[0] - 15)), end="") generateThumbnail = needsThumbnail \ and mapIndex == len(data["maps"]) - 1 \ and surfaceName == ("nauvis" if "nauvis" in map["surfaces"] else sorted(map["surfaces"].keys())[0]) \ and daytime == daytimes[0] allBigChunks = {} minX = float("inf") maxX = float("-inf") minY = float("inf") maxY = float("-inf") imageSize = None for xStr in os.listdir( os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(maxzoom))): x = int(xStr) minX = min(minX, x) maxX = max(maxX, x) for yStr in os.listdir( os.path.join( basepath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr)): if imageSize is None: imageSize = Image.open( os.path.join( basepath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr, yStr), mode='r').size[0] y = int(yStr.split('.', 2)[0]) minY = min(minY, y) maxY = max(maxY, y) allBigChunks[( x >> maxzoom - minzoom, y >> maxzoom - minzoom)] = True pathList = [] for otherMapIndex in range(mapIndex, -1, -1): pathList.append( str(data["maps"][otherMapIndex] ["path"])) threadsplit = 0 while 4**threadsplit * len( allBigChunks) < maxthreads: threadsplit = threadsplit + 1 threadsplit = min( max(maxzoom - minzoom - 3, 0), threadsplit + 3) allChunks = [] for pos in list(allBigChunks): for i in range(2**threadsplit): for j in range(2**threadsplit): allChunks.append( (pos[0] * (2**threadsplit) + i, pos[1] * (2**threadsplit) + j)) threads = min(len(allChunks), maxthreads) processes = [] originalSize = len(allChunks) # print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList))) # print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks)))) counter = mp.Value('i', originalSize) resultQueue = mp.Queue() for _ in range(0, threads): p = mp.Process( target=thread, args=(basepath, pathList, surfaceName, daytime, imageSize, maxzoom, minzoom + threadsplit, minzoom, allChunks, counter, resultQueue, generateThumbnail)) p.start() processes.append(p) doneSize = 0 for _ in range(originalSize): resultQueue.get(True) doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0] - 15 print("\rzoom {:5.1f}% [{}{}]".format( round(progress * 98, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") for p in processes: p.join() if threadsplit > 0: #print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks)))) processes = [] i = len(allBigChunks) - 1 for chunk in list(allBigChunks): p = mp.Process( target=work, args=(basepath, pathList, surfaceName, daytime, imageSize, minzoom + threadsplit, minzoom, minzoom, chunk, generateThumbnail)) i = i - 1 p.start() processes.append(p) for p in processes: p.join() if generateThumbnail: printErase("generating thumbnail") minzoompath = os.path.join( basepath, str(map["path"]), surfaceName, daytime, str(minzoom)) thumbnail = Image.new( 'RGB', ((maxX - minX + 1) * imageSize >> maxzoom - minzoom, (maxY - minY + 1) * imageSize >> maxzoom - minzoom), BACKGROUNDCOLOR) bigMinX = minX >> maxzoom - minzoom bigMinY = minY >> maxzoom - minzoom xOffset = ( (bigMinX * imageSize << maxzoom - minzoom) - minX * imageSize) >> maxzoom - minzoom yOffset = ( (bigMinY * imageSize << maxzoom - minzoom) - minY * imageSize) >> maxzoom - minzoom for chunk in list(allBigChunks): path = os.path.join( minzoompath, str(chunk[0]), str(chunk[1]) + EXT) thumbnail.paste(box=( xOffset + (chunk[0] - bigMinX) * imageSize, yOffset + (chunk[1] - bigMinY) * imageSize), im=Image.open( path, mode='r'). convert("RGB").resize( (imageSize, imageSize), Image.ANTIALIAS)) if OUTEXT != EXT: os.remove(path) thumbnail.save( os.path.join( basepath, "thumbnail" + THUMBNAILEXT)) print("\rzoom {:5.1f}% [{}]".format( 100, "=" * (tsize()[0] - 15)))
def crop(*args, **kwargs): psutil.Process(os.getpid()).nice( psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) subname = os.path.join(*args[1:4]) toppath = os.path.join( (args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) basepath = os.path.join(toppath, "Images") datapath = os.path.join(basepath, subname, "crop.txt") maxthreads = int( kwargs["cropthreads" if kwargs["cropthreads"] else "maxthreads"]) if not os.path.exists(datapath): #print("waiting for game") while not os.path.exists(datapath): time.sleep(1) print("crop {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="") files = [] with open(datapath, "r") as data: assert (data.readline().rstrip('\n') == "v2") for line in data: files.append(line) pool = mp.Pool(processes=maxthreads) m = mp.Manager() progressQueue = m.Queue() originalSize = len(files) doneSize = 0 try: while len(files) > 0: workers = pool.map_async( partial(work, folder=basepath, progressQueue=progressQueue), files, 128) for _ in range(len(files)): if progressQueue.get(True): doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0] - 15 print("\rcrop {:5.1f}% [{}{}]".format( round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") workers.wait() files = [x for x in workers.get() if x] if len(files) > 0: time.sleep(10 if len(files) > 1000 else 1) print("\rcrop {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15))) except KeyboardInterrupt: time.sleep(0.2) print(f"Keyboardinterrupt caught with {len(files)} files left.") if len(files) < 40: for line in files: print(line) raise
def zoom( outFolder: Path, timestamp: str = None, surfaceReference: str = None, daytimeReference: str = None, basepath: Path = None, needsThumbnail: bool = True, args: Namespace = Namespace(), ): psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10) workFolder = basepath if basepath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps") topPath = Path(workFolder, outFolder) dataPath = Path(topPath, "mapInfo.json") imagePath = Path(topPath, "Images") maxthreads = args.zoomthreads if args.zoomthreads else args.maxthreads with dataPath.open("r", encoding="utf-8") as f: data = json.load(f) for mapIndex, map in enumerate(data["maps"]): if timestamp is None or map["path"] == timestamp: for surfaceName, surface in map["surfaces"].items(): if surfaceReference is None or surfaceName == surfaceReference: maxzoom = surface["zoom"]["max"] minzoom = surface["zoom"]["min"] daytimes = [] if "day" in surface: daytimes.append("day") if "night" in surface: daytimes.append("night") for daytime in daytimes: if daytimeReference is None or daytime == daytimeReference: if not Path(topPath, "Images", str(map["path"]), surfaceName, daytime, str(maxzoom - 1)).is_dir(): print(f"zoom {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="") generateThumbnail = ( needsThumbnail and mapIndex == len(data["maps"]) - 1 and surfaceName == ( "nauvis" if "nauvis" in map["surfaces"] else sorted(map["surfaces"].keys())[0] ) and daytime == daytimes[0] ) allBigChunks = {} minX = float("inf") maxX = float("-inf") minY = float("inf") maxY = float("-inf") imageSize: int = None for xStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom)).iterdir(): x = int(xStr.name) minX = min(minX, x) maxX = max(maxX, x) for yStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr).iterdir(): if imageSize is None: imageSize = Image.open(Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr, yStr), mode="r").size[0] y = int(yStr.stem) minY = min(minY, y) maxY = max(maxY, y) allBigChunks[ ( x >> maxzoom-minzoom, y >> maxzoom-minzoom, ) ] = True if len(allBigChunks) <= 0: continue pathList = [] for otherMapIndex in range(mapIndex, -1, -1): pathList.append(str(data["maps"][otherMapIndex]["path"])) threadsplit = 0 while 4**threadsplit * len(allBigChunks) < maxthreads: threadsplit = threadsplit + 1 threadsplit = min(max(maxzoom - minzoom - 3, 0), threadsplit + 3) allChunks = [] for pos in list(allBigChunks): for i in range(2**threadsplit): for j in range(2**threadsplit): allChunks.append( ( pos[0] * (2**threadsplit) + i, pos[1] * (2**threadsplit) + j, ) ) threads = min(len(allChunks), maxthreads) processes = [] originalSize = len(allChunks) # print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList))) # print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks)))) counter = mp.Value("i", originalSize) resultQueue = mp.Queue() for _ in range(0, threads): p = mp.Process( target=thread, args=( imagePath, pathList, surfaceName, daytime, imageSize, maxzoom, minzoom + threadsplit, minzoom, allChunks, counter, resultQueue, generateThumbnail, ), ) p.start() processes.append(p) doneSize = 0 for _ in range(originalSize): resultQueue.get(True) doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0] - 15 print( "\rzoom {:5.1f}% [{}{}]".format( round(progress * 98, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz)), ), end="", ) for p in processes: p.join() if threadsplit > 0: # print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks)))) processes = [] i = len(allBigChunks) - 1 for chunk in list(allBigChunks): p = mp.Process( target=work, args=( imagePath, pathList, surfaceName, daytime, imageSize, minzoom + threadsplit, minzoom, minzoom, chunk, generateThumbnail, ), ) i = i - 1 p.start() processes.append(p) for p in processes: p.join() if generateThumbnail: printErase("generating thumbnail") minzoompath = Path( imagePath, str(map["path"]), surfaceName, daytime, str(minzoom), ) if imageSize is None: raise Exception("Missing imageSize for thumbnail generation") thumbnail = Image.new( "RGB", ( (maxX - minX + 1) * imageSize >> maxzoom-minzoom, (maxY - minY + 1) * imageSize >> maxzoom-minzoom, ), BACKGROUNDCOLOR, ) bigMinX = minX >> maxzoom-minzoom bigMinY = minY >> maxzoom-minzoom xOffset = ((bigMinX * imageSize << maxzoom-minzoom) - minX * imageSize) >> maxzoom-minzoom yOffset = ((bigMinY * imageSize << maxzoom-minzoom) - minY * imageSize) >> maxzoom-minzoom for chunk in list(allBigChunks): path = Path(minzoompath, str(chunk[0]), str(chunk[1])).with_suffix(EXT) thumbnail.paste( box=( xOffset + (chunk[0] - bigMinX) * imageSize, yOffset + (chunk[1] - bigMinY) * imageSize, ), im=Image.open(path, mode="r") .convert("RGB") .resize((imageSize, imageSize), Image.ANTIALIAS), ) if OUTEXT != EXT: path.unlink() thumbnail.save(Path(imagePath, "thumbnail" + THUMBNAILEXT)) print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
def ref( outFolder: Path, timestamp: str = None, surfaceReference: str = None, daytimeReference: str = None, basepath: Path = None, args: Namespace = Namespace(), ): psutil.Process(os.getpid()).nice( psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) workFolder = basepath if basepath else Path( __file__, "..", "..", "..", "script-output", "FactorioMaps").resolve() topPath = Path(workFolder, outFolder) dataPath = Path(topPath, "mapInfo.json") maxthreads = args.refthreads if args.refthreads else args.maxthreads pool = mp.Pool(processes=maxthreads) with open(dataPath, "r", encoding="utf-8") as f: data = json.load(f) outFile = Path(topPath, "mapInfo.out.json") if outFile.exists(): with outFile.open("r", encoding="utf-8") as mapInfoOutFile: outdata = json.load(mapInfoOutFile) else: outdata = {} if timestamp: for i, mapObj in enumerate(data["maps"]): if mapObj["path"] == timestamp: new = i break else: new = len(data["maps"]) - 1 changed = False if "maps" not in outdata: outdata["maps"] = {} if str(new) not in outdata["maps"]: outdata["maps"][str(new)] = {"surfaces": {}} newMap = data["maps"][new] allImageIndex = {} allDayImages = {} for daytime in ("day", "night"): newComparedSurfaces = [] compareList = [] keepList = [] firstRemoveList = [] cropList = {} didAnything = False if daytime is None or daytime == daytimeReference: for surfaceName, surface in newMap["surfaces"].items(): if (surfaceReference is None or surfaceName == surfaceReference) and daytime in surface and str( surface[daytime]) and (daytime is None or daytime == daytimeReference): didAnything = True z = surface["zoom"]["max"] dayImages = [] newComparedSurfaces.append((surfaceName, daytime)) oldMapsList = [] for old in range(new): if surfaceName in data["maps"][old]["surfaces"]: oldMapsList.append(old) def readCropList(path, combinePrevious): with open(path, "r", encoding="utf-8") as f: version = 2 if f.readline().rstrip( '\n') == "v2" else 1 for line in f: if version == 1: split = line.rstrip("\n").split(" ", 5) key = (surfaceName, daytime, str(z), int(split[0]), int(os.path.splitext(split[1])[0])) value = split[4] else: split = line.rstrip("\n").split(" ", 5) pathSplit = split[5].split("/", 5) if pathSplit[3] != str(z): continue #(surfaceName, daytime, z, str(x+1), str(y+1) + ext) key = (surfaceName, daytime, str(z), int(pathSplit[4]), int( os.path.splitext( pathSplit[5])[0])) value = split[2] cropList[key] = int(value, 16) | cropList.get( key, 0) if combinePrevious else int( value, 16) for old in oldMapsList: readCropList( os.path.join(topPath, "Images", data["maps"][old]["path"], surfaceName, daytime, "crop.txt"), False) readCropList( os.path.join(topPath, "Images", newMap["path"], surfaceName, daytime, "crop.txt"), True) oldImages = {} for old in oldMapsList: if surfaceName in data["maps"][old][ "surfaces"] and daytime in surface and z == surface[ "zoom"]["max"]: if surfaceName not in allImageIndex: allImageIndex[surfaceName] = {} path = os.path.join(topPath, "Images", data["maps"][old]["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): oldImages[(x, y.replace( ext, outext))] = data["maps"][old]["path"] if daytime != "day": if not os.path.isfile( os.path.join(topPath, "Images", newMap["path"], surfaceName, "day", "ref.txt")): print( "WARNING: cannot find day surface to copy non-day surface from. running ref.py on night surfaces is not very accurate." ) else: if args.verbose: print( "found day surface, reuse results from ref.py from there" ) with Path(topPath, "Images", newMap["path"], surfaceName, "day", "ref.txt").open("r", encoding="utf-8") as f: for line in f: dayImages.append( tuple(line.rstrip("\n").split(" ", 2))) allDayImages[surfaceName] = dayImages path = os.path.join(topPath, "Images", newMap["path"], surfaceName, daytime, str(z)) for x in os.listdir(path): for y in os.listdir(os.path.join(path, x)): if (x, os.path.splitext(y)[0]) in dayImages or ( x, y.replace(ext, outext)) not in oldImages: keepList.append( (surfaceName, daytime, str(z), x, y)) elif (x, y.replace(ext, outext)) in oldImages: compareList.append( (oldImages[(x, y.replace(ext, outext))], surfaceName, daytime, str(z), x, y)) if not didAnything: continue if args.verbose: print("found %s new images" % len(keepList)) if len(compareList) > 0: if args.verbose: print("comparing %s existing images" % len(compareList)) m = mp.Manager() progressQueue = m.Queue() #compare(compareList[0], treshold=treshold, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue) workers = pool.map_async( partial(compare, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue), compareList, 128) doneSize = 0 print("ref {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="") for i in range(len(compareList)): progressQueue.get(True) doneSize += 1 progress = float(doneSize) / len(compareList) tsiz = tsize()[0] - 15 print("\rref {:5.1f}% [{}{}]".format( round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") workers.wait() resultList = workers.get() newList = [x[1] for x in [x for x in resultList if x[0]]] firstRemoveList += [ x[1] for x in [x for x in resultList if not x[0]] ] if args.verbose: print("found %s changed in %s images" % (len(newList), len(compareList))) keepList += newList print("\rref {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15))) if args.verbose: print("scanning %s chunks for neighbour cropping" % len(firstRemoveList)) resultList = pool.map( partial(neighbourScan, keepList=keepList, cropList=cropList), firstRemoveList, 64) neighbourList = [x[1] for x in [x for x in resultList if x[0]]] removeList = [x[1] for x in [x for x in resultList if not x[0]]] if args.verbose: print("keeping %s neighbouring images" % len(neighbourList)) if args.verbose: print("deleting %s, keeping %s of %s existing images" % (len(removeList), len(keepList) + len(neighbourList), len(keepList) + len(neighbourList) + len(removeList))) if args.verbose: print("removing identical images") for x in removeList: os.remove(os.path.join(topPath, "Images", newMap["path"], *x)) if args.verbose: print("creating render index") for surfaceName, daytime in newComparedSurfaces: z = surface["zoom"]["max"] with Path(topPath, "Images", newMap["path"], surfaceName, daytime, "ref.txt").open("w", encoding="utf-8") as f: for aList in (keepList, neighbourList): for coord in aList: if coord[0] == surfaceName and coord[ 1] == daytime and coord[2] == str(z): f.write("%s %s\n" % (coord[3], os.path.splitext(coord[4])[0])) if args.verbose: print("creating client index") for aList in (keepList, neighbourList): for coord in aList: x = int(coord[3]) y = int(os.path.splitext(coord[4])[0]) if coord[0] not in allImageIndex: allImageIndex[coord[0]] = {} if coord[1] not in allImageIndex[coord[0]]: allImageIndex[coord[0]][coord[1]] = {} if y not in allImageIndex[coord[0]][coord[1]]: allImageIndex[coord[0]][coord[1]][y] = [x] elif x not in allImageIndex[coord[0]][coord[1]][y]: allImageIndex[coord[0]][coord[1]][y].append(x) if args.verbose: print("comparing renderboxes") if "renderboxesCompared" not in outdata["maps"][str(new)]: changed = True outdata["maps"][str(new)]["renderboxesCompared"] = True compareList = {} totalCount = 0 for surfaceName, surface in newMap["surfaces"].items(): linksByPath = {} for linkIndex, link in enumerate(surface["links"]): if surfaceName not in outdata["maps"][str( new)]["surfaces"]: outdata["maps"][str(new)]["surfaces"][surfaceName] = { "links": [] } outdata["maps"][str( new)]["surfaces"][surfaceName]["links"].append( {"path": newMap["path"]}) for daytime in ("day", "night"): if link["type"] == "link_renderbox_area" and ( link["daynight"] or daytime == "day"): path = os.path.join( link["toSurface"], daytime if link["daynight"] else "day", "renderboxes", str(surface["zoom"]["max"]), link["filename"]) if path not in linksByPath: linksByPath[path] = [(surfaceName, linkIndex)] else: linksByPath[path].append( (surfaceName, linkIndex)) totalCount += 1 for old in range(new - 1, -1, -1): if surfaceName in data["maps"][old]["surfaces"]: for linkIndex, link in enumerate( data["maps"][old]["surfaces"][surfaceName] ["links"]): for daytime in ("day", "night"): if link["type"] == "link_renderbox_area" and ( link["daynight"] or daytime == "day"): path = os.path.join( link["toSurface"], daytime if link["daynight"] else "day", "renderboxes", str(surface["zoom"]["max"]), link["filename"]) if path in linksByPath and path not in compareList: oldPath = link[ "path"] if "path" in link else outdata[ "maps"][str( old )]["surfaces"][surfaceName][ "links"][linkIndex]["path"] compareList[path] = (path, oldPath, linksByPath[path]) compareList = compareList.values() resultList = pool.map( partial(compareRenderbox, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"])), compareList, 16) count = 0 for (isDifferent, path, oldPath, links) in resultList: if not isDifferent: os.remove(path) for (surfaceName, linkIndex) in links: outdata["maps"][str(new)]["surfaces"][surfaceName][ "links"][linkIndex] = { "path": oldPath } else: count += 1 if args.verbose: print("removed %s of %s compared renderboxes, found %s new" % (count, len(compareList), totalCount)) # compress and build string for surfaceName, daytimeImageIndex in allImageIndex.items(): indexList = [] daytime = "night" if "night" in daytimeImageIndex and data["maps"][ new]["surfaces"][surfaceName] and str( data["maps"][new]["surfaces"][surfaceName]["night"]) else "day" if daytime not in daytimeImageIndex: # this is true if nothing changed continue surfaceImageIndex = daytimeImageIndex[daytime] for y, xList in surfaceImageIndex.items(): string = getBase64(y, False) isLastChangedImage = False isLastNightImage = False for x in range(min(xList), max(xList) + 2): isChangedImage = x in xList #does the image exist at all? isNightImage = daytime == "night" and ( str(x), str(y)) not in allDayImages[ surfaceName] #is this image only in night? if isLastChangedImage != isChangedImage or ( isChangedImage and isLastNightImage != isNightImage ): #differential encoding string += getBase64( x, isNightImage if isChangedImage else isLastNightImage) isLastChangedImage = isChangedImage isLastNightImage = isNightImage indexList.append(string) if surfaceName not in outdata["maps"][str(new)]["surfaces"]: outdata["maps"][str(new)]["surfaces"][surfaceName] = {} outdata["maps"][str( new)]["surfaces"][surfaceName]["chunks"] = '='.join(indexList) if len(indexList) > 0: changed = True if changed: if args.verbose: print("writing mapInfo.out.json") with outFile.open("w+", encoding="utf-8") as f: json.dump(outdata, f) if args.verbose: print("deleting empty folders") for curdir, subdirs, files in os.walk( Path(topPath, timestamp, surfaceReference, daytimeReference)): if len(subdirs) == 0 and len(files) == 0: os.rmdir(curdir)
def zoom(*args, **kwargs): psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10) toppath = os.path.join((args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0]) datapath = os.path.join(toppath, "mapInfo.json") basepath = os.path.join(toppath, "Images") maxthreads = int(kwargs["zoomthreads"]) if "zoomthreads" in kwargs else (int(kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count()) #print(basepath) with open(datapath, "r") as f: data = json.load(f) for mapIndex, map in enumerate(data["maps"]): if len(args) <= 1 or map["path"] == args[1]: for surfaceName, surface in map["surfaces"].items(): if len(args) <= 2 or surfaceName == args[2]: start = surface["zoom"]["max"] stop = surface["zoom"]["min"] daytimes = [] try: if surface["day"]: daytimes.append("day") except KeyError: pass try: if surface["night"]: daytimes.append("night") except KeyError: pass for daytime in daytimes: if len(args) <= 3 or daytime == args[3]: if not os.path.isdir(os.path.join(toppath, "Images", str(map["path"]), surfaceName, daytime, str(start - 1))): print("zoom {:5.1f}% [{}]".format(0, " " * (tsize()[0]-15)), end="") allBigChunks = {} for x in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(surface["zoom"]["max"]))): for y in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(surface["zoom"]["max"]), x)): allBigChunks[(int(x) >> start - stop, int(y.split('.', 2)[0]) >> start - stop)] = True pathList = [] for otherMapIndex in range(mapIndex, -1, -1): pathList.append(str(data["maps"][otherMapIndex]["path"])) threadsplit = 0 while 4**threadsplit * len(allBigChunks) < maxthreads: threadsplit = threadsplit + 1 threadsplit = min(max(start - stop - 3, 0), threadsplit + 3) allChunks = [] for pos in list(allBigChunks): for i in range(2**threadsplit): for j in range(2**threadsplit): allChunks.append((pos[0]*(2**threadsplit) + i, pos[1]*(2**threadsplit) + j)) threads = min(len(allChunks), maxthreads) processes = [] originalSize = len(allChunks) # print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList))) # print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks)))) counter = mp.Value('i', originalSize) resultQueue = mp.Queue() for _ in range(0, threads): p = mp.Process(target=thread, args=(basepath, pathList, surfaceName, daytime, start, stop + threadsplit, stop, allChunks, counter, resultQueue)) p.start() processes.append(p) doneSize = 0 for _ in range(originalSize): resultQueue.get(True) doneSize += 1 progress = float(doneSize) / originalSize tsiz = tsize()[0]-15 print("\rzoom {:5.1f}% [{}{}]".format(round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="") for p in processes: p.join() if threadsplit > 0: #print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks)))) processes = [] i = len(allBigChunks) - 1 for chunk in list(allBigChunks): p = mp.Process(target=work, args=(basepath, pathList, surfaceName, daytime, stop + threadsplit, stop, stop, chunk)) i = i - 1 p.start() processes.append(p) for p in processes: p.join() print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0]-15)))