def crop(*args, **kwargs):
psutil.Process(os.getpid()).nice(
psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
subname = os.path.join(*args[1:4])
toppath = os.path.join(
(args[4] if len(args) > 4 else "../../script-output/FactorioMaps"),
args[0])
basepath = os.path.join(toppath, "Images", subname)
while not os.path.isdir(basepath) or len(
os.walk(basepath).__next__()[1]) == 0:
time.sleep(0.4)
folder = os.path.join(basepath, os.walk(basepath).__next__()[1][0])
datapath = os.path.join(basepath, "crop.txt")
maxthreads = int(kwargs["cropthreads"]) if "cropthreads" in kwargs else (
int(kwargs["maxthreads"])
if "maxthreads" in kwargs else mp.cpu_count())
if not os.path.exists(datapath):
#print("waiting for game")
while not os.path.exists(datapath):
time.sleep(1)
print("crop {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="")
files = []
with open(datapath, "r") as data:
imgsize = int(data.readline().rstrip('\n'))
for line in data:
files.append(line)
pool = mp.Pool(processes=maxthreads)
m = mp.Manager()
progressQueue = m.Queue()
originalSize = len(files)
doneSize = 0
while len(files) > 0:
workers = pool.map_async(
partial(work,
imgsize=imgsize,
folder=folder,
progressQueue=progressQueue), files, 128)
for _ in range(len(files)):
if progressQueue.get(True):
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0] - 15
print("\rcrop {:5.1f}% [{}{}]".format(
round(progress * 100, 1), "=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz))),
end="")
workers.wait()
files = [x for x in workers.get() if x]
if len(files) > 0:
time.sleep(10 if len(files) > 1000 else 1)
print("\rcrop {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
def printErase(arg):
try:
tsiz = tsize()[0]
print("\r{}{}\n".format(arg, " " * (tsiz*math.ceil(len(arg)/tsiz)-len(arg) - 1)), end="", flush=True)
except:
#raise
pass
def crop(outFolder, timestamp, surface, daytime, basePath=None, args: Namespace = Namespace()):
psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10)
subname = Path(timestamp, surface, daytime)
toppath = Path(
basePath if basePath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps"),
outFolder,
)
imagePath = Path(toppath, "Images")
datapath = Path(imagePath, subname, "crop.txt")
maxthreads = args.cropthreads if args.cropthreads else args.maxthreads
while not datapath.exists():
time.sleep(1)
print(f"crop {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="")
files = []
with datapath.open("r", encoding="utf-8") as data:
assert data.readline().rstrip("\n") == "v2"
for line in data:
files.append(line)
pool = mp.Pool(processes=maxthreads)
m = mp.Manager()
progressQueue = m.Queue()
originalSize = len(files)
doneSize = 0
try:
while len(files) > 0:
workers = pool.map_async(
partial(work, folder=imagePath, progressQueue=progressQueue),
files,
128,
)
for _ in range(len(files)):
if progressQueue.get(True):
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0] - 15
print(f"\rcrop {round(progress * 100, 1):5.1f}% [{'=' * int(progress * tsiz)}{' ' * (tsiz - int(progress * tsiz))}]",end="",)
workers.wait()
files = [x for x in workers.get() if x]
if len(files) > 0:
time.sleep(10 if len(files) > 1000 else 1)
print(f"\rcrop {100:5.1f}% [{'=' * (tsize()[0]-15)}]")
except KeyboardInterrupt:
time.sleep(0.2)
print(f"Keyboardinterrupt caught with {len(files)} files left.")
if len(files) < 40:
for line in files:
print(line)
raise
def image_index(directory):
'''function that uses os.walk(directory) - to output an index of all images
in directory and subdirectories of extensions: ("jpg", "JPG", "jpeg", "JPEG", "png", "PNG")'''
from os import walk
from shutil import get_terminal_size as tsize
l = list(walk(directory))
width = tsize()[0]
imgcount = 0
counter = 0
filenum = 0
while counter < len(l):
for i in range(len(l[counter][2])):
filenum += 1
if len(l[counter][2]) >= 1:
imagelist = [
i for i in l[counter][2]
if i.endswith(("jpg", "JPG", "jpeg", "JPEG", "png", "PNG"))
]
if len(imagelist) > 0:
print('\n')
print("FOLDER:".center(width))
print(str(l[counter][0]).center(width))
print("#", counter)
hr = print("-" * width)
hr
print("FILES: {}".format(len(l[counter][2])).center(width))
print("IMAGES: {}".format(len(imagelist)).center(width))
for x in imagelist:
print("[!]\t{}".format(x))
imgcount += 1
counter += 1
else:
pass
counter += 1
else:
print("\n{} subdirectories,\n{} files scanned\n{} images".format(
len(l), filenum, imgcount))
def maine():
from shutil import get_terminal_size as tsize
from os import listdir, getcwd
import sys
global sudo
global yes
print("Accio\n".center(tsize()[0]))
print(
'Program that finds images (jpg, jpeg, png, gif, bmp) in a given directory \nand subdirectories, and optionally transfers them into another one.\n'
)
print("Enter path/to/parent/directory to be scanned: \n")
loc = str(input(">>> ").strip(" "))
wd = lambda: getcwd()
try:
tl = listdir(loc)
assert len(tl) >= 1
image_index(loc)
except FileNotFoundError:
print("Not found")
except AssertionError:
print("Empty")
yes = ['y', 'Y', 'yes', 'YES']
while 1:
try:
print("Directory to be extracted: ")
origin = loc + "/" + str(input(">>> {}/".format(loc)).strip(" "))
print("Destination: ")
destination = str(input(">>> ").strip(" "))
print("Run as superuser? (Better chance of success):")
sudo = input(">>> ")
move_contents(origin, destination)
except KeyboardInterrupt:
print("Good Bye!")
sys.exit()
def ref(*args, **kwargs):
psutil.Process(os.getpid()).nice(
psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
toppath = os.path.join(
(args[4] if len(args) > 4 else "../../script-output/FactorioMaps"),
args[0])
datapath = os.path.join(toppath, "mapInfo.json")
maxthreads = int(kwargs["refthreads"]) if "refthreads" in kwargs else (int(
kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count())
pool = mp.Pool(processes=maxthreads)
with open(datapath, "r") as f:
data = json.load(f)
if os.path.isfile(datapath[:-5] + ".out.json"):
with open(datapath[:-5] + ".out.json", "r") as f:
outdata = json.load(f)
else:
outdata = {}
if len(args) > 1:
for i, mapObj in enumerate(data["maps"]):
if mapObj["path"] == args[1]:
new = i
break
else:
new = len(data["maps"]) - 1
newMap = data["maps"][new]
allImageIndex = {}
allDayImages = {}
for daytime in ("day", "night"):
newComparedSurfaces = []
compareList = []
keepList = []
firstRemoveList = []
cropList = {}
didAnything = False
if len(args) <= 3 or daytime == args[3]:
for surfaceName, surface in newMap["surfaces"].items():
if (len(args) <= 2 or surfaceName
== args[2]) and daytime in surface and str(
surface[daytime]) == "true" and (
len(args) <= 3 or daytime == args[3]):
didAnything = True
z = surface["zoom"]["max"]
dayImages = []
newComparedSurfaces.append((surfaceName, daytime))
oldMapsList = []
for old in range(new):
if surfaceName in data["maps"][old]["surfaces"]:
oldMapsList.append(old)
for old in oldMapsList:
with open(
os.path.join(toppath, "Images",
data["maps"][old]["path"],
surfaceName, daytime, "crop.txt"),
"r") as f:
next(f)
for line in f:
split = line.rstrip("\n").split(" ", 5)
cropList[(surfaceName, daytime, str(z),
int(split[0]),
int(os.path.splitext(
split[1])[0]))] = int(
split[4], 16)
with open(
os.path.join(toppath, "Images", newMap["path"],
surfaceName, daytime, "crop.txt"),
"r") as f:
next(f)
for line in f:
split = line.rstrip("\n").split(" ", 5)
cropList[(
surfaceName, daytime, str(z), int(split[0]),
int(os.path.splitext(split[1])[0])
)] = int(split[4], 16) | cropList.get(
(surfaceName, daytime, str(z), int(split[0]),
int(os.path.splitext(split[1])[0])), 0)
oldImages = {}
for old in oldMapsList:
if surfaceName in data["maps"][old][
"surfaces"] and daytime in surface and z == surface[
"zoom"]["max"]:
if surfaceName not in allImageIndex:
allImageIndex[surfaceName] = {}
path = os.path.join(toppath, "Images",
data["maps"][old]["path"],
surfaceName, daytime, str(z))
for x in os.listdir(path):
for y in os.listdir(os.path.join(path, x)):
oldImages[(x, y.replace(
ext,
outext))] = data["maps"][old]["path"]
if daytime != "day":
if not os.path.isfile(
os.path.join(toppath, "Images", newMap["path"],
surfaceName, "day", "ref.txt")):
print(
"WARNING: cannot find day surface to copy non-day surface from. running ref.py on night surfaces is not very accurate."
)
else:
if DEBUG:
print(
"found day surface, reuse results from ref.py from there"
)
with open(
os.path.join(toppath, "Images",
newMap["path"], surfaceName,
"day", "ref.txt"), "r") as f:
for line in f:
#if (line.rstrip("\n").split(" ", 2)[1] == "6"): print("YUP", line.rstrip("\n").split(" ", 2)[0])
dayImages.append(
tuple(line.rstrip("\n").split(" ", 2)))
allDayImages[surfaceName] = dayImages
path = os.path.join(toppath, "Images", newMap["path"],
surfaceName, daytime, str(z))
for x in os.listdir(path):
for y in os.listdir(os.path.join(path, x)):
#if (y == "6.png"): print("hoi", x)
if (x, os.path.splitext(y)[0]) in dayImages or (
x, y.replace(ext,
outext)) not in oldImages:
keepList.append(
(surfaceName, daytime, str(z), x, y))
elif (x, y.replace(ext, outext)) in oldImages:
compareList.append(
(oldImages[(x, y.replace(ext, outext))],
surfaceName, daytime, str(z), x, y))
if not didAnything:
continue
if DEBUG: print("found %s new images" % len(keepList))
if len(compareList) > 0:
if DEBUG: print("comparing %s existing images" % len(compareList))
treshold = .3 * Image.open(
os.path.join(toppath, "Images", *compareList[0]).replace(
ext, outext)).size[0]**2
#print(treshold)
#compare(compareList[0], treshold=treshold, basePath=os.path.join(toppath, "Images"), new=str(newMap["path"]))
m = mp.Manager()
progressQueue = m.Queue()
workers = pool.map_async(
partial(compare,
treshold=treshold,
basePath=os.path.join(toppath, "Images"),
new=str(newMap["path"]),
progressQueue=progressQueue), compareList, 128)
doneSize = 0
print("ref {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)),
end="")
for i in range(len(compareList)):
progressQueue.get(True)
doneSize += 1
progress = float(doneSize) / len(compareList)
tsiz = tsize()[0] - 15
print("\rref {:5.1f}% [{}{}]".format(
round(progress * 100, 1), "=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz))),
end="")
workers.wait()
resultList = workers.get()
newList = [x[1] for x in [x for x in resultList if x[0]]]
firstRemoveList += [
x[1] for x in [x for x in resultList if not x[0]]
]
if DEBUG:
print("found %s changed in %s images" %
(len(newList), len(compareList)))
keepList += newList
print("\rref {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
if DEBUG:
print("scanning %s chunks for neighbour cropping" %
len(firstRemoveList))
resultList = pool.map(
partial(neighbourScan, keepList=keepList, cropList=cropList),
firstRemoveList, 64)
neighbourList = [x[1] for x in [x for x in resultList if x[0]]]
removeList = [x[1] for x in [x for x in resultList if not x[0]]]
if DEBUG: print("keeping %s neighbouring images" % len(neighbourList))
if DEBUG:
print("deleting %s, keeping %s of %s existing images" %
(len(removeList), len(keepList) + len(neighbourList),
len(keepList) + len(neighbourList) + len(removeList)))
if DEBUG: print("removing identical images")
for x in removeList:
os.remove(os.path.join(toppath, "Images", newMap["path"], *x))
if DEBUG: print("creating render index")
for surfaceName, daytime in newComparedSurfaces:
z = surface["zoom"]["max"]
with open(
os.path.join(toppath, "Images", newMap["path"],
surfaceName, daytime, "ref.txt"), "w") as f:
for aList in (keepList, neighbourList):
for coord in aList:
if coord[0] == surfaceName and coord[
1] == daytime and coord[2] == str(z):
f.write("%s %s\n" %
(coord[3], os.path.splitext(coord[4])[0]))
if DEBUG: print("creating client index")
for aList in (keepList, neighbourList):
for coord in aList:
x = int(coord[3])
y = int(os.path.splitext(coord[4])[0])
if coord[0] not in allImageIndex:
allImageIndex[coord[0]] = {}
if coord[1] not in allImageIndex[coord[0]]:
allImageIndex[coord[0]][coord[1]] = {}
if y not in allImageIndex[coord[0]][coord[1]]:
allImageIndex[coord[0]][coord[1]][y] = [x]
elif x not in allImageIndex[coord[0]][coord[1]][y]:
allImageIndex[coord[0]][coord[1]][y].append(x)
# compress and build string
changed = False
if "maps" not in outdata:
outdata["maps"] = {}
if str(new) not in outdata["maps"]:
outdata["maps"][str(new)] = {"surfaces": {}}
for surfaceName, daytimeImageIndex in allImageIndex.items():
indexList = []
daytime = "night" if "night" in daytimeImageIndex and data["maps"][
new]["surfaces"][surfaceName] and str(
data["maps"][new]["surfaces"][surfaceName]
["night"]) == "true" else "day"
surfaceImageIndex = daytimeImageIndex[daytime]
for y, xList in surfaceImageIndex.items():
string = getBase64(y, False)
isLastChangedImage = False
isLastNightImage = False
for x in range(min(xList), max(xList) + 2):
isChangedImage = x in xList #does the image exist at all?
isNightImage = daytime == "night" and (
str(x), str(y)) not in allDayImages[
surfaceName] #is this image only in night?
if isLastChangedImage != isChangedImage or (
isChangedImage and isLastNightImage != isNightImage
): #differential encoding
string += getBase64(
x,
isNightImage if isChangedImage else isLastNightImage)
isLastChangedImage = isChangedImage
isLastNightImage = isNightImage
indexList.append(string)
if surfaceName not in outdata["maps"][str(new)]["surfaces"]:
outdata["maps"][str(new)]["surfaces"][surfaceName] = {}
outdata["maps"][str(
new)]["surfaces"][surfaceName]["chunks"] = '='.join(indexList)
if len(indexList) > 0:
changed = True
if changed:
if DEBUG: print("writing mapInfo.out.json")
with open(datapath[:-5] + ".out.json", "w+") as f:
json.dump(outdata, f)
if DEBUG: print("deleting empty folders")
for curdir, subdirs, files in os.walk(toppath, *args[1:4]):
if len(subdirs) == 0 and len(files) == 0:
os.rmdir(curdir)
def zoom(*args, **kwargs):
psutil.Process(os.getpid()).nice(
psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
needsThumbnail = (
str(args[5]).lower() != "false") if len(args) > 5 else True
toppath = os.path.join(
(args[4] if len(args) > 4 else "../../script-output/FactorioMaps"),
args[0])
datapath = os.path.join(toppath, "mapInfo.json")
basepath = os.path.join(toppath, "Images")
maxthreads = int(
kwargs["zoomthreads" if kwargs["zoomthreads"] else "maxthreads"])
#print(basepath)
with open(datapath, "r") as f:
data = json.load(f)
for mapIndex, map in enumerate(data["maps"]):
if len(args) <= 1 or map["path"] == args[1]:
for surfaceName, surface in map["surfaces"].items():
if len(args) <= 2 or surfaceName == args[2]:
maxzoom = surface["zoom"]["max"]
minzoom = surface["zoom"]["min"]
daytimes = []
try:
if surface["day"]: daytimes.append("day")
except KeyError:
pass
try:
if surface["night"]: daytimes.append("night")
except KeyError:
pass
for daytime in daytimes:
if len(args) <= 3 or daytime == args[3]:
if not os.path.isdir(
os.path.join(toppath, "Images",
str(map["path"]), surfaceName,
daytime, str(maxzoom - 1))):
print("zoom {:5.1f}% [{}]".format(
0, " " * (tsize()[0] - 15)),
end="")
generateThumbnail = needsThumbnail \
and mapIndex == len(data["maps"]) - 1 \
and surfaceName == ("nauvis" if "nauvis" in map["surfaces"] else sorted(map["surfaces"].keys())[0]) \
and daytime == daytimes[0]
allBigChunks = {}
minX = float("inf")
maxX = float("-inf")
minY = float("inf")
maxY = float("-inf")
imageSize = None
for xStr in os.listdir(
os.path.join(basepath,
str(map["path"]),
surfaceName, daytime,
str(maxzoom))):
x = int(xStr)
minX = min(minX, x)
maxX = max(maxX, x)
for yStr in os.listdir(
os.path.join(
basepath, str(map["path"]),
surfaceName, daytime,
str(maxzoom), xStr)):
if imageSize is None:
imageSize = Image.open(
os.path.join(
basepath, str(map["path"]),
surfaceName, daytime,
str(maxzoom), xStr, yStr),
mode='r').size[0]
y = int(yStr.split('.', 2)[0])
minY = min(minY, y)
maxY = max(maxY, y)
allBigChunks[(
x >> maxzoom - minzoom,
y >> maxzoom - minzoom)] = True
pathList = []
for otherMapIndex in range(mapIndex, -1, -1):
pathList.append(
str(data["maps"][otherMapIndex]
["path"]))
threadsplit = 0
while 4**threadsplit * len(
allBigChunks) < maxthreads:
threadsplit = threadsplit + 1
threadsplit = min(
max(maxzoom - minzoom - 3, 0),
threadsplit + 3)
allChunks = []
for pos in list(allBigChunks):
for i in range(2**threadsplit):
for j in range(2**threadsplit):
allChunks.append(
(pos[0] * (2**threadsplit) + i,
pos[1] * (2**threadsplit) +
j))
threads = min(len(allChunks), maxthreads)
processes = []
originalSize = len(allChunks)
# print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList)))
# print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks))))
counter = mp.Value('i', originalSize)
resultQueue = mp.Queue()
for _ in range(0, threads):
p = mp.Process(
target=thread,
args=(basepath, pathList, surfaceName,
daytime, imageSize, maxzoom,
minzoom + threadsplit, minzoom,
allChunks, counter, resultQueue,
generateThumbnail))
p.start()
processes.append(p)
doneSize = 0
for _ in range(originalSize):
resultQueue.get(True)
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0] - 15
print("\rzoom {:5.1f}% [{}{}]".format(
round(progress * 98,
1), "=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz))),
end="")
for p in processes:
p.join()
if threadsplit > 0:
#print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks))))
processes = []
i = len(allBigChunks) - 1
for chunk in list(allBigChunks):
p = mp.Process(
target=work,
args=(basepath, pathList,
surfaceName, daytime,
imageSize,
minzoom + threadsplit,
minzoom, minzoom, chunk,
generateThumbnail))
i = i - 1
p.start()
processes.append(p)
for p in processes:
p.join()
if generateThumbnail:
printErase("generating thumbnail")
minzoompath = os.path.join(
basepath, str(map["path"]),
surfaceName, daytime, str(minzoom))
thumbnail = Image.new(
'RGB', ((maxX - minX + 1) * imageSize
>> maxzoom - minzoom,
(maxY - minY + 1) * imageSize
>> maxzoom - minzoom),
BACKGROUNDCOLOR)
bigMinX = minX >> maxzoom - minzoom
bigMinY = minY >> maxzoom - minzoom
xOffset = (
(bigMinX * imageSize <<
maxzoom - minzoom) -
minX * imageSize) >> maxzoom - minzoom
yOffset = (
(bigMinY * imageSize <<
maxzoom - minzoom) -
minY * imageSize) >> maxzoom - minzoom
for chunk in list(allBigChunks):
path = os.path.join(
minzoompath, str(chunk[0]),
str(chunk[1]) + EXT)
thumbnail.paste(box=(
xOffset +
(chunk[0] - bigMinX) * imageSize,
yOffset +
(chunk[1] - bigMinY) * imageSize),
im=Image.open(
path, mode='r').
convert("RGB").resize(
(imageSize,
imageSize),
Image.ANTIALIAS))
if OUTEXT != EXT:
os.remove(path)
thumbnail.save(
os.path.join(
basepath,
"thumbnail" + THUMBNAILEXT))
print("\rzoom {:5.1f}% [{}]".format(
100, "=" * (tsize()[0] - 15)))
def crop(*args, **kwargs):
psutil.Process(os.getpid()).nice(
psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
subname = os.path.join(*args[1:4])
toppath = os.path.join(
(args[4] if len(args) > 4 else "../../script-output/FactorioMaps"),
args[0])
basepath = os.path.join(toppath, "Images")
datapath = os.path.join(basepath, subname, "crop.txt")
maxthreads = int(
kwargs["cropthreads" if kwargs["cropthreads"] else "maxthreads"])
if not os.path.exists(datapath):
#print("waiting for game")
while not os.path.exists(datapath):
time.sleep(1)
print("crop {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)), end="")
files = []
with open(datapath, "r") as data:
assert (data.readline().rstrip('\n') == "v2")
for line in data:
files.append(line)
pool = mp.Pool(processes=maxthreads)
m = mp.Manager()
progressQueue = m.Queue()
originalSize = len(files)
doneSize = 0
try:
while len(files) > 0:
workers = pool.map_async(
partial(work, folder=basepath, progressQueue=progressQueue),
files, 128)
for _ in range(len(files)):
if progressQueue.get(True):
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0] - 15
print("\rcrop {:5.1f}% [{}{}]".format(
round(progress * 100, 1), "=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz))),
end="")
workers.wait()
files = [x for x in workers.get() if x]
if len(files) > 0:
time.sleep(10 if len(files) > 1000 else 1)
print("\rcrop {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
except KeyboardInterrupt:
time.sleep(0.2)
print(f"Keyboardinterrupt caught with {len(files)} files left.")
if len(files) < 40:
for line in files:
print(line)
raise
def zoom(
outFolder: Path,
timestamp: str = None,
surfaceReference: str = None,
daytimeReference: str = None,
basepath: Path = None,
needsThumbnail: bool = True,
args: Namespace = Namespace(),
):
psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == "nt" else 10)
workFolder = basepath if basepath else Path(__file__, "..", "..", "..", "script-output", "FactorioMaps")
topPath = Path(workFolder, outFolder)
dataPath = Path(topPath, "mapInfo.json")
imagePath = Path(topPath, "Images")
maxthreads = args.zoomthreads if args.zoomthreads else args.maxthreads
with dataPath.open("r", encoding="utf-8") as f:
data = json.load(f)
for mapIndex, map in enumerate(data["maps"]):
if timestamp is None or map["path"] == timestamp:
for surfaceName, surface in map["surfaces"].items():
if surfaceReference is None or surfaceName == surfaceReference:
maxzoom = surface["zoom"]["max"]
minzoom = surface["zoom"]["min"]
daytimes = []
if "day" in surface:
daytimes.append("day")
if "night" in surface:
daytimes.append("night")
for daytime in daytimes:
if daytimeReference is None or daytime == daytimeReference:
if not Path(topPath, "Images", str(map["path"]), surfaceName, daytime, str(maxzoom - 1)).is_dir():
print(f"zoom {0:5.1f}% [{' ' * (tsize()[0]-15)}]", end="")
generateThumbnail = (
needsThumbnail
and mapIndex == len(data["maps"]) - 1
and surfaceName
== (
"nauvis"
if "nauvis" in map["surfaces"]
else sorted(map["surfaces"].keys())[0]
)
and daytime == daytimes[0]
)
allBigChunks = {}
minX = float("inf")
maxX = float("-inf")
minY = float("inf")
maxY = float("-inf")
imageSize: int = None
for xStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom)).iterdir():
x = int(xStr.name)
minX = min(minX, x)
maxX = max(maxX, x)
for yStr in Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr).iterdir():
if imageSize is None:
imageSize = Image.open(Path(imagePath, str(map["path"]), surfaceName, daytime, str(maxzoom), xStr, yStr), mode="r").size[0]
y = int(yStr.stem)
minY = min(minY, y)
maxY = max(maxY, y)
allBigChunks[
(
x >> maxzoom-minzoom,
y >> maxzoom-minzoom,
)
] = True
if len(allBigChunks) <= 0:
continue
pathList = []
for otherMapIndex in range(mapIndex, -1, -1):
pathList.append(str(data["maps"][otherMapIndex]["path"]))
threadsplit = 0
while 4**threadsplit * len(allBigChunks) < maxthreads:
threadsplit = threadsplit + 1
threadsplit = min(max(maxzoom - minzoom - 3, 0), threadsplit + 3)
allChunks = []
for pos in list(allBigChunks):
for i in range(2**threadsplit):
for j in range(2**threadsplit):
allChunks.append(
(
pos[0] * (2**threadsplit) + i,
pos[1] * (2**threadsplit) + j,
)
)
threads = min(len(allChunks), maxthreads)
processes = []
originalSize = len(allChunks)
# print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList)))
# print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks))))
counter = mp.Value("i", originalSize)
resultQueue = mp.Queue()
for _ in range(0, threads):
p = mp.Process(
target=thread,
args=(
imagePath,
pathList,
surfaceName,
daytime,
imageSize,
maxzoom,
minzoom + threadsplit,
minzoom,
allChunks,
counter,
resultQueue,
generateThumbnail,
),
)
p.start()
processes.append(p)
doneSize = 0
for _ in range(originalSize):
resultQueue.get(True)
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0] - 15
print(
"\rzoom {:5.1f}% [{}{}]".format(
round(progress * 98, 1),
"=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz)),
),
end="",
)
for p in processes:
p.join()
if threadsplit > 0:
# print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks))))
processes = []
i = len(allBigChunks) - 1
for chunk in list(allBigChunks):
p = mp.Process(
target=work,
args=(
imagePath,
pathList,
surfaceName,
daytime,
imageSize,
minzoom + threadsplit,
minzoom,
minzoom,
chunk,
generateThumbnail,
),
)
i = i - 1
p.start()
processes.append(p)
for p in processes:
p.join()
if generateThumbnail:
printErase("generating thumbnail")
minzoompath = Path(
imagePath,
str(map["path"]),
surfaceName,
daytime,
str(minzoom),
)
if imageSize is None:
raise Exception("Missing imageSize for thumbnail generation")
thumbnail = Image.new(
"RGB",
(
(maxX - minX + 1) * imageSize >> maxzoom-minzoom,
(maxY - minY + 1) * imageSize >> maxzoom-minzoom,
),
BACKGROUNDCOLOR,
)
bigMinX = minX >> maxzoom-minzoom
bigMinY = minY >> maxzoom-minzoom
xOffset = ((bigMinX * imageSize << maxzoom-minzoom) - minX * imageSize) >> maxzoom-minzoom
yOffset = ((bigMinY * imageSize << maxzoom-minzoom) - minY * imageSize) >> maxzoom-minzoom
for chunk in list(allBigChunks):
path = Path(minzoompath, str(chunk[0]), str(chunk[1])).with_suffix(EXT)
thumbnail.paste(
box=(
xOffset + (chunk[0] - bigMinX) * imageSize,
yOffset + (chunk[1] - bigMinY) * imageSize,
),
im=Image.open(path, mode="r")
.convert("RGB")
.resize((imageSize, imageSize), Image.ANTIALIAS),
)
if OUTEXT != EXT:
path.unlink()
thumbnail.save(Path(imagePath, "thumbnail" + THUMBNAILEXT))
print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
def ref(
outFolder: Path,
timestamp: str = None,
surfaceReference: str = None,
daytimeReference: str = None,
basepath: Path = None,
args: Namespace = Namespace(),
):
psutil.Process(os.getpid()).nice(
psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
workFolder = basepath if basepath else Path(
__file__, "..", "..", "..", "script-output", "FactorioMaps").resolve()
topPath = Path(workFolder, outFolder)
dataPath = Path(topPath, "mapInfo.json")
maxthreads = args.refthreads if args.refthreads else args.maxthreads
pool = mp.Pool(processes=maxthreads)
with open(dataPath, "r", encoding="utf-8") as f:
data = json.load(f)
outFile = Path(topPath, "mapInfo.out.json")
if outFile.exists():
with outFile.open("r", encoding="utf-8") as mapInfoOutFile:
outdata = json.load(mapInfoOutFile)
else:
outdata = {}
if timestamp:
for i, mapObj in enumerate(data["maps"]):
if mapObj["path"] == timestamp:
new = i
break
else:
new = len(data["maps"]) - 1
changed = False
if "maps" not in outdata:
outdata["maps"] = {}
if str(new) not in outdata["maps"]:
outdata["maps"][str(new)] = {"surfaces": {}}
newMap = data["maps"][new]
allImageIndex = {}
allDayImages = {}
for daytime in ("day", "night"):
newComparedSurfaces = []
compareList = []
keepList = []
firstRemoveList = []
cropList = {}
didAnything = False
if daytime is None or daytime == daytimeReference:
for surfaceName, surface in newMap["surfaces"].items():
if (surfaceReference is None or surfaceName
== surfaceReference) and daytime in surface and str(
surface[daytime]) and (daytime is None or daytime
== daytimeReference):
didAnything = True
z = surface["zoom"]["max"]
dayImages = []
newComparedSurfaces.append((surfaceName, daytime))
oldMapsList = []
for old in range(new):
if surfaceName in data["maps"][old]["surfaces"]:
oldMapsList.append(old)
def readCropList(path, combinePrevious):
with open(path, "r", encoding="utf-8") as f:
version = 2 if f.readline().rstrip(
'\n') == "v2" else 1
for line in f:
if version == 1:
split = line.rstrip("\n").split(" ", 5)
key = (surfaceName, daytime, str(z),
int(split[0]),
int(os.path.splitext(split[1])[0]))
value = split[4]
else:
split = line.rstrip("\n").split(" ", 5)
pathSplit = split[5].split("/", 5)
if pathSplit[3] != str(z):
continue
#(surfaceName, daytime, z, str(x+1), str(y+1) + ext)
key = (surfaceName, daytime, str(z),
int(pathSplit[4]),
int(
os.path.splitext(
pathSplit[5])[0]))
value = split[2]
cropList[key] = int(value, 16) | cropList.get(
key, 0) if combinePrevious else int(
value, 16)
for old in oldMapsList:
readCropList(
os.path.join(topPath, "Images",
data["maps"][old]["path"],
surfaceName, daytime, "crop.txt"),
False)
readCropList(
os.path.join(topPath, "Images", newMap["path"],
surfaceName, daytime, "crop.txt"), True)
oldImages = {}
for old in oldMapsList:
if surfaceName in data["maps"][old][
"surfaces"] and daytime in surface and z == surface[
"zoom"]["max"]:
if surfaceName not in allImageIndex:
allImageIndex[surfaceName] = {}
path = os.path.join(topPath, "Images",
data["maps"][old]["path"],
surfaceName, daytime, str(z))
for x in os.listdir(path):
for y in os.listdir(os.path.join(path, x)):
oldImages[(x, y.replace(
ext,
outext))] = data["maps"][old]["path"]
if daytime != "day":
if not os.path.isfile(
os.path.join(topPath, "Images", newMap["path"],
surfaceName, "day", "ref.txt")):
print(
"WARNING: cannot find day surface to copy non-day surface from. running ref.py on night surfaces is not very accurate."
)
else:
if args.verbose:
print(
"found day surface, reuse results from ref.py from there"
)
with Path(topPath, "Images", newMap["path"],
surfaceName, "day",
"ref.txt").open("r",
encoding="utf-8") as f:
for line in f:
dayImages.append(
tuple(line.rstrip("\n").split(" ", 2)))
allDayImages[surfaceName] = dayImages
path = os.path.join(topPath, "Images", newMap["path"],
surfaceName, daytime, str(z))
for x in os.listdir(path):
for y in os.listdir(os.path.join(path, x)):
if (x, os.path.splitext(y)[0]) in dayImages or (
x, y.replace(ext,
outext)) not in oldImages:
keepList.append(
(surfaceName, daytime, str(z), x, y))
elif (x, y.replace(ext, outext)) in oldImages:
compareList.append(
(oldImages[(x, y.replace(ext, outext))],
surfaceName, daytime, str(z), x, y))
if not didAnything:
continue
if args.verbose: print("found %s new images" % len(keepList))
if len(compareList) > 0:
if args.verbose:
print("comparing %s existing images" % len(compareList))
m = mp.Manager()
progressQueue = m.Queue()
#compare(compareList[0], treshold=treshold, basePath=os.path.join(topPath, "Images"), new=str(newMap["path"]), progressQueue=progressQueue)
workers = pool.map_async(
partial(compare,
basePath=os.path.join(topPath, "Images"),
new=str(newMap["path"]),
progressQueue=progressQueue), compareList, 128)
doneSize = 0
print("ref {:5.1f}% [{}]".format(0, " " * (tsize()[0] - 15)),
end="")
for i in range(len(compareList)):
progressQueue.get(True)
doneSize += 1
progress = float(doneSize) / len(compareList)
tsiz = tsize()[0] - 15
print("\rref {:5.1f}% [{}{}]".format(
round(progress * 100, 1), "=" * int(progress * tsiz),
" " * (tsiz - int(progress * tsiz))),
end="")
workers.wait()
resultList = workers.get()
newList = [x[1] for x in [x for x in resultList if x[0]]]
firstRemoveList += [
x[1] for x in [x for x in resultList if not x[0]]
]
if args.verbose:
print("found %s changed in %s images" %
(len(newList), len(compareList)))
keepList += newList
print("\rref {:5.1f}% [{}]".format(100, "=" * (tsize()[0] - 15)))
if args.verbose:
print("scanning %s chunks for neighbour cropping" %
len(firstRemoveList))
resultList = pool.map(
partial(neighbourScan, keepList=keepList, cropList=cropList),
firstRemoveList, 64)
neighbourList = [x[1] for x in [x for x in resultList if x[0]]]
removeList = [x[1] for x in [x for x in resultList if not x[0]]]
if args.verbose:
print("keeping %s neighbouring images" % len(neighbourList))
if args.verbose:
print("deleting %s, keeping %s of %s existing images" %
(len(removeList), len(keepList) + len(neighbourList),
len(keepList) + len(neighbourList) + len(removeList)))
if args.verbose: print("removing identical images")
for x in removeList:
os.remove(os.path.join(topPath, "Images", newMap["path"], *x))
if args.verbose: print("creating render index")
for surfaceName, daytime in newComparedSurfaces:
z = surface["zoom"]["max"]
with Path(topPath, "Images", newMap["path"], surfaceName, daytime,
"ref.txt").open("w", encoding="utf-8") as f:
for aList in (keepList, neighbourList):
for coord in aList:
if coord[0] == surfaceName and coord[
1] == daytime and coord[2] == str(z):
f.write("%s %s\n" %
(coord[3], os.path.splitext(coord[4])[0]))
if args.verbose: print("creating client index")
for aList in (keepList, neighbourList):
for coord in aList:
x = int(coord[3])
y = int(os.path.splitext(coord[4])[0])
if coord[0] not in allImageIndex:
allImageIndex[coord[0]] = {}
if coord[1] not in allImageIndex[coord[0]]:
allImageIndex[coord[0]][coord[1]] = {}
if y not in allImageIndex[coord[0]][coord[1]]:
allImageIndex[coord[0]][coord[1]][y] = [x]
elif x not in allImageIndex[coord[0]][coord[1]][y]:
allImageIndex[coord[0]][coord[1]][y].append(x)
if args.verbose: print("comparing renderboxes")
if "renderboxesCompared" not in outdata["maps"][str(new)]:
changed = True
outdata["maps"][str(new)]["renderboxesCompared"] = True
compareList = {}
totalCount = 0
for surfaceName, surface in newMap["surfaces"].items():
linksByPath = {}
for linkIndex, link in enumerate(surface["links"]):
if surfaceName not in outdata["maps"][str(
new)]["surfaces"]:
outdata["maps"][str(new)]["surfaces"][surfaceName] = {
"links": []
}
outdata["maps"][str(
new)]["surfaces"][surfaceName]["links"].append(
{"path": newMap["path"]})
for daytime in ("day", "night"):
if link["type"] == "link_renderbox_area" and (
link["daynight"] or daytime == "day"):
path = os.path.join(
link["toSurface"],
daytime if link["daynight"] else "day",
"renderboxes", str(surface["zoom"]["max"]),
link["filename"])
if path not in linksByPath:
linksByPath[path] = [(surfaceName, linkIndex)]
else:
linksByPath[path].append(
(surfaceName, linkIndex))
totalCount += 1
for old in range(new - 1, -1, -1):
if surfaceName in data["maps"][old]["surfaces"]:
for linkIndex, link in enumerate(
data["maps"][old]["surfaces"][surfaceName]
["links"]):
for daytime in ("day", "night"):
if link["type"] == "link_renderbox_area" and (
link["daynight"] or daytime == "day"):
path = os.path.join(
link["toSurface"],
daytime if link["daynight"] else "day",
"renderboxes",
str(surface["zoom"]["max"]),
link["filename"])
if path in linksByPath and path not in compareList:
oldPath = link[
"path"] if "path" in link else outdata[
"maps"][str(
old
)]["surfaces"][surfaceName][
"links"][linkIndex]["path"]
compareList[path] = (path, oldPath,
linksByPath[path])
compareList = compareList.values()
resultList = pool.map(
partial(compareRenderbox,
basePath=os.path.join(topPath, "Images"),
new=str(newMap["path"])), compareList, 16)
count = 0
for (isDifferent, path, oldPath, links) in resultList:
if not isDifferent:
os.remove(path)
for (surfaceName, linkIndex) in links:
outdata["maps"][str(new)]["surfaces"][surfaceName][
"links"][linkIndex] = {
"path": oldPath
}
else:
count += 1
if args.verbose:
print("removed %s of %s compared renderboxes, found %s new" %
(count, len(compareList), totalCount))
# compress and build string
for surfaceName, daytimeImageIndex in allImageIndex.items():
indexList = []
daytime = "night" if "night" in daytimeImageIndex and data["maps"][
new]["surfaces"][surfaceName] and str(
data["maps"][new]["surfaces"][surfaceName]["night"]) else "day"
if daytime not in daytimeImageIndex: # this is true if nothing changed
continue
surfaceImageIndex = daytimeImageIndex[daytime]
for y, xList in surfaceImageIndex.items():
string = getBase64(y, False)
isLastChangedImage = False
isLastNightImage = False
for x in range(min(xList), max(xList) + 2):
isChangedImage = x in xList #does the image exist at all?
isNightImage = daytime == "night" and (
str(x), str(y)) not in allDayImages[
surfaceName] #is this image only in night?
if isLastChangedImage != isChangedImage or (
isChangedImage and isLastNightImage != isNightImage
): #differential encoding
string += getBase64(
x,
isNightImage if isChangedImage else isLastNightImage)
isLastChangedImage = isChangedImage
isLastNightImage = isNightImage
indexList.append(string)
if surfaceName not in outdata["maps"][str(new)]["surfaces"]:
outdata["maps"][str(new)]["surfaces"][surfaceName] = {}
outdata["maps"][str(
new)]["surfaces"][surfaceName]["chunks"] = '='.join(indexList)
if len(indexList) > 0:
changed = True
if changed:
if args.verbose: print("writing mapInfo.out.json")
with outFile.open("w+", encoding="utf-8") as f:
json.dump(outdata, f)
if args.verbose: print("deleting empty folders")
for curdir, subdirs, files in os.walk(
Path(topPath, timestamp, surfaceReference, daytimeReference)):
if len(subdirs) == 0 and len(files) == 0:
os.rmdir(curdir)
def zoom(*args, **kwargs):
psutil.Process(os.getpid()).nice(psutil.BELOW_NORMAL_PRIORITY_CLASS if os.name == 'nt' else 10)
toppath = os.path.join((args[4] if len(args) > 4 else "../../script-output/FactorioMaps"), args[0])
datapath = os.path.join(toppath, "mapInfo.json")
basepath = os.path.join(toppath, "Images")
maxthreads = int(kwargs["zoomthreads"]) if "zoomthreads" in kwargs else (int(kwargs["maxthreads"]) if "maxthreads" in kwargs else mp.cpu_count())
#print(basepath)
with open(datapath, "r") as f:
data = json.load(f)
for mapIndex, map in enumerate(data["maps"]):
if len(args) <= 1 or map["path"] == args[1]:
for surfaceName, surface in map["surfaces"].items():
if len(args) <= 2 or surfaceName == args[2]:
start = surface["zoom"]["max"]
stop = surface["zoom"]["min"]
daytimes = []
try:
if surface["day"]: daytimes.append("day")
except KeyError: pass
try:
if surface["night"]: daytimes.append("night")
except KeyError: pass
for daytime in daytimes:
if len(args) <= 3 or daytime == args[3]:
if not os.path.isdir(os.path.join(toppath, "Images", str(map["path"]), surfaceName, daytime, str(start - 1))):
print("zoom {:5.1f}% [{}]".format(0, " " * (tsize()[0]-15)), end="")
allBigChunks = {}
for x in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(surface["zoom"]["max"]))):
for y in os.listdir(os.path.join(basepath, str(map["path"]), surfaceName, daytime, str(surface["zoom"]["max"]), x)):
allBigChunks[(int(x) >> start - stop, int(y.split('.', 2)[0]) >> start - stop)] = True
pathList = []
for otherMapIndex in range(mapIndex, -1, -1):
pathList.append(str(data["maps"][otherMapIndex]["path"]))
threadsplit = 0
while 4**threadsplit * len(allBigChunks) < maxthreads:
threadsplit = threadsplit + 1
threadsplit = min(max(start - stop - 3, 0), threadsplit + 3)
allChunks = []
for pos in list(allBigChunks):
for i in range(2**threadsplit):
for j in range(2**threadsplit):
allChunks.append((pos[0]*(2**threadsplit) + i, pos[1]*(2**threadsplit) + j))
threads = min(len(allChunks), maxthreads)
processes = []
originalSize = len(allChunks)
# print(("%s %s %s %s" % (pathList[0], str(surfaceName), daytime, pathList)))
# print(("%s-%s (total: %s):" % (start, stop + threadsplit, len(allChunks))))
counter = mp.Value('i', originalSize)
resultQueue = mp.Queue()
for _ in range(0, threads):
p = mp.Process(target=thread, args=(basepath, pathList, surfaceName, daytime, start, stop + threadsplit, stop, allChunks, counter, resultQueue))
p.start()
processes.append(p)
doneSize = 0
for _ in range(originalSize):
resultQueue.get(True)
doneSize += 1
progress = float(doneSize) / originalSize
tsiz = tsize()[0]-15
print("\rzoom {:5.1f}% [{}{}]".format(round(progress * 100, 1), "=" * int(progress * tsiz), " " * (tsiz - int(progress * tsiz))), end="")
for p in processes:
p.join()
if threadsplit > 0:
#print(("finishing up: %s-%s (total: %s)" % (stop + threadsplit, stop, len(allBigChunks))))
processes = []
i = len(allBigChunks) - 1
for chunk in list(allBigChunks):
p = mp.Process(target=work, args=(basepath, pathList, surfaceName, daytime, stop + threadsplit, stop, stop, chunk))
i = i - 1
p.start()
processes.append(p)
for p in processes:
p.join()
print("\rzoom {:5.1f}% [{}]".format(100, "=" * (tsize()[0]-15)))
