def loadToNPArray(qkey):
beginningcoords = bingtile.QuadKeyToTileXY(qkey + '0')
xoffset = -1
yoffset = -1
loadingarr = np.zeros((1024, 1024), np.int16)
found = False
while yoffset < 3:
while xoffset < 3:
subqkey = bingtile.TileXYToQuadKey(beginningcoords[0] + xoffset,
beginningcoords[1] + yoffset,
beginningcoords[2])
if os.path.isfile("Tile/" + str(len(subqkey)) + "/dem_" + subqkey +
".rw"):
infile = open(
"Tile/" + str(len(subqkey)) + "/dem_" + subqkey + ".rw",
'rb')
tilearr = infile.read()
infile.close()
nparr = np.frombuffer(tilearr, np.int16).reshape((257, 257))
x_offset = (xoffset + 1) * 256
y_offset = (yoffset + 1) * 256
loadingarr[y_offset:y_offset + nparr.shape[0] - 1,
x_offset:x_offset + nparr.shape[1] -
1] = nparr[0:256, 0:256]
found = True
xoffset += 1
xoffset = -1
yoffset += 1
return found, loadingarr
import os
import glob
import cgl_generate as cglc
import bingtile
qkeybase = '102231'
qkeyx, qkeyy, qkeylvl = bingtile.QuadKeyToTileXY(qkeybase)
qkeystoprocess = []
tilesx = 1
tilesy = 1
padleft = 0
padtop = 0
padright = 0
padbottom = 0
idx = (-1) * padleft
idy = (-1) * padtop
while idy < tilesy + padbottom:
while idx < tilesx + padright:
qkeystoprocess.append(
bingtile.TileXYToQuadKey(qkeyx + idx, qkeyy + idy, qkeylvl))
idx += 1
idx = -1
idy += 1
for qkey in qkeystoprocess:
files = glob.glob("Tile/6/dem_" + qkey + "*.rw")
files += glob.glob("Delta/7/dem_" + qkey + "*.rw")
files += glob.glob("Delta/8/dem_" + qkey + "*.rw")
files += glob.glob("Delta/9/dem_" + qkey + "*.rw")
files += glob.glob("Delta/10/dem_" + qkey + "*.rw")
files += glob.glob("Delta/11/dem_" + qkey + "*.rw")
def decompress(src, baseqkey, dst):
"""
Decompressed SRC cgl that represents BASEQKEY to DST folder
"""
# Decompressing all sections from a CGL file
infile = open(src, 'rb')
inarr = infile.read()
headerstart = inarr[36]
headerlength = int.from_bytes(inarr[40:43], 'little') # UINT24
headerarr = inarr[headerstart:headerstart + headerlength]
prop = inarr[49]
pb = math.floor(prop / (9 * 5))
prop = prop - (pb * 9 * 5)
lp = math.floor(prop / 9)
lc = math.floor(prop - lp * 9)
my_filters = [
{
"id": lzma.FILTER_LZMA1,
"preset": lzma.PRESET_DEFAULT,
"lc": lc,
"lp": lp,
"pb": pb
},
]
headerunlzma = lzma.decompress(headerarr, lzma.FORMAT_RAW, None,
my_filters)
if not os.path.exists(os.path.dirname(dst)):
os.makedirs(os.path.dirname(dst))
if (len(headerunlzma) > 0):
outfile = open(dst + "headerunlzma.bin", "bw")
outfile.write(headerunlzma)
outfile.close()
compressedsizes = []
uncompressedsizes = []
headersize = len(headerunlzma)
# count dwords that have zero value at file start
# if 2, qkey len = WORD, if 4 OR 6, qkey len = DWORD
qkeysize = 0
i = 0
while True:
value = int.from_bytes(headerunlzma[i:i + 2], 'little')
i += 2
if value == 0:
qkeysize += 2
else:
break
if qkeysize == 6:
qkeysize = 4
# Extract tile qkeys #! QKEY LENGTH WORD
tilecount = int.from_bytes(inarr[32:34], 'little')
counter = 0
cbegin = 0
qkeysinbase10 = []
qkeys = []
previousval = 0
level = 0
while counter < tilecount:
val = int.from_bytes(headerunlzma[counter * 2:counter * 2 + 2],
'little')
newval = val + previousval
if newval >= 4096:
level += 1
newval -= 4096
qkeysinbase10.append([level, newval])
previousval = qkeysinbase10[counter][1]
qkey = ToTileQKey(level, qkeysinbase10[counter][1])
if counter == 0 and qkey == '0':
qkey = ''
qkeys.append(qkey)
counter += 1
startpoints = list(
misc.find_all(
inarr,
inarr[headerstart + headerlength:headerstart + headerlength + 4]))
lengths = []
i = 0
while i < len(startpoints) - 1:
lengths.append(startpoints[i + 1] - startpoints[i])
i += 1
lengths.append(len(inarr) - startpoints[-1])
# extract compressed sizes
compressedsizes = []
uncompressedsizes = []
previoussize = 0
counter = 0
tilecount = int.from_bytes(inarr[32:34], 'little')
cbegin = tilecount * qkeysize
i = cbegin
while counter < tilecount:
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print(counter)
print("Delta:" + str(val))
if val == 32768:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print(val)
val = previoussize + val
print("addzero")
print("Adding:" + str(val))
elif val == 32769:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print(val)
val = previoussize + 65536 + val
print("addfull")
print("Adding:" + str(val))
elif val == 65535:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print("Adding:" + str(val))
val = previoussize - 65536 + val
print("subfull")
elif val == 65534:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print("Adding:" + str(val))
val = previoussize - 65536 * 2 + val
print("subfull2")
elif val > 16384:
val = val + previoussize - 32768
print("Adding:" + str(val))
else:
val = previoussize + val
print("normal")
print("Adding:" + str(val))
i += 2
print("Newcs:" + str(val))
compressedsizes.append(val)
previoussize = val
counter += 1
for size in compressedsizes:
print(size)
i += 0
counter = 0
while counter < tilecount:
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
print(val)
if val == 32768:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little')
elif val == 32769:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little') + 65536
elif val == 32770:
i += 2
val = int.from_bytes(headerunlzma[i:i + 2], 'little') + 65536 * 2
i += 2
print(val)
print(compressedsizes[counter])
uncompressedsizes.append(val + compressedsizes[counter])
counter += 1
# decompress files DEM from blob
i = 0
datastart = headerstart + headerlength
dataend = datastart + compressedsizes[i]
prop = inarr[48]
pb = math.floor(prop / (9 * 5))
prop = math.floor(prop - (pb * 9 * 5))
lp = math.floor(prop / 9)
lc = math.floor(prop - lp * 9)
my_filters = [
{
"id": lzma.FILTER_LZMA1,
"preset": lzma.PRESET_DEFAULT,
"lc": lc,
"lp": lp,
"pb": pb
},
]
dummyhdr = open("misc/dummy.hdr", 'rb')
dummyarr = dummyhdr.read()
qkeybase = baseqkey
qkey = qkeybase
while i < len(compressedsizes):
print("Index: " + str(i))
print("Length:" + str(dataend - datastart))
# print(dataend)
# print(i)
print("datastart: " + f'{datastart:08}' + "end: " + f'{dataend:08}')
fullqkey = qkeybase + qkeys[i]
outfile = open(dst + "dem_" + fullqkey + ".rw", "wb")
outsize = outfile.write(
lzma.decompress(inarr[datastart:dataend], lzma.FORMAT_RAW, None,
my_filters)[0:])
outfile.close()
# write hdr file
xyl = bingtile.QuadKeyToTileXY(fullqkey)
pxy = bingtile.TileXYToPixelXY(xyl[0], xyl[1])
print(pxy)
print(xyl)
platlon = bingtile.PixelXYToLatLong(pxy[0], pxy[1], xyl[2])
yul = platlon[0]
xul = platlon[1]
pxy = bingtile.TileXYToPixelXY(xyl[0] + 1, xyl[1] + 1)
platlon = bingtile.PixelXYToLatLong(pxy[0], pxy[1], xyl[2])
yhep = (yul - platlon[0]) / 256
xwdp = (platlon[1] - xul) / 256
xul = xul + (xwdp / 2)
yul = yul - (yhep / 2)
outfile = open(dst + "dem_" + fullqkey + ".hdr", "w")
outfile.write("BYTEORDER I\n")
outfile.write("LAYOUT BIL\n")
outfile.write("NROWS 257\n")
outfile.write("NCOLS 257\n")
outfile.write("NBANDS 1\n")
if outsize == 66056:
outfile.write("NBITS 8\n")
outfile.write("BANDROWBYTES 257\n")
outfile.write("TOTALROWBYTES 257\n")
elif outsize == 132105:
outfile.write("NBITS 16\n")
outfile.write("BANDROWBYTES 514\n")
outfile.write("TOTALROWBYTES 514\n")
outfile.write("BANDGAPBYTES 0\n")
outfile.write("NODATA -9999\n")
outfile.write("SKIPBYTES 7\n")
outfile.write("PIXELTYPE SIGNEDINT\n")
outfile.write("ULXMAP " + str(xul) + "\n")
outfile.write("ULYMAP " + str(yul) + "\n")
outfile.write("XDIM " + str(xwdp) + "\n")
outfile.write("YDIM " + str(yhep) + "\n")
outfile.close()
i = i + 1
if (i < len(compressedsizes)):
datastart = dataend
dataend = dataend + compressedsizes[i]
