def test_misc_9():
old_val = gdal.GetCacheMax()
gdal.SetCacheMax(3000000000)
ret_val = gdal.GetCacheMax()
gdal.SetCacheMax(old_val)
assert ret_val == 3000000000, 'did not get expected value'
def misc_9():
old_val = gdal.GetCacheMax()
gdal.SetCacheMax(3000000000)
ret_val = gdal.GetCacheMax()
gdal.SetCacheMax(old_val)
if ret_val != 3000000000:
gdaltest.post_reason('did not get expected value')
print(ret_val)
return 'fail'
return 'success'
def mask_23():
drv = gdal.GetDriverByName('GTiff')
md = drv.GetMetadata()
if md['DMD_CREATIONOPTIONLIST'].find('JPEG') == -1:
return 'skip'
src_ds = drv.Create('tmp/mask_23_src.tif', 3000, 2000, 3, options=['TILED=YES', 'SPARSE_OK=YES'])
src_ds.CreateMaskBand(gdal.GMF_PER_DATASET)
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', 'YES')
old_val = gdal.GetCacheMax()
gdal.SetCacheMax(15000000)
gdal.ErrorReset()
ds = drv.CreateCopy('tmp/mask_23_dst.tif', src_ds, options=['TILED=YES', 'COMPRESS=JPEG'])
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', 'NO')
gdal.SetCacheMax(old_val)
del ds
error_msg = gdal.GetLastErrorMsg()
src_ds = None
drv.Delete('tmp/mask_23_src.tif')
drv.Delete('tmp/mask_23_dst.tif')
# 'ERROR 1: TIFFRewriteDirectory:Error fetching directory count' was triggered before
if error_msg != '':
return 'fail'
return 'success'
def basic_test_7_internal():
try:
gdal.Open('non_existing_ds', gdal.GA_ReadOnly)
gdaltest.post_reason('opening should have thrown an exception')
return 'fail'
except:
# Special case: we should still be able to get the error message
# until we call a new GDAL function
if not matches_non_existing_error_msg(gdal.GetLastErrorMsg()):
gdaltest.post_reason('did not get expected error message, got %s' %
gdal.GetLastErrorMsg())
return 'fail'
if gdal.GetLastErrorType() == 0:
gdaltest.post_reason('did not get expected error type')
return 'fail'
# Should issue an implicit CPLErrorReset()
gdal.GetCacheMax()
if gdal.GetLastErrorType() != 0:
gdaltest.post_reason('got unexpected error type')
return 'fail'
return 'success'
def basic_test_7_internal():
try:
gdal.Open('non_existing_ds', gdal.GA_ReadOnly)
gdaltest.post_reason('opening should have thrown an exception')
return 'fail'
except:
# Special case: we should still be able to get the error message
# until we call a new GDAL function
if gdal.GetLastErrorMsg(
) != '`non_existing_ds\' does not exist in the file system,\nand is not recognised as a supported dataset name.\n':
gdaltest.post_reason('did not get expected error message')
return 'fail'
if gdal.GetLastErrorType() == 0:
gdaltest.post_reason('did not get expected error type')
return 'fail'
# Should issue an implicit CPLErrorReset()
gdal.GetCacheMax()
if gdal.GetLastErrorType() != 0:
gdaltest.post_reason('got unexpected error type')
return 'fail'
return 'success'
def test_jpeg_18():
height = 1024
width = 1024
src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_18.tif', width,
height, 1)
for i in range(height):
data = struct.pack('B' * 1, int(i / (height / 256)))
src_ds.WriteRaster(0, i, width, 1, data, 1, 1)
ds = gdal.GetDriverByName('JPEG').CreateCopy('/vsimem/jpeg_18.jpg',
src_ds,
options=['QUALITY=99'])
src_ds = None
gdal.Unlink('/vsimem/jpeg_18.tif')
oldSize = gdal.GetCacheMax()
gdal.SetCacheMax(0)
line0 = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
data = struct.unpack('B' * width, line0)
assert abs(data[0] - 0) <= 10
line1023 = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1)
data = struct.unpack('B' * width, line1023)
assert abs(data[0] - 255) <= 10
line0_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0, 0, int(width / 4), 1)
data = struct.unpack('B' * (int(width / 4)), line0_ovr1)
assert abs(data[0] - 0) <= 10
line1023_bis = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1)
assert line1023_bis != line0 and line1023 == line1023_bis
line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
assert line0 == line0_bis
line255_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0,
int(height / 4) - 1, int(width / 4), 1)
data = struct.unpack('B' * int(width / 4), line255_ovr1)
assert abs(data[0] - 255) <= 10
line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
assert line0 == line0_bis
line0_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0, 0, int(width / 4), 1)
assert line0_ovr1 == line0_ovr1_bis
line255_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0,
int(height / 4) - 1, int(width / 4), 1)
assert line255_ovr1 == line255_ovr1_bis
gdal.SetCacheMax(oldSize)
ds = None
gdal.Unlink('/vsimem/jpeg_18.jpg')
def gdal_config_options(self, cmd=''):
extra_args = []
if 'GDAL_CACHEMAX' not in cmd:
value = gdal.GetCacheMax()
extra_args.extend(('--config', 'GDAL_CACHEMAX', str(value)))
for key in ('CPL_DEBUG', 'GDAL_SKIP', 'GDAL_DATA',
'GDAL_DRIVER_PATH', 'OGR_DRIVER_PATH'):
if key not in cmd:
value = gdal.GetConfigOption(key, None)
if value:
extra_args.extend(('--config', key, '"%s"' % value))
return extra_args
def basic_test_7_internal():
with pytest.raises(Exception):
gdal.Open('non_existing_ds', gdal.GA_ReadOnly)
# Special case: we should still be able to get the error message
# until we call a new GDAL function
assert matches_non_existing_error_msg(gdal.GetLastErrorMsg()), ('did not get expected error message, got %s' % gdal.GetLastErrorMsg())
# Special case: we should still be able to get the error message
# until we call a new GDAL function
assert matches_non_existing_error_msg(gdal.GetLastErrorMsg()), 'did not get expected error message, got %s' % gdal.GetLastErrorMsg()
assert gdal.GetLastErrorType() != 0, 'did not get expected error type'
# Should issue an implicit CPLErrorReset()
gdal.GetCacheMax()
assert gdal.GetLastErrorType() == 0, 'got unexpected error type'
def jpeg_18():
height = 1024
width = 1024
src_ds = gdal.GetDriverByName('GTiff').Create('/vsimem/jpeg_18.tif', width,
height, 1)
for i in range(height):
data = struct.pack('B' * 1, int(i / (height / 256)))
src_ds.WriteRaster(0, i, width, 1, data, 1, 1)
ds = gdal.GetDriverByName('JPEG').CreateCopy('/vsimem/jpeg_18.jpg',
src_ds,
options=['QUALITY=99'])
src_ds = None
gdal.Unlink('/vsimem/jpeg_18.tif')
oldSize = gdal.GetCacheMax()
gdal.SetCacheMax(0)
line0 = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
data = struct.unpack('B' * width, line0)
if abs(data[0] - 0) > 10:
return 'fail'
line1023 = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1)
data = struct.unpack('B' * width, line1023)
if abs(data[0] - 255) > 10:
return 'fail'
line0_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0, 0, int(width / 4), 1)
data = struct.unpack('B' * (int(width / 4)), line0_ovr1)
if abs(data[0] - 0) > 10:
return 'fail'
line1023_bis = ds.GetRasterBand(1).ReadRaster(0, height - 1, width, 1)
if line1023_bis == line0 or line1023 != line1023_bis:
gdaltest.post_reason('fail')
return 'fail'
line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
if line0 != line0_bis:
gdaltest.post_reason('fail')
return 'fail'
line255_ovr1 = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0,
int(height / 4) - 1, int(width / 4), 1)
data = struct.unpack('B' * int(width / 4), line255_ovr1)
if abs(data[0] - 255) > 10:
return 'fail'
line0_bis = ds.GetRasterBand(1).ReadRaster(0, 0, width, 1)
if line0 != line0_bis:
gdaltest.post_reason('fail')
return 'fail'
line0_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0, 0, int(width / 4), 1)
if line0_ovr1 != line0_ovr1_bis:
gdaltest.post_reason('fail')
return 'fail'
line255_ovr1_bis = ds.GetRasterBand(1).GetOverview(1).ReadRaster(
0,
int(height / 4) - 1, int(width / 4), 1)
if line255_ovr1 != line255_ovr1_bis:
gdaltest.post_reason('fail')
return 'fail'
gdal.SetCacheMax(oldSize)
ds = None
gdal.Unlink('/vsimem/jpeg_18.jpg')
return 'success'
def test_mask_14():
src_ds = gdal.Open('data/byte.tif')
assert src_ds is not None, 'Failed to open test dataset.'
drv = gdal.GetDriverByName('GTiff')
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'FALSE'):
ds = drv.CreateCopy('tmp/byte_with_mask.tif', src_ds)
src_ds = None
# The only flag value supported for internal mask is GMF_PER_DATASET
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(0)
assert ret != 0, 'Error expected'
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(gdal.GMF_PER_DATASET)
assert ret == 0, 'Creation failed'
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
assert cs == 0, 'Got wrong checksum for the mask (1)'
ds.GetRasterBand(1).GetMaskBand().Fill(1)
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
assert cs == 400, 'Got wrong checksum for the mask (2)'
# This TIFF dataset has already an internal mask band
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(gdal.GMF_PER_DATASET)
assert ret != 0, 'Error expected'
# This TIFF dataset has already an internal mask band
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET)
assert ret != 0, 'Error expected'
ds = None
with pytest.raises(OSError,
message='tmp/byte_with_mask.tif.msk should not exist'):
os.stat('tmp/byte_with_mask.tif.msk')
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'FALSE'):
ds = gdal.Open('tmp/byte_with_mask.tif')
assert ds.GetRasterBand(1).GetMaskFlags() == gdal.GMF_PER_DATASET, \
'wrong mask flags'
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
assert cs == 400, 'Got wrong checksum for the mask (3)'
# Test fix for #5884
old_val = gdal.GetCacheMax()
gdal.SetCacheMax(0)
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
out_ds = drv.CreateCopy('/vsimem/byte_with_mask.tif',
ds,
options=['COMPRESS=JPEG'])
gdal.SetCacheMax(old_val)
assert out_ds.GetRasterBand(1).Checksum() != 0
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
assert cs == 400, 'Got wrong checksum for the mask (4)'
out_ds = None
drv.Delete('/vsimem/byte_with_mask.tif')
ds = None
drv.Delete('tmp/byte_with_mask.tif')
def _create_thumbnail(red_file,
green_file,
blue_file,
output_path,
x_constraint=None,
nodata=-999,
work_dir=None,
overwrite=True):
"""
Create JPEG thumbnail image using individual R, G, B images.
This method comes from the old ULA codebase.
:param red_file: red band data file
:param green_file: green band data file
:param blue_file: blue band data file
:param output_path: thumbnail file to write to.
:param x_constraint: thumbnail width (if not full resolution)
:param nodata: null/fill data value
:param work_dir: temp/work directory to use.
:param overwrite: overwrite existing thumbnail?
Thumbnail height is adjusted automatically to match the aspect ratio
of the input images.
"""
nodata = int(nodata)
# GDAL calls need absolute paths.
thumbnail_path = pathlib.Path(output_path).absolute()
if thumbnail_path.exists() and not overwrite:
_LOG.warning('File already exists. Skipping creation of %s',
thumbnail_path)
return None, None, None
# thumbnail_image = os.path.abspath(thumbnail_image)
out_directory = str(thumbnail_path.parent)
work_dir = os.path.abspath(work_dir) if work_dir else tempfile.mkdtemp(
prefix='.thumb-tmp', dir=out_directory)
try:
# working files
file_to = os.path.join(work_dir, 'rgb.vrt')
warp_to_file = os.path.join(work_dir, 'rgb-warped.vrt')
outtif = os.path.join(work_dir, 'thumbnail.tif')
# Build the RGB Virtual Raster at full resolution
run_command([
"gdalbuildvrt", "-overwrite", "-separate", file_to,
str(red_file),
str(green_file),
str(blue_file)
], work_dir)
assert os.path.exists(file_to), "VRT must exist"
# Determine the pixel scaling to get the correct width thumbnail
vrt = gdal.Open(file_to)
intransform = vrt.GetGeoTransform()
inpixelx = intransform[1]
# inpixely = intransform[5]
inrows = vrt.RasterYSize
incols = vrt.RasterXSize
# If a specific resolution is asked for.
if x_constraint:
outresx = inpixelx * incols / x_constraint
_LOG.info('Input pixel res %r, output pixel res %r', inpixelx,
outresx)
outrows = int(
math.ceil((float(inrows) / float(incols)) * x_constraint))
run_command([
"gdalwarp", "--config", "GDAL_CACHEMAX",
str(GDAL_CACHE_MAX_MB), "-of", "VRT", "-tr",
str(outresx),
str(outresx), "-r", "near", "-overwrite", file_to, warp_to_file
], work_dir)
else:
# Otherwise use a full resolution browse image.
outrows = inrows
x_constraint = incols
warp_to_file = file_to
outresx = inpixelx
_LOG.debug('Current GDAL cache max %rMB. Setting to %rMB',
gdal.GetCacheMax() / 1024 / 1024, GDAL_CACHE_MAX_MB)
gdal.SetCacheMax(GDAL_CACHE_MAX_MB * 1024 * 1024)
# Open VRT file to array
vrt = gdal.Open(warp_to_file)
driver = gdal.GetDriverByName("GTiff")
outdataset = driver.Create(outtif, x_constraint, outrows, 3,
gdalconst.GDT_Byte)
# Loop through bands and apply Scale and Offset
for band_number in (1, 2, 3):
band = vrt.GetRasterBand(band_number)
scale, offset = _calculate_scale_offset(nodata, band)
# Apply gain and offset
outdataset.GetRasterBand(band_number).WriteArray(
(numpy.ma.masked_less_equal(band.ReadAsArray(), nodata) *
scale) + offset)
_LOG.debug('Scale %r, offset %r', scale, offset)
# Must close datasets to flush to disk.
# noinspection PyUnusedLocal
outdataset = None
# noinspection PyUnusedLocal
vrt = None
# GDAL Create doesn't support JPEG so we need to make a copy of the GeoTIFF
run_command([
"gdal_translate", "--config", "GDAL_CACHEMAX",
str(GDAL_CACHE_MAX_MB), "-of", "JPEG", outtif,
str(thumbnail_path)
], work_dir)
_LOG.debug('Cleaning work files')
finally:
# Clean up work files
if os.path.exists(work_dir):
shutil.rmtree(work_dir)
# Newer versions of GDAL create aux files due to the histogram. Clean them up.
for f in (red_file, blue_file, green_file):
f = pathlib.Path(f)
aux_file = f.with_name(f.name + '.aux.xml')
if aux_file.exists():
_LOG.info('Cleaning aux: %s', aux_file)
os.remove(str(aux_file.absolute()))
return x_constraint, outrows, outresx
def genTiles(task):
time_start = datetime.now()
try:
cpu_count = 1
if thread_count is None:
cpu_count = multiprocessing.cpu_count()
else:
cpu_count = thread_count
s_file = None
if source_file.endswith(".zip"):
src = source_file.replace("\\", "/")
fileinfo = QFileInfo(src)
filename = fileinfo.completeBaseName()
s_file = [src, filename]
else:
return [0, time_start]
#cpu_count = min(6, cpu_count)
#if cpu_count % 2 == 0:
# cpu_count = int(cpu_count / 2)
# converted_source = os.path.join(source_folder, "converted").replace("\\","/")
QgsMessageLog.logMessage(
'Started creating dataset from {name}'.format(name=s_file[1]),
CATEGORY, Qgis.Info)
source_folder = os.path.dirname(source_file)
org_vrt = os.path.join(source_folder,
"OrgSource.vrt").replace("\\", "/")
if os.path.isfile(org_vrt):
os.remove(org_vrt)
org_files = []
c_file = '/vsizip/{archive}/{file}'.format(archive=s_file[0],
file=source_file_name)
QgsMessageLog.logMessage(c_file, CATEGORY, Qgis.Info)
org_files.append(c_file)
ds = gdal.BuildVRT(org_vrt,
org_files,
resolution="highest",
resampleAlg="bilinear")
ds.FlushCache()
ds = None
sleep(0.05)
QgsMessageLog.logMessage('Created original vrt', CATEGORY, Qgis.Info)
vrt = os.path.join(source_folder, "Source.vrt").replace("\\", "/")
if os.path.isfile(vrt):
os.remove(vrt)
pds = gdal.Open(org_vrt)
pds = gdal.Warp(vrt, pds, dstSRS="EPSG:3857")
pds = None
QgsMessageLog.logMessage('Created reporojected vrt', CATEGORY,
Qgis.Info)
terrarium_tile = os.path.join(source_folder,
"TerrariumSource.vrt").replace(
"\\", "/")
if os.path.isfile(terrarium_tile):
os.remove(terrarium_tile)
statistics = []
task.setProgress(0)
ress = calculateStat(c_file)
sleep(0.05)
if ress is not None:
statistics.append(ress)
task.setProgress(12)
else:
return [0, time_start]
ress = None
org_files = None
# Find minV and maxV from statistics
minV = None
maxV = None
for stat in statistics:
if stat.minV is not None and stat.maxV is not None:
if minV is None:
minV = stat.minV
elif stat.minV < minV:
minV = stat.minV
if maxV is None:
maxV = stat.maxV
elif stat.maxV > maxV:
maxV = stat.maxV
if minV is None or maxV is None:
QgsMessageLog.logMessage(
'Error: Minimum and maximum height are None', CATEGORY,
Qgis.Info)
return None
else:
QgsMessageLog.logMessage(
'Minimum and maximum height are {minv} and {maxv}'.format(
minv=minV, maxv=maxV), CATEGORY, Qgis.Info)
statistics = None
color_ramp_file = os.path.join(source_folder,
"TerrariumSource.txt").replace(
"\\", "/")
if os.path.isfile(color_ramp_file):
os.remove(color_ramp_file)
color_ramp = []
altitudes = []
# Create color ramp
for meters in range(minV, maxV):
for fraction in range(0, 256):
altitudes.append(meters + (fraction / 256))
exx = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count)
ff = exx.map(createRamp, repeat(convert_feet_to_meters), altitudes)
for res in ff:
if res is not None:
color_ramp.append(res)
QgsMessageLog.logMessage('Creating color ramp file', CATEGORY,
Qgis.Info)
#sorted_color_ramp = sorted(color_ramp, key=itemgetter(0))
color_ramp.sort(key=lambda x: x.altitude)
f = open(color_ramp_file, "w")
rt = 0
cf = len(color_ramp)
for ramp in color_ramp:
if manual_nodata_value is not None:
if ramp.altitude == manual_nodata_value:
f.write('{altitude}\t0\t0\t0\t0\n'.format(
altitude=manual_nodata_value))
continue
f.write('{altitude}\t{red}\t{green}\t{blue}\n'.format(
altitude=ramp.altitude,
red=ramp.red,
green=ramp.green,
blue=ramp.blue))
rt += 1
task.setProgress(max(0, min(int(((rt * 8) / cf) + 12), 100)))
f.write('nv\t0\t0\t0\t0')
f.close()
sleep(0.05)
QgsMessageLog.logMessage('Created color ramp file', CATEGORY,
Qgis.Info)
QgsMessageLog.logMessage(
'Rendering vrt to terrarium format. This might take a while',
CATEGORY, Qgis.Info)
dst_ds = gdal.Open(vrt)
ds = gdal.DEMProcessing(terrarium_tile,
dst_ds,
'color-relief',
colorFilename=color_ramp_file,
format="VRT",
addAlpha=True)
sleep(0.05)
dst_ds = None
QgsMessageLog.logMessage('Created vrt in terrarium format', CATEGORY,
Qgis.Info)
input_file = terrarium_tile
QgsMessageLog.logMessage('Input file is {inp}'.format(inp=input_file),
CATEGORY, Qgis.Info)
dst_ds = gdal.Open(input_file, gdal.GA_ReadOnly)
QgsMessageLog.logMessage('Calculating bounds', CATEGORY, Qgis.Info)
diff = (20037508.3427892439067364 +
20037508.3427892439067364) / 2**zoom
x_diff = diff
y_diff = diff
ulx, xres, xskew, uly, yskew, yres = dst_ds.GetGeoTransform()
#lrx = ulx + (dst_ds.RasterXSize * xres)
#lry = uly + (dst_ds.RasterYSize * yres)
info = gdal.Info(dst_ds, format='json')
ulx, uly = info['cornerCoordinates']['upperLeft'][0:2]
lrx, lry = info['cornerCoordinates']['lowerRight'][0:2]
wgs_minx, wgs_minz = info['wgs84Extent']['coordinates'][0][1][0:2]
wgs_maxx, wgs_maxz = info['wgs84Extent']['coordinates'][0][3][0:2]
QgsMessageLog.logMessage(
'The dataset bounds are (in WGS84 [EPSG:4326]), minX: {minX}, minZ: {minZ}, maxX: {maxX}, maxZ: {maxZ}'
.format(minX=str(wgs_minx),
minZ=str(wgs_minz),
maxX=str(wgs_maxx),
maxZ=str(wgs_maxz)), CATEGORY, Qgis.Info)
QgsMessageLog.logMessage(
'Use this info for following step F in Part two: Generating/using your dataset',
CATEGORY, Qgis.Info)
base_x = 0
base_y = 0
start_tile_x = base_x
start_tile_y = base_y
min_x = -20037508.3427892439067364
max_y = 20037508.3427892439067364
x_tiles = 0
y_tiles = 0
# Find min lot
lon = min_x
while lon <= 20037508.3427892439067364:
if lon >= ulx:
break
start_tile_x += 1
min_x = lon
lon += x_diff
# Find max lat
lat = max_y
while lat >= -20037508.3427892550826073:
if lat <= uly:
break
start_tile_y += 1
max_y = lat
lat -= y_diff
# Find how many lon tiles to make
lon = min_x
while lon < lrx:
x_tiles += 1
lon += x_diff
# Find how many lat tiles to make
lat = max_y
while lat >= lry:
y_tiles += 1
lat -= y_diff
if start_tile_x > 0:
start_tile_x -= 1
if start_tile_y > 0:
start_tile_y -= 1
QgsMessageLog.logMessage(
'Start tile: {tx} ({mlon}), {ty} ({mxlat}), Tiles to generate: {xt} (Width: {xtx} Height: {xty})'
.format(tx=start_tile_x,
mlon=min_x,
ty=start_tile_y,
mxlat=max_y,
xt=((x_tiles + 1) * (y_tiles + 1)),
xtx=x_tiles,
xty=y_tiles), CATEGORY, Qgis.Info)
QgsMessageLog.logMessage('Creating output folders', CATEGORY,
Qgis.Info)
mx = start_tile_x + x_tiles
zoom_folder = ''
temp_folder = os.path.join(source_folder, 'TEMP')
if not ftp_upload:
zoom_folder = os.path.join(output_directory,
str(zoom)).replace("\\", "/")
else:
if os.path.isdir(temp_folder):
shutil.rmtree(temp_folder)
os.mkdir(temp_folder)
zoom_folder = os.path.join(temp_folder,
str(zoom)).replace("\\", "/")
if not os.path.isdir(zoom_folder):
os.mkdir(zoom_folder)
for x in range(start_tile_x, mx):
folderx = os.path.join(zoom_folder, str(x)).replace("\\", "/")
if not os.path.isdir(folderx):
os.mkdir(folderx)
zip_file = os.path.join(zoom_folder,
'RenderedDataset.zip').replace("\\", "/")
rd_file = None
if ftp_upload:
if ftp_one_file:
rd_file = zipfile.ZipFile(zip_file, 'w', zipfile.ZIP_DEFLATED)
else:
QgsMessageLog.logMessage(
'Creating output folders on target ftp server', CATEGORY,
Qgis.Info)
ftp = None
if ftp_s:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect(ftp_upload_url, ftp_upload_port)
if ftp_user is None or ftp_password is None:
ftp.login()
else:
ftp.login(user=ftp_user, passwd=ftp_password)
if ftp_upload_folder is not None:
ftp.cwd(ftp_upload_folder)
if not isFTPDir(ftp, str(zoom)):
ftp.mkd(str(zoom))
ftp.cwd(str(zoom))
for x in range(start_tile_x, mx):
ftp.mkd(str(x))
ftp.quit()
sleep(0.01)
QgsMessageLog.logMessage('Created {ct} folders'.format(ct=x_tiles),
CATEGORY, Qgis.Info)
tiled = []
# Tile dataset
sub_min_x = min_x
sub_max_x = min_x + x_diff
for x in range(start_tile_x, start_tile_x + x_tiles):
sub_min_y = max_y - y_diff
sub_max_y = max_y
for y in range(start_tile_y, start_tile_y + y_tiles):
tiled.append(
Tile(dst_ds,
str(x),
str(y),
sub_min_x,
sub_max_y,
sub_max_x,
sub_min_y,
ulx,
xres,
uly,
yres,
querysize=1024))
sub_min_y -= y_diff
sub_max_y -= y_diff
sub_min_x += x_diff
sub_max_x += x_diff
job = Job(zoom_folder, str(zoom), input_file, getBands(dst_ds),
resampling_algorithm, ftp_upload, ftp_one_file, ftp_s,
ftp_upload_url, ftp_upload_port, ftp_upload_folder, ftp_user,
ftp_password)
dst_ds = None
# Tile the dataset
realtiles = 0
if cpu_count == 1:
QgsMessageLog.logMessage(
'Started tilling vrt in singlethread mode', CATEGORY,
Qgis.Info)
rt = 0
cf = len(tiled)
for tile in tiled:
res = tileVrt(job, tile)
if res is not None:
if job.ftpUpload and job.ftpOnefile:
addToZIP(job, res, rd_file)
realtiles += 1
rt += 1
if job.ftpUpload and job.ftpOnefile:
task.setProgress(
max(0, min(int(((rt * 40) / cf) + 20), 100)))
else:
task.setProgress(
max(0, min(int(((rt * 80) / cf) + 20), 100)))
if getattr(localThread, 'ds', None):
del localThread.ds
else:
QgsMessageLog.logMessage(
'Started tilling vrt in multithread mode with {count} threads'.
format(count=cpu_count), CATEGORY, Qgis.Info)
gdal_cache_max = gdal.GetCacheMax()
gdal_cache_max_per_process = max(
1024 * 1024, math.floor(gdal_cache_max / cpu_count))
setCacheMax(gdal_cache_max_per_process)
tpe = concurrent.futures.ThreadPoolExecutor(max_workers=cpu_count)
tm = tpe.map(tileVrt, repeat(job), tiled)
rt = 0
cf = len(tiled)
realtiles = 0
for res in tm:
if res is not None:
if job.ftpUpload and job.ftpOnefile:
addToZIP(job, res, rd_file)
realtiles += 1
rt += 1
if job.ftpUpload and job.ftpOnefile:
task.setProgress(
max(0, min(int(((rt * 40) / cf) + 20), 100)))
else:
task.setProgress(
max(0, min(int(((rt * 80) / cf) + 20), 100)))
setCacheMax(gdal_cache_max)
if job.ftpUpload and job.ftpOnefile:
rd_file.close()
totalSize = os.path.getsize(zip_file)
QgsMessageLog.logMessage(
'Starting uploading renderned archive ({size} GB) out of {count} tiles'
.format(count=realtiles,
size=str(round(totalSize * 9.3132257461548E-10,
4))), CATEGORY, Qgis.Info)
ftp = None
if job.ftpS:
ftp = FTP_TLS()
else:
ftp = FTP()
ftp.connect(job.ftpUrl, job.ftpPort)
if job.ftpUser is None or job.ftpPassword is None:
ftp.login()
else:
ftp.login(user=job.ftpUser, passwd=job.ftpPassword)
if job.ftpFolder is not None:
ftp.cwd(job.ftpFolder)
ftapr = FTPArchiveProgress(int(totalSize), task)
with open(zip_file, 'rb') as rdf_file:
ftp.storbinary('STOR RenderedDataset.zip',
rdf_file,
blocksize=1024,
callback=ftapr.updateProgress)
ftp.quit()
QgsMessageLog.logMessage(
'Uploaded rendered dataset archive RenderedDataset.zip with {count} tiles. You can now unzip it.'
.format(count=realtiles), CATEGORY, Qgis.Info)
elif job.ftpUpload:
QgsMessageLog.logMessage(
'Tiled and uploaded dataset with {count} tiles to ftp server'.
format(count=realtiles), CATEGORY, Qgis.Info)
else:
QgsMessageLog.logMessage(
'Tiled dataset with {count} tiles'.format(count=realtiles),
CATEGORY, Qgis.Info)
# Clean up
if cleanup:
os.remove(org_vrt)
os.remove(vrt)
os.remove(color_ramp_file)
os.remove(terrarium_tile)
if job.ftpUpload:
shutil.rmtree(temp_folder)
QgsMessageLog.logMessage('Cleaned up temp files', CATEGORY,
Qgis.Info)
return [realtiles, time_start]
except Exception as e:
QgsMessageLog.logMessage('Error: ' + str(e), CATEGORY, Qgis.Info)
return None
def __init__(self, parent: QtWidgets.QWidget=None):
super().__init__(parent)
self.layout = QtWidgets.QVBoxLayout()
self.setLayout(self.layout)
# Top
self.top_layout = QtWidgets.QHBoxLayout()
self.layout.addLayout(self.top_layout)
self.version_groupbox = QtWidgets.QGroupBox(self)
self.version_groupbox.setTitle("Version")
self.top_layout.addWidget(self.version_groupbox)
self.version_layout = QtWidgets.QVBoxLayout()
self.version_groupbox.setLayout(self.version_layout)
self.version_release = QtWidgets.QLabel("Release: %s" % gdal.VersionInfo('RELEASE_NAME'))
self.version_layout.addWidget(self.version_release)
self.version_date = QtWidgets.QLabel("Date: %s" % gdal.VersionInfo('RELEASE_DATE'))
self.version_layout.addWidget(self.version_date)
self.cache_groupbox = QtWidgets.QGroupBox(self)
self.cache_groupbox.setTitle("Cache")
self.top_layout.addWidget(self.cache_groupbox)
self.cache_layout = QtWidgets.QVBoxLayout()
self.cache_groupbox.setLayout(self.cache_layout)
self.cache_max = QtWidgets.QLabel("Max: %.1f MB" % (gdal.GetCacheMax() / (1024 * 1024),))
self.cache_layout.addWidget(self.cache_max)
self.cache_used = QtWidgets.QLabel("Used: %.1f MB" % (gdal.GetCacheUsed() / (1024 * 1024),))
self.cache_layout.addWidget(self.cache_used)
# Bottom
self.bottom_layout = QtWidgets.QHBoxLayout()
self.layout.addLayout(self.bottom_layout)
self.drivers_groupbox = QtWidgets.QGroupBox(self)
self.drivers_groupbox.setTitle("Drivers")
self.bottom_layout.addWidget(self.drivers_groupbox)
self.drivers_layout = QtWidgets.QVBoxLayout()
self.drivers_groupbox.setLayout(self.drivers_layout)
self.drivers_count = QtWidgets.QLabel("Number of drivers: %d" % (gdal.GetDriverCount(),))
self.drivers_layout.addWidget(self.drivers_count)
self.table = QtWidgets.QTableWidget(self)
self.table.setObjectName("GdalDriversTable")
self.table.setColumnCount(8)
self.table.setRowCount(0)
item = QtWidgets.QTableWidgetItem()
item.setText("Short Name")
self.table.setHorizontalHeaderItem(0, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Long Name")
self.table.setHorizontalHeaderItem(1, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Help Page")
self.table.setHorizontalHeaderItem(2, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Mime Type")
self.table.setHorizontalHeaderItem(3, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Extensions")
self.table.setHorizontalHeaderItem(4, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Data Types")
self.table.setHorizontalHeaderItem(5, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Creation Options")
self.table.setHorizontalHeaderItem(6, item)
item = QtWidgets.QTableWidgetItem()
item.setText("Metadata")
self.table.setHorizontalHeaderItem(7, item)
self.table.horizontalHeader().setStretchLastSection(True)
self.table.horizontalHeader().setVisible(True)
self.table.verticalHeader().setVisible(False)
self.table.setRowCount(gdal.GetDriverCount())
for row in range(gdal.GetDriverCount()):
driver = gdal.GetDriver(row)
self.table.setItem(row, 0, QtWidgets.QTableWidgetItem(driver.ShortName))
self.table.setItem(row, 1, QtWidgets.QTableWidgetItem(driver.LongName))
self.table.setItem(row, 2, QtWidgets.QTableWidgetItem(driver.HelpTopic))
metadata = driver.GetMetadata()
if metadata:
self.table.setItem(row, 3, QtWidgets.QTableWidgetItem(str(metadata.pop(gdal.DMD_MIMETYPE, ''))))
self.table.setItem(row, 4, QtWidgets.QTableWidgetItem(str(metadata.pop(gdal.DMD_EXTENSION, ''))))
self.table.setItem(row, 5,
QtWidgets.QTableWidgetItem(str(metadata.pop(gdal.DMD_CREATIONDATATYPES, ''))))
full_data = metadata.pop(gdal.DMD_CREATIONOPTIONLIST, '')
if full_data:
data = full_data[:10] + "[..]"
else:
data = full_data
table_item = QtWidgets.QTableWidgetItem(data)
table_item.setToolTip(full_data)
self.table.setItem(row, 6, table_item)
metadata_list = ['%s=%s' % (k, v) for k, v in metadata.items()]
metadata = ", ".join(metadata_list)[:10] + "[..]"
table_item = QtWidgets.QTableWidgetItem(metadata)
table_item.setToolTip('\n'.join(metadata_list))
self.table.setItem(row, 7, table_item)
self.table.horizontalHeader().resizeSections(QtWidgets.QHeaderView.ResizeToContents)
self.table.setSortingEnabled(True)
self.table.sortItems(0, QtCore.Qt.AscendingOrder)
self.drivers_layout.addWidget(self.table)
def __init__(self, arguments):
"""Constructor function - initialization"""
try:
subprocess.call(["gdalbuildvrt", "--help"])
except:
print "gdalbuildvrt is required to run gdal2cesium in multi inumpyuts mode"
exit(1)
self.stopped = False
self.multi_suffix = ''
self.inumpyut = None
self.default_base_output = 'tiles'
self.min_tile_tz = None
self.inumpyuts_data = {}
self.inumpyuts_files_or_vrt = []
self.vrts = {}
self.tminmax = None
self.zoom_resolutions = {}
self.tminz = None
self.tmaxz = None
gdal.AllRegister()
self.mem_drv = gdal.GetDriverByName('MEM')
self.geodetic = GlobalGeodetic()
# Tile format
self.tilesize = 64
self.tileext = 'terrain'
self.epsg4326 = "EPSG:4326"
self.tilelayer = None
self.scaledquery = True
# How big should be query window be for scaling down
# Later on reset according the chosen resampling algorightm
self.querysize = 4 * self.tilesize
# pixel overlap between tiles according to Ceiusm heightmap format
self.extrapixels = 0
# RUN THE ARGUMENT PARSER:
self.optparse_init()
self.options, self.args = self.parser.parse_args(args=arguments)
self.options.srcnodata = None
if not self.args:
self.error("No inumpyut file specified")
# POSTPROCESSING OF PARSED ARGUMENTS:
# Workaround for old versions of GDAL
try:
if (self.options.verbose and self.options.resampling
== 'near') or gdal.TermProgress_nocb:
pass
except:
self.error(
"This version of GDAL is not supported. Please upgrade to 1.6+."
)
#,"You can try run crippled version of gdal2tiles with parameters: -v -r 'near'")
self.inumpyuts = [i for i in self.args]
# Default values for not given options
if self.options.output:
self.output = self.options.output
else:
if len(self.inumpyuts) > 0:
self.multi_suffix = '_multi'
self.output = os.path.join(
self.default_base_output,
os.path.basename(self.inumpyuts[0]).split('.')[0] +
self.multi_suffix)
self.options.title = os.path.basename(self.inumpyuts[0] +
self.multi_suffix)
self.tmpoutput = os.path.join(self.output, 'tmp')
# Supported options
self.resampling = None
if self.options.resampling == 'average':
try:
if gdal.RegenerateOverview:
pass
except:
self.error(
"'average' resampling algorithm is not available.",
"Please use -r 'near' argument or upgrade to newer version of GDAL."
)
elif self.options.resampling == 'near':
self.resampling = gdal.GRA_NearestNeighbour
self.querysize = self.tilesize
elif self.options.resampling == 'bilinear':
self.resampling = gdal.GRA_Bilinear
self.querysize = self.tilesize * 2
elif self.options.resampling == 'cubic':
self.resampling = gdal.GRA_Cubic
elif self.options.resampling == 'cubicspline':
self.resampling = gdal.GRA_CubicSpline
elif self.options.resampling == 'lanczos':
self.resampling = gdal.GRA_Lanczos
# User specified zoom levels
self.user_tminz = None
self.user_tmaxz = None
if self.options.zoom:
minmax = self.options.zoom.split('-', 1)
minmax.extend([''])
min, max = minmax[:2]
self.user_tminz = int(min)
if max:
self.user_tmaxz = int(max)
else:
self.user_tmaxz = int(min)
# Output the results
if self.options.verbose:
print("Options:", self.options)
print("Inumpyut:", self.inumpyuts[0] + self.multi_suffix)
print("Output:", self.output)
print("Cache: %s MB" % (gdal.GetCacheMax() / 1024 / 1024))
print('')
def main(cmdargs):
rank = 0 # residual from MPI version, here is single-process version, rank of the process is simply 0.
hcr_rasters = cmdargs.input_rasters
# check the input rasters and calcuate the number of tiles (docs)
hcr_datasets = [
gdal.Open(rfile, gdal.GA_ReadOnly) for rfile in hcr_rasters
]
hcr_raster_xsize, hcr_raster_ysize = hcr_datasets[
0].RasterXSize, hcr_datasets[0].RasterYSize
if not np.all([ds.RasterXSize == hcr_raster_xsize for ds in hcr_datasets]):
raise RuntimeError("Input rasters have different X dimensions!")
if not np.all([ds.RasterYSize == hcr_raster_ysize for ds in hcr_datasets]):
raise RuntimeError("Input rasters have different Y dimensions!")
hcr_bands = [
ds.GetRasterBand(i) for ds, i in zip(hcr_datasets, cmdargs.bands)
]
cv_split = cmdargs.cv_split
hcr_test_ds = None
hcr_test_bd = None
if cmdargs.test is not None:
hcr_test_ds = gdal.Open(cmdargs.test, gdal.GA_ReadOnly)
hcr_test_bd = hcr_test_ds.GetRasterBand(1)
tile_xsize = cmdargs.doc_tile_size
tile_ysize = cmdargs.doc_tile_size
ndocs_batch = cmdargs.n_batch_docs
# Set up dict of class codes within raster to common class codes/names across raster
class_code2vocab = []
for csvfname in cmdargs.class2vocab:
arr = np.loadtxt(csvfname, dtype=int, delimiter=',')
class_code2vocab.append({row[0]: row[1] for row in arr})
class_errmat = None
use_errmat = False
if cmdargs.error_matrix is not None:
class_errmat = []
for csvfname in cmdargs.error_matrix:
tmp = pd.read_csv(csvfname, index_col=0, header=0)
tmp.columns = [int(val) for val in tmp.columns]
class_errmat.append(tmp)
use_errmat = True
if cmdargs.N_factor is None:
if class_errmat is not None:
tmpval = np.min(
[np.min(em.values[em.values != 0]) for em in class_errmat])
tmpval = 10**(len(str(int(1. / tmpval))) + 1)
N_factor = 1000 if 1000 > tmpval else tmpval
else:
N_factor = 1000
else:
N_factor = cmdargs.N_factor
if cmdargs.doc_topic_prior is None:
doc_topic_prior = 1. / cmdargs.n_topics
else:
doc_topic_prior = cmdargs.doc_topic_prior
vocab = set(
list(itertools.chain(*[c2v.values() for c2v in class_code2vocab])))
if cmdargs.topic_word_prior is None:
topic_word_prior = 1. / len(vocab)
else:
topic_word_prior = cmdargs.topic_word_prior
vocab_creation = cmdargs.vocab_creation
hcr = HarmonizeClassRasters(class_code2vocab,
class_errmat,
vocab_creation=vocab_creation,
n_components=cmdargs.n_topics,
max_iter=1000,
evaluate_every=1,
perp_tol=1e-1,
n_jobs=cmdargs.n_jobs,
batch_size=ndocs_batch,
doc_topic_prior=doc_topic_prior,
topic_word_prior=topic_word_prior)
# Set up a look-up table (LUT) to store the occurrence counts, LDA scores,
# topic prob. estimates of all the combinations of input class labels from
# different rasters.
#
# MultiIndex: class legends of input rasters
# Columns: occurrence counts, LDA scores, topic probs., primary topic ID
# (the topic with the largest prob.)
index_list = [set(c2v.values()) for c2v in class_code2vocab]
hcr_lut_index = pd.MultiIndex.from_product(index_list)
nrows = len(hcr_lut_index)
ncols = 1 + 1 + 1 + 1 + cmdargs.n_topics
prob_topic_colnames = [
"prob_topic_{0:d}".format(i + 1) for i in range(cmdargs.n_topics)
]
hcr_lut = pd.DataFrame(
np.zeros((nrows, ncols)),
index=hcr_lut_index,
columns=["total_npix", "test_npix", "lda_score", "primary_topic_id"] +
prob_topic_colnames)
hcr_lut_values = hcr_lut.values.copy()
ntiles_x = np.ceil(hcr_raster_xsize / tile_xsize).astype(np.int)
ntiles_y = np.ceil(hcr_raster_ysize / tile_ysize).astype(np.int)
dw_mat = np.zeros((ndocs_batch, len(hcr.vocab)))
word_count = np.zeros(len(hcr.vocab))
doc_idx = 0
tmp = np.argmax([
np.dtype(gdal_array.GDALTypeCodeToNumericTypeCode(
bd.DataType)).itemsize for bd in hcr_bands
])
img_dtype = gdal_array.GDALTypeCodeToNumericTypeCode(
hcr_bands[tmp].DataType)
nodata = np.iinfo(img_dtype).max
# prf = cProfile.Profile()
# prf.enable()
ndigits = max(len(str(ntiles_x)), len(str(ntiles_y)))
progress_tot = ntiles_x * ntiles_y
progress_pct = 10
progress_frc = int(progress_pct / 100. * progress_tot)
if progress_frc == 0:
progress_frc = 1
progress_pct = int(progress_frc / float(progress_tot) * 100)
progress_cnt = 0
progress_npct = 0
if cv_split is not None:
logger.info(
"Process {0:d}: Search non-empty tiles for cross-validation sampling ..."
.format(rank))
valid_tiles = []
for iby in range(ntiles_y):
for ibx in range(ntiles_x):
xoff, yoff = tile_xsize * ibx, tile_ysize * iby
win_xsize = tile_xsize if ibx < ntiles_x - 1 else hcr_raster_xsize - xoff
win_ysize = tile_ysize if iby < ntiles_y - 1 else hcr_raster_ysize - yoff
mb_img = [
bd.ReadAsArray(xoff, yoff, win_xsize,
win_ysize).astype(img_dtype)
for bd in hcr_bands
]
valid_flag = False
for ib, img in enumerate(mb_img):
tmp = set(hcr.class_code2vocab[ib].index.values)
tmp_len = len(tmp)
if len(tmp - set(np.unique(img))) < tmp_len:
valid_flag = True
break
if valid_flag:
valid_tiles.append(
np.ravel_multi_index((iby, ibx), (ntiles_y, ntiles_x)))
progress_cnt += 1
if progress_cnt % progress_frc == 0:
progress_npct += progress_pct
if progress_npct <= 100:
logger.info(
"Process {1:d}: Finish searching non-empty tiles {0:d}%"
.format(progress_npct, rank))
logger.info(
"Process {1:d}: Finish searching non-empty tiles, {0:d} non-empty tiles found"
.format(len(valid_tiles), rank))
# Random sampling for each CV test
cv_test_tiles = []
cv_hcr_list = []
cv_dw_mat_list = [np.zeros((ndocs_batch, len(hcr.vocab)))] * cv_split
cv_doc_idx_list = [0] * cv_split
cv_score_sum_list = [0] * cv_split
cv_perplexity_list = [0] * cv_split
cv_size = int(len(valid_tiles) / cv_split)
for i in range(cv_split):
tmp = np.random.choice(valid_tiles, size=cv_size, replace=False)
tmprow, tmpcol = np.unravel_index(tmp, (ntiles_y, ntiles_x))
tmp = spspa.coo_matrix(
(np.ones_like(tmprow, dtype=np.bool), (tmprow, tmpcol)),
shape=(ntiles_y, ntiles_x),
dtype=np.bool)
cv_test_tiles.append(tmp.tocsr())
cv_hcr_list.append(
HarmonizeClassRasters(class_code2vocab,
class_errmat,
vocab_creation=vocab_creation,
n_components=cmdargs.n_topics,
max_iter=1000,
evaluate_every=1,
perp_tol=1e-1,
n_jobs=cmdargs.n_jobs,
batch_size=ndocs_batch,
doc_topic_prior=doc_topic_prior,
topic_word_prior=topic_word_prior))
progress_cnt = 0
progress_npct = 0
logger.info(
"Process {0:d}: Start building document-word matrix from input classification rasters ..."
.format(rank))
for iby in range(ntiles_y):
for ibx in range(ntiles_x):
xoff, yoff = tile_xsize * ibx, tile_ysize * iby
win_xsize = tile_xsize if ibx < ntiles_x - 1 else hcr_raster_xsize - xoff
win_ysize = tile_ysize if iby < ntiles_y - 1 else hcr_raster_ysize - yoff
mb_img = [
bd.ReadAsArray(xoff, yoff, win_xsize,
win_ysize).astype(img_dtype) for bd in hcr_bands
]
# mask out invalid pixels
img_mask = np.zeros_like(mb_img[0], dtype=np.bool)
for ib, img in enumerate(mb_img):
tmp_mask = np.ones_like(img, dtype=np.bool)
for v in hcr.class_code2vocab[ib].index:
tmp_mask = np.logical_and(tmp_mask, img != v)
img_mask = np.logical_or(img_mask, tmp_mask)
tmp_mask = np.logical_not(img_mask)
to_do_flag = tmp_mask.sum() > 0
if to_do_flag:
uq_idx, uq_cnt = np.unique(np.array([
hcr._translateArray(img[tmp_mask], hcr.class_code2vocab[i])
for i, img in enumerate(mb_img)
]),
axis=1,
return_counts=True)
hcr_lut.loc[[*zip(*uq_idx.tolist())], "total_npix"] += uq_cnt
if to_do_flag and hcr_test_bd is not None:
test_mask = hcr_test_bd.ReadAsArray(xoff, yoff, win_xsize,
win_ysize)
# img_mask = np.logical_or(img_mask, test_mask==1)
tmp_mask = np.logical_not(
np.logical_or(img_mask, test_mask == 1))
# to_do_flag = tmp_mask.sum() > 0
if tmp_mask.sum() > 0: # to_do_flag:
uq_idx, uq_cnt = np.unique(np.array([
hcr._translateArray(img[tmp_mask],
hcr.class_code2vocab[i])
for i, img in enumerate(mb_img)
]),
axis=1,
return_counts=True)
hcr_lut.loc[[*zip(*uq_idx.tolist())],
"test_npix"] += uq_cnt
if to_do_flag:
for img in mb_img:
img[img_mask] = nodata
dw_mat[doc_idx, :] = hcr.genDocWordFromArray(
np.dstack(mb_img),
use_errmat=use_errmat,
N_factor=N_factor)
if cv_split is not None:
for i in range(cv_split):
if not cv_test_tiles[i][iby, ibx]:
cv_dw_mat_list[i][cv_doc_idx_list[i], :] = dw_mat[
doc_idx, :]
cv_doc_idx_list[i] += 1
doc_idx += 1
if doc_idx == ndocs_batch:
word_count += np.sum(dw_mat[0:doc_idx, :], axis=0)
hcr.fitTopicModel(dw_mat[0:doc_idx, :])
doc_idx = 0
if cv_split is not None:
for i in range(cv_split):
if cv_doc_idx_list[i] == ndocs_batch:
cv_hcr_list[i].fitTopicModel(
cv_dw_mat_list[i][0:cv_doc_idx_list[i], :])
cv_doc_idx_list[i] = 0
progress_cnt += 1
if progress_cnt % progress_frc == 0:
progress_npct += progress_pct
if progress_npct <= 100:
logger.info(
"Process {1:d}: Finish reading input rasters and building document-word matrix {0:d}%"
.format(progress_npct, rank))
# if doc_idx > ndocs_batch-2:
# break
if doc_idx > 0:
word_count += np.sum(dw_mat[0:doc_idx, :], axis=0)
hcr.fitTopicModel(dw_mat[0:doc_idx, :])
doc_idx = 0
tmp = np.where(word_count == 0)[0]
if len(tmp) > 0:
logger.warning(
"Process {0:d}: Some classes never appeared in the input rasters: \n"
.format(rank) + str(np.array(list(hcr.vocab))[tmp]))
if cv_split is not None:
for i in range(cv_split):
if cv_doc_idx_list[i] > 0:
cv_hcr_list[i].fitTopicModel(
cv_dw_mat_list[i][0:cv_doc_idx_list[i], :])
cv_doc_idx_list[i] = 0
# prf.disable()
# prf.print_stats(sort="time")
# The following only on master node, no parallelization.
logger.info("Process {0:d}: Finish fitting LDA model ...".format(rank))
# Estimate the topic probs. for all the combinations of the input class
# labels and their LDA scores.
logger.info(
"Process {0:d}: Start building LUT of the topics and LDA score per combination of input class legends ..."
.format(rank))
# prf = cProfile.Profile()
# prf.enable()
dw_mat = np.array([
hcr.genDocWordFromArray(np.array(idx)[np.newaxis, np.newaxis, :],
use_errmat=use_errmat,
N_factor=N_factor) for idx in hcr_lut.index
])
dt_dist = hcr.estDocTopicDist(dw_mat)
hcr_lut.loc[:, prob_topic_colnames] = dt_dist
hcr_lut.loc[:, "primary_topic_id"] = np.argmax(dt_dist, axis=1) + 1
for i, idx in enumerate(hcr_lut.index):
hcr_lut.loc[idx, "lda_score"] = hcr.lda.score(dw_mat[[i], :])
# prf.disable()
# prf.print_stats(sort="time")
logger.info(
"Process {0:d}: Finish building LUT of the topics and LDA score per combination of input class legends ..."
.format(rank))
if hcr_test_bd is not None:
# Calculate the perplexity and score over the test pixels
logger.info(
"Process {0:d}: Start estimating perplexity and score over test pixels ..."
.format(rank))
# prf = cProfile.Profile()
# prf.enable()
score_sum = np.sum(hcr_lut["lda_score"] * hcr_lut["test_npix"])
perplexity = np.exp(
-1 * np.sum(hcr_lut["lda_score"] * hcr_lut["test_npix"]) /
(N_factor * np.sum(hcr_lut["test_npix"])))
else:
score_sum = np.nan
perplexity = np.nan
if cv_split is not None:
logger.info(
"Process {0:d}: Start estimating perplexity and score over test tiles in the cross validation ..."
.format(rank))
progress_npct = 0
progress_cnt = 0
for iby in range(ntiles_y):
for ibx in range(ntiles_x):
if np.any(
[cv_test_tiles[i][iby, ibx] for i in range(cv_split)]):
xoff, yoff = tile_xsize * ibx, tile_ysize * iby
win_xsize = tile_xsize if ibx < ntiles_x - 1 else hcr_raster_xsize - xoff
win_ysize = tile_ysize if iby < ntiles_y - 1 else hcr_raster_ysize - yoff
mb_img = [
bd.ReadAsArray(xoff, yoff, win_xsize,
win_ysize).astype(img_dtype)
for bd in hcr_bands
]
# mask out invalid pixels
img_mask = np.zeros_like(mb_img[0], dtype=np.bool)
for ib, img in enumerate(mb_img):
tmp_mask = np.ones_like(img, dtype=np.bool)
for v in hcr.class_code2vocab[ib].index:
tmp_mask = np.logical_and(tmp_mask, img != v)
img_mask = np.logical_or(img_mask, tmp_mask)
tmp_mask = np.logical_not(img_mask)
to_do_flag = tmp_mask.sum() > 0
if to_do_flag:
for img in mb_img:
img[img_mask] = nodata
dw_mat = hcr.genDocWordFromArray(np.dstack(mb_img),
use_errmat=use_errmat,
N_factor=N_factor)
for i in range(cv_split):
if cv_test_tiles[i][iby, ibx]:
cv_score_sum_list[i] += cv_hcr_list[
i].lda.score(dw_mat[np.newaxis, :])
cv_perplexity_list[i] += np.log(
cv_hcr_list[i].lda.perplexity(
dw_mat[np.newaxis, :]))
progress_cnt += 1
if progress_cnt % progress_frc == 0:
progress_npct += progress_pct
if progress_npct <= 100:
logger.info(
"Process {1:d}: Finish inference of test tiles in cross validation {0:d}%"
.format(progress_npct, rank))
logger.info(
"Process {4:d}: doc_size = {0:d}, n_topics = {1:d}, n_factor = {2:d}, cv_size = {3:d}, cross validation report: "
.format(cmdargs.doc_tile_size, cmdargs.n_topics, N_factor, cv_size,
rank))
logger.info(
"Process {0:d}: cv_seq, perplexity_cv, score_cv".format(rank))
for i in range(cv_split):
logger.info("Process {3:d}: {0:d}, {1:e}, {2:e}".format(
i, np.exp(cv_perplexity_list[i] / cv_size),
cv_score_sum_list[i] / cv_size, rank))
logger.info("Process {3:d}: {0:s}, {1:e}, {2:e}".format(
"mean", np.exp(np.mean(cv_perplexity_list) / cv_size),
np.mean(cv_score_sum_list) / cv_size, rank))
# prf.disable()
# prf.print_stats(sort="time")
logger.info(
"Process {5:d}: doc_size = {2:d}, n_topics = {3:d}, n_factor = {4:d}, perplexity_test_pixels = {0:e}, score_test_pixels = {1:e}"
.format(perplexity, score_sum, cmdargs.doc_tile_size, cmdargs.n_topics,
N_factor, rank))
joblib.dump(hcr.lda, cmdargs.out_model)
logger.info("Process {1:d}: trained LDA model saved to {0:s}".format(
cmdargs.out_model, rank))
vocab_joblib = ".".join(
cmdargs.out_model.split('.')[0:-1]) + "_vocab.joblib"
joblib.dump(hcr.vocab, vocab_joblib)
logger.info(
"Process {1:d}: vocabulary list of trained LDA model saved to {0:s}".
format(vocab_joblib, rank))
if cmdargs.out_topic_word is not None:
pd.DataFrame(hcr.getTopicWordDist(),
columns=hcr._dw.index).to_csv(cmdargs.out_topic_word)
logger.info(
"Process {1:d}: Topic-word distribution written to {0:s}".format(
cmdargs.out_topic_word, rank))
if cmdargs.out_lut is not None:
hcr_lut.to_csv(cmdargs.out_lut)
logger.info(
"Process {1:d}: Look-up table of LDA model written to {0:s}".
format(cmdargs.out_lut, rank))
if cmdargs.out_class is not None:
class_raster = cmdargs.out_class
class_format = cmdargs.out_format
prob_raster = cmdargs.out_prob
prob_format = cmdargs.out_format
class_nodata = 0
prob_nodata = 0
if hcr.lda.n_components < np.iinfo(np.int8).max:
out_type = np.int8
elif hcr.lda.n_components < np.iinfo(np.int16).max:
out_type = np.int16
elif hcr.lda.n_components < np.iinfo(np.int32).max:
out_type = np.int32
elif hcr.lda.n_components < np.iinfo(np.int64).max:
out_type = np.int64
else:
out_type = np.float
prob_type = np.float32
block_xsize, block_ysize = hcr_bands[0].GetBlockSize()
hcr_type_size = np.max([
np.dtype(gdal_array.GDALTypeCodeToNumericTypeCode(
bd.DataType)).itemsize for bd in hcr_bands
])
tmpn = int(gdal.GetCacheMax() /
(block_xsize * block_ysize * hcr_type_size))
if tmpn > int(hcr_raster_ysize / block_ysize):
tmpn = int(hcr_raster_ysize / block_ysize)
if tmpn > 1:
block_ysize = tmpn * block_ysize
nblocks_x, nblocks_y = np.ceil(
hcr_raster_xsize / block_xsize).astype(int), np.ceil(
hcr_raster_ysize / block_ysize).astype(int)
block_meta_data = np.zeros(4, dtype=np.int)
logger.info(
"Process {0:d}: Start estimating and writing pixel topics (harmonized class) ..."
.format(rank))
logger.info(
"Process {4:d}: n_blocks_x = {0:d}, n_blocks_y = {1:d}, block_xsize = {2:d}, block_ysize = {3:d}"
.format(nblocks_x, nblocks_y, block_xsize, block_ysize, rank))
# Use N-dimensional array to speed up LUT search
hcr_lut_class_arr = np.zeros(
[len(ilevel) for ilevel in hcr_lut.index.levels], dtype=out_type)
hcr_lut_prob_arr = np.zeros(
[len(ilevel)
for ilevel in hcr_lut.index.levels] + [len(prob_topic_colnames)],
dtype=prob_type)
# Put the LUT into the array
hcr_lut_class_arr[tuple(
hcr_lut.index.labels)] = hcr_lut.loc[:, "primary_topic_id"]
for i, coln in enumerate(prob_topic_colnames):
hcr_lut_prob_arr[tuple(hcr_lut.index.labels +
[[i] *
len(hcr_lut.index)])] = hcr_lut.loc[:,
coln]
class_driver = gdal.GetDriverByName(class_format)
class_ds = class_driver.Create(
class_raster, hcr_raster_xsize, hcr_raster_ysize, 1,
gdal_array.NumericTypeCodeToGDALTypeCode(out_type))
class_bd = class_ds.GetRasterBand(1)
if prob_format is not None:
prob_driver = gdal.GetDriverByName(prob_format)
else:
prob_driver = class_driver
if prob_raster is not None:
prob_ds = prob_driver.Create(
prob_raster, hcr_raster_xsize, hcr_raster_ysize,
hcr.lda.n_components,
gdal_array.NumericTypeCodeToGDALTypeCode(prob_type))
else:
prob_ds = None
progress_cnt = 0
progress_tot = nblocks_x * nblocks_y
progress_pct = 10
progress_frc = int(progress_pct / 100. * progress_tot)
if progress_frc == 0:
progress_frc = 1
progress_npct = 0
for iby in range(nblocks_y):
for ibx in range(nblocks_x):
xoff, yoff = ibx * block_xsize, iby * block_ysize
# win_xsize, win_ysize = hcr_bands[0].GetActualBlockSize(ibx, iby)
win_xsize = block_xsize if ibx < nblocks_x - 1 else hcr_raster_xsize - xoff
win_ysize = block_ysize if iby < nblocks_y - 1 else hcr_raster_ysize - yoff
block_meta_data[:] = [xoff, yoff, win_xsize, win_ysize]
# On slave processes,
mb_img = [
bd.ReadAsArray(xoff, yoff, win_xsize, win_ysize)
for bd in hcr_bands
]
# mask out invalid pixels
img_mask = np.zeros_like(mb_img[0], dtype=np.bool)
for ib, img in enumerate(mb_img):
tmp_mask = np.ones_like(img, dtype=np.bool)
for v in hcr.class_code2vocab[ib].index:
tmp_mask = np.logical_and(tmp_mask, img != v)
img_mask = np.logical_or(img_mask, tmp_mask)
img_mask = np.logical_not(img_mask)
to_do_flag = np.sum(img_mask) > 0
class_img = np.zeros_like(img_mask,
dtype=out_type) + class_nodata
if to_do_flag:
# Convert image of vocabulary class labels to image of indexes to these classes in each input raster
idx_img_list = []
for i, img in enumerate(mb_img):
tmp_idx = hcr._translateArray(img[img_mask],
hcr.class_code2vocab[i])
for idx, ilevel in enumerate(hcr_lut.index.levels[i]):
tmp_idx[tmp_idx == ilevel] = idx
idx_img_list.append(tmp_idx)
class_img[img_mask] = hcr_lut_class_arr[tuple(
idx_img_list)]
if prob_raster is not None:
prob_img_list = [
np.zeros_like(img_mask, dtype=prob_type) + prob_nodata
for coln in prob_topic_colnames
]
if to_do_flag:
for i, coln in enumerate(prob_topic_colnames):
prob_img_list[i][img_mask] = hcr_lut_prob_arr[
tuple(idx_img_list +
[[i] * len(idx_img_list[0])])]
prob_img = np.dstack(prob_img_list)
class_bd.WriteArray(class_img, int(block_meta_data[0]),
int(block_meta_data[1]))
if prob_raster is not None:
for i in range(prob_img.shape[2]):
prob_ds.GetRasterBand(i + 1).WriteArray(
prob_img[:, :, i], int(block_meta_data[0]),
int(block_meta_data[1]))
progress_cnt += 1
if progress_cnt % progress_frc == 0:
if progress_frc == 1:
progress_npct = int(100 * progress_cnt / progress_tot)
else:
progress_npct += progress_pct
if progress_npct <= 100:
logger.info(
"Process {1:d}: Finish pixel inference and writing {0:d}%"
.format(progress_npct, rank))
class_ds.FlushCache()
prob_ds.FlushCache()
class_ds.SetGeoTransform(hcr_datasets[0].GetGeoTransform())
class_ds.SetProjection(hcr_datasets[0].GetProjectionRef())
prob_ds.SetGeoTransform(hcr_datasets[0].GetGeoTransform())
prob_ds.SetProjection(hcr_datasets[0].GetProjectionRef())
class_ds = None
prob_ds = None
# On both master and slave processes, close the raster files that GDAL has
# opened.
for i in range(len(hcr_datasets)):
hcr_datasets[i] = None
if cmdargs.test is not None:
hcr_test_ds = None
def mask_14():
src_ds = gdal.Open('data/byte.tif')
if src_ds is None:
gdaltest.post_reason('Failed to open test dataset.')
return 'fail'
drv = gdal.GetDriverByName('GTiff')
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'FALSE')
ds = drv.CreateCopy('tmp/byte_with_mask.tif', src_ds)
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'TRUE')
src_ds = None
# The only flag value supported for internal mask is GMF_PER_DATASET
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(0)
if ret == 0:
gdaltest.post_reason('Error expected')
return 'fail'
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(gdal.GMF_PER_DATASET)
if ret != 0:
gdaltest.post_reason('Creation failed')
return 'fail'
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
if cs != 0:
print(cs)
gdaltest.post_reason('Got wrong checksum for the mask (1)')
return 'fail'
ds.GetRasterBand(1).GetMaskBand().Fill(1)
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
if cs != 400:
print(cs)
gdaltest.post_reason('Got wrong checksum for the mask (2)')
return 'fail'
# This TIFF dataset has already an internal mask band
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.CreateMaskBand(gdal.GMF_PER_DATASET)
if ret == 0:
gdaltest.post_reason('Error expected')
return 'fail'
# This TIFF dataset has already an internal mask band
with gdaltest.error_handler():
with gdaltest.config_option('GDAL_TIFF_INTERNAL_MASK', 'YES'):
ret = ds.GetRasterBand(1).CreateMaskBand(gdal.GMF_PER_DATASET)
if ret == 0:
gdaltest.post_reason('Error expected')
return 'fail'
ds = None
try:
os.stat('tmp/byte_with_mask.tif.msk')
gdaltest.post_reason('tmp/byte_with_mask.tif.msk should not exist')
return 'fail'
except:
pass
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'FALSE')
ds = gdal.Open('tmp/byte_with_mask.tif')
if ds.GetRasterBand(1).GetMaskFlags() != gdal.GMF_PER_DATASET:
gdaltest.post_reason('wrong mask flags')
return 'fail'
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK_TO_8BIT', 'TRUE')
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
if cs != 400:
print(cs)
gdaltest.post_reason('Got wrong checksum for the mask (3)')
return 'fail'
# Test fix for #5884
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', 'YES')
old_val = gdal.GetCacheMax()
gdal.SetCacheMax(0)
out_ds = drv.CreateCopy('/vsimem/byte_with_mask.tif',
ds,
options=['COMPRESS=JPEG'])
gdal.SetConfigOption('GDAL_TIFF_INTERNAL_MASK', None)
gdal.SetCacheMax(old_val)
if out_ds.GetRasterBand(1).Checksum() == 0:
gdaltest.post_reason('failure')
return 'fail'
cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
if cs != 400:
print(cs)
gdaltest.post_reason('Got wrong checksum for the mask (4)')
return 'fail'
out_ds = None
drv.Delete('/vsimem/byte_with_mask.tif')
ds = None
drv.Delete('tmp/byte_with_mask.tif')
return 'success'
def __init__(self, arguments ):
"""Constructor function - initialization"""
self.stopped = False
self.input = None
self.output = None
# Tile format
self.tilesize = 256
self.tiledriver = 'PNG'
self.tileext = 'png'
# Should we read bigger window of the input raster and scale it down?
# Note: Modified leter by open_input()
# Not for 'near' resampling
# Not for Wavelet based drivers (JPEG2000, ECW, MrSID)
# Not for 'raster' profile
self.scaledquery = True
# How big should be query window be for scaling down
# Later on reset according the chosen resampling algorightm
self.querysize = 4 * self.tilesize
# Should we use Read on the input file for generating overview tiles?
# Note: Modified later by open_input()
# Otherwise the overview tiles are generated from existing underlying tiles
self.overviewquery = False
# RUN THE ARGUMENT PARSER:
self.optparse_init()
self.options, self.args = self.parser.parse_args(args=arguments)
if not self.args:
self.error("No input file specified")
# POSTPROCESSING OF PARSED ARGUMENTS:
# Workaround for old versions of GDAL
try:
if (self.options.verbose and self.options.resampling == 'near') or gdal.TermProgress_nocb:
pass
except:
self.error("This version of GDAL is not supported. Please upgrade to 1.6+.")
#,"You can try run crippled version of gdal2tiles with parameters: -v -r 'near'")
# Is output directory the last argument?
# Overwrite output, default to 'input.sqlitedb'
self.output=self.args[-1]+'.sqlitedb'
self.store = SqliteTileStorage('TMS')
self.store.create(self.output,True)
# More files on the input not directly supported yet
if (len(self.args) > 1):
self.error("Processing of several input files is not supported.",
"""Please first use a tool like gdal_vrtmerge.py or gdal_merge.py on the files:
gdal_vrtmerge.py -o merged.vrt %s""" % " ".join(self.args))
# TODO: Call functions from gdal_vrtmerge.py directly
self.input = self.args[0]
# Supported options
self.resampling = 'antialias'
# User specified zoom levels
self.tminz = None
self.tmaxz = None
if self.options.zoom:
minmax = self.options.zoom.split('-',1)
minmax.extend([''])
min, max = minmax[:2]
self.tminz = int(min)
if max:
self.tmaxz = int(max)
else:
self.tmaxz = int(min)
# Output the results
if self.options.verbose:
print("Options:", self.options)
print("Input:", self.input)
print("Output:", self.output)
print("Cache: %s MB" % (gdal.GetCacheMax() / 1024 / 1024))
print('')
