def gdalos_make_vrt(
filenames: MaybeSequence[PathOrDS],
filenames_expand: Optional[bool] = None,
vrt_path=None,
kind=None,
resampling_alg=...,
overwrite=True,
vrt_options: Optional[dict] = None,
logger=None,
):
if gdalos_util.is_list_like(filenames):
flatten_filenames = gdalos_util.flatten_and_expand_file_list(filenames, do_expand_glob=filenames_expand)
else:
flatten_filenames = [filenames]
flatten_filenames = [str(f) if isinstance(f, Path) else f for f in flatten_filenames]
if not flatten_filenames:
return None
first_filename = flatten_filenames[0]
if vrt_path is None:
vrt_path = f'/vsimem/{uuid.uuid4()}.vrt'
elif vrt_path is ...:
vrt_path = first_filename + ".vrt"
else:
vrt_path = str(vrt_path)
is_vsimem = vrt_path.startswith('/vsimem/')
if not is_vsimem:
if os.path.isdir(vrt_path):
vrt_path = os.path.join(vrt_path, os.path.basename(first_filename) + ".vrt")
if do_skip_if_exists(vrt_path, overwrite, logger):
return vrt_path
if os.path.isfile(vrt_path):
raise Exception("could not delete vrt file: {}".format(vrt_path))
os.makedirs(os.path.dirname(vrt_path), exist_ok=True)
vrt_options = dict(vrt_options or dict())
if resampling_alg is None:
if kind in [None, ...]:
kind = RasterKind.guess(first_filename)
resampling_alg = kind.resampling_alg_by_kind(kind)
if resampling_alg is not ...:
vrt_options["resampleAlg"] = enum_to_str(resampling_alg)
vrt_options = gdal.BuildVRTOptions(**vrt_options)
ds = gdal.BuildVRT(vrt_path, flatten_filenames, options=vrt_options)
return vrt_path, ds
def composite(self, which_bands, out_composite):
"""
Creates a three band composite out of the specified bands.
Parameters
----------
which_bands : list
A list of bands to save as a three band composite. Must be in
order of how the bands are to saved within the output TIFF.
e.g. which_bands=['band_1', 'band_2', 'band_3']
out_composite : str
The output filename to save the composite,
"""
paths = self.path
bands = []
for i in range(len(which_bands)):
if which_bands[i] == 'band_1':
bands.append(paths["band_1"])
if which_bands[i] == 'band_2':
bands.append(paths["band_2"])
if which_bands[i] == 'band_3':
bands.append(paths["band_3"])
if which_bands[i] == 'band_4':
bands.append(paths["band_4"])
if which_bands[i] == 'band_5':
bands.append(paths["band_5"])
if which_bands[i] == 'band_6':
bands.append(paths["band_6"])
if which_bands[i] == 'band_7':
bands.append(paths["band_7"])
if which_bands[i] == 'band_8':
bands.append(paths["band_8"])
if which_bands[i] == 'band_9':
bands.append(paths["band_9"])
if which_bands[i] == 'band_10':
bands.append(paths["band_10"])
vrt = gdal.BuildVRT('tmp.vrt',
bands,
separate=True,
bandList=[1, 1, 1])
trans = gdal.Translate(out_composite, vrt, format='GTiff')
def test_gdalbuildvrt_lib_usemaskband_on_alpha_band():
src1_ds = gdal.GetDriverByName('MEM').Create('src1', 3, 1, 2)
src1_ds.SetGeoTransform([2, 0.001, 0, 49, 0, -0.001])
src1_ds.GetRasterBand(1).Fill(255)
src1_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand)
src1_ds.GetRasterBand(2).WriteRaster(0, 0, 1, 1, b'\xff')
src2_ds = gdal.GetDriverByName('MEM').Create('src2', 3, 1, 2)
src2_ds.SetGeoTransform([2, 0.001, 0, 49, 0, -0.001])
src2_ds.GetRasterBand(1).Fill(127)
src2_ds.GetRasterBand(2).SetColorInterpretation(gdal.GCI_AlphaBand)
src2_ds.GetRasterBand(2).WriteRaster(1, 0, 1, 1, b'\xff')
ds = gdal.BuildVRT('', [src1_ds, src2_ds])
assert struct.unpack('B' * 3,
ds.GetRasterBand(1).ReadRaster()) == (255, 127, 0)
assert struct.unpack('B' * 3,
ds.GetRasterBand(2).ReadRaster()) == (255, 255, 0)
def test_gdalbuildvrt_lib_usemaskband_on_mask_band():
src1_ds = gdal.GetDriverByName('MEM').Create('src1', 3, 1)
src1_ds.SetGeoTransform([2, 0.001, 0, 49, 0, -0.001])
src1_ds.GetRasterBand(1).Fill(255)
src1_ds.CreateMaskBand(0)
src1_ds.GetRasterBand(1).GetMaskBand().WriteRaster(0, 0, 1, 1, b'\xff')
src2_ds = gdal.GetDriverByName('MEM').Create('src2', 3, 1)
src2_ds.SetGeoTransform([2, 0.001, 0, 49, 0, -0.001])
src2_ds.GetRasterBand(1).Fill(127)
src2_ds.CreateMaskBand(0)
src2_ds.GetRasterBand(1).GetMaskBand().WriteRaster(1, 0, 1, 1, b'\xff')
ds = gdal.BuildVRT('', [src1_ds, src2_ds])
assert struct.unpack('B' * 3, ds.ReadRaster()) == (255, 127, 0)
assert struct.unpack(
'B' * 3,
ds.GetRasterBand(1).GetMaskBand().ReadRaster()) == (255, 255, 0)
def test_vrtmask_4():
gtiff_drv = gdal.GetDriverByName('GTiff')
md = gtiff_drv.GetMetadata()
if md['DMD_CREATIONOPTIONLIST'].find('JPEG') == -1:
pytest.skip()
src_ds = gdal.Open('../gcore/data/ycbcr_with_mask.tif')
gdal.BuildVRT('tmp/vrtmask_4.vrt', [src_ds])
expected_msk_cs = src_ds.GetRasterBand(1).GetMaskBand().Checksum()
src_ds = None
ds = gdal.Open('tmp/vrtmask_4.vrt')
msk_cs = ds.GetRasterBand(1).GetMaskBand().Checksum()
ds = None
os.remove('tmp/vrtmask_4.vrt')
assert msk_cs == expected_msk_cs, 'did not get expected mask band checksum'
def make_vrts(filepath,output_dest,shapefile,epsg,field,region):
"""Create vrt file of all tiles in a repo and then gdal_translate that to a tif file."""
dem_fps = [str(file) for file in Path(filepath).glob('*.tif')]
if not shapefile:
pass
else:
print('filtering inputs to mosaic....')
features = gpd.read_file(shapefile)[field].tolist()
#features = [i[:3]+'*'+i[3:] for i in features]
dem_out = []
for i in dem_fps:
for e in features:
pattern1 = '*'+e[:3]+'*'+e[3:]+'*'
pattern2 = '*'+e[:3].lower()+'*'+e[3:].lower()+'*'
if fnmatch(i,pattern1):# in i:
dem_out.append(i)
#print(i)
#print(e)
elif fnmatch(i,pattern2):# in i:
dem_out.append(i)
#print(i)
#print(e.lower())
else:
continue
#dem_chunks=list(chunks(dem_out,250))
output_vrt = output_dest+f'AK_5m_original_mosaic_{region}_region.vrt'
output_tif = output_dest+f'AK_5m_original_mosaic_{region}_region.tif'
#input_files = filepath+'*.tif' #[str(file) for file in Path(filepath).glob('*.tif')]
cmd = gdal.BuildVRT(output_vrt, dem_out, outputSRS = epsg,allowProjectionDifference=True,srcNodata=-10000)
#make a tif from the vrt file
#print('making the tif file')
#ds = gdal.Translate(output_tif,output_vrt,format='GTiff',outputSRS=epsg)
#close the datasets
#ds = None
cmd = None
return
def write_vrt(infiles, outfile):
"""
Parameters
----------
infiles: list of strings
the input files
outfile: string
the output .vrt
"""
virtpath = outfile
outvirt = gdal.BuildVRT(virtpath, infiles)
outvirt.FlushCache()
outvirt=None
def get_angle_images(self, DST=None):
"""
:param DST: Optional name of the output tif containing all angle images
:return: set self.angles_file
Following band order : SAT_AZ , SAT_ZENITH, SUN_AZ, SUN_ZENITH ')
The unit is DEGREES
"""
if DST is not None:
root_dir = os.path.dirname(DST)
else:
root_dir = os.path.dirname(self.tile_metadata)
# Viewing Angles (SAT_AZ / SAT_ZENITH)
dst_file = os.path.join(root_dir, 'VAA.tif')
out_file_list = self.extract_viewing_angle(dst_file, 'Azimuth')
dst_file = os.path.join(root_dir, 'VZA.tif')
out_file_list.extend(self.extract_viewing_angle(dst_file, 'Zenith'))
# Solar Angles (SUN_AZ, SUN_ZENITH)
dst_file = os.path.join(root_dir, 'SAA.tif')
self.extract_sun_angle(dst_file, 'Azimuth')
out_file_list.append(dst_file)
dst_file = os.path.join(root_dir, 'SZA.tif')
self.extract_sun_angle(dst_file, 'Zenith')
out_file_list.append(dst_file)
out_vrt_file = os.path.join(root_dir, 'tie_points.vrt')
gdal.BuildVRT(out_vrt_file, out_file_list, separate=True)
if DST is not None:
out_tif_file = DST
else:
out_tif_file = os.path.join(root_dir, 'tie_points.tif')
gdal.Translate(out_tif_file, out_vrt_file, format="GTiff")
self.angles_file = out_vrt_file
log.info('SAT_AZ, SAT_ZENITH, SUN_AZ, SUN_ZENITH')
log.info('UNIT = DEGREES (scale: x100) :')
log.info(' ' + out_tif_file)
self.angles_file = out_tif_file
def country_biomass_mosaic(iso3,
input_dir="data_raw",
output_dir="data",
proj="EPSG:3395"):
"""Function to mosaic biomass images from WHRC.
This function mosaics the biomass data obtained from GEE. No
reprojection is performed.
:param iso3: Country ISO 3166-1 alpha-3 code.
:param input_dir: Directory with input files for biomass.
:param output_dir: Output directory.
:param proj: Projection definition (EPSG, PROJ.4, WKT) as in
GDAL/OGR. Default to "EPSG:3395" (World Mercator).
"""
# Create output directory
make_dir("data")
# Mosaicing
files_tif = input_dir + "/biomass_whrc_" + iso3 + "*.tif"
input_list = glob(files_tif)
output_file = input_dir + "/biomass_whrc.vrt"
gdal.BuildVRT(output_file, input_list)
# Compressing
input_file = input_dir + "/biomass_whrc.vrt"
output_file = output_dir + "/biomass_whrc.tif"
param = gdal.TranslateOptions(options=["overwrite", "tap"],
format="GTiff",
noData=-9999,
outputSRS=proj,
creationOptions=[
"TILED=YES", "BLOCKXSIZE=256",
"BLOCKYSIZE=256", "COMPRESS=LZW",
"PREDICTOR=2", "BIGTIFF=YES"
])
gdal.Translate(output_file, input_file, options=param)
def test_stacta_east_hemisphere():
# Test a json file with min_tile_col = 1 at zoom level 2
ds = gdal.OpenEx('data/stacta/test_east_hemisphere.json',
open_options=['WHOLE_METATILE=YES'])
assert ds.RasterCount == 3
assert ds.RasterXSize == 1024
assert ds.RasterYSize == 1024
assert ds.GetSpatialRef().GetName() == 'WGS 84'
assert ds.GetGeoTransform() == pytest.approx(
[0.0, 0.17578125, 0.0, 90.0, 0.0, -0.17578125], rel=1e-8)
assert ds.GetRasterBand(1).GetOverviewCount() == 2
# Create a reference dataset, that is externally the same as the STACTA one
vrt_ds = gdal.BuildVRT('', ['data/stacta/WorldCRS84Quad/2/0/1.tif'])
ref_ds = gdal.Translate('', vrt_ds, format='MEM')
ref_ds.BuildOverviews('NEAR', [2, 4])
assert ds.ReadRaster(600, 500, 50, 100, 10, 20) == \
ref_ds.ReadRaster(600, 500, 50, 100, 10, 20)
def gdalbuildvrt(src, dst, options):
"""
a simple wrapper for gdal.BuildVRT
Parameters
----------
src: str, ogr.DataSource or gdal.DataSource
the input data set
dst: str
the output data set
options: dict
additional parameters passed to gdal.BuildVRT;
see http://gdal.org/python/osgeo.gdal-module.html#BuildVRTOptions
Returns
-------
"""
out = gdal.BuildVRT(dst, src, options=gdal.BuildVRTOptions(**options))
out = None
def extract_chunk_tile_bands(tif_file):
tile_dir = get_tile_dir(tif_file)
create_tile_dir(tile_dir)
os.chdir(tile_dir)
rel_tif_file = relpath(tif_file, tile_dir)
ds = gdal.Open(rel_tif_file, GA_ReadOnly)
for band_index in range(1, ds.RasterCount + 1):
band_name = ds.GetRasterBand(band_index).GetDescription()
band_file = join(tile_dir, band_name + '.vrt')
gdal.SetConfigOption('VRT_SHARED_SOURCE', '0')
vrt = gdal.BuildVRT(
band_file, rel_tif_file,
bandList=[band_index],
VRTNodata=nodata_value
)
vrt.GetRasterBand(1).SetDescription(band_name)
vrt.FlushCache()
make_relative_to_vrt(band_file)
print('.', end='', flush=True)
def finished(self, result):
if result:
layers = {}
valid = True
for mosaic, files in self.filenames.items():
mosaiclayers = []
for filename in files:
mosaiclayers.append(QgsRasterLayer(filename, os.path.basename(filename), "gdal"))
layers[mosaic] = mosaiclayers
valid = valid and (False not in [lay.isValid() for lay in mosaiclayers])
if not valid:
widget = iface.messageBar().createMessage("Planet Explorer",
f"Order '{self.order.name}' correctly downloaded ")
button = QPushButton(widget)
button.setText("Open order folder")
button.clicked.connect(lambda: QDesktopServices.openUrl(
QUrl.fromLocalFile(self.order.download_folder()))
)
widget.layout().addWidget(button)
iface.messageBar().pushWidget(widget, level=Qgis.Success)
else:
if self.order.load_as_virtual:
for mosaic, files in self.filenames.items():
vrtpath = os.path.join(self.order.download_folder(), mosaic, f'{mosaic}.vrt')
gdal.BuildVRT(vrtpath, files)
layer = QgsRasterLayer(vrtpath, mosaic, "gdal")
QgsProject.instance().addMapLayer(layer)
else:
for mosaic, mosaiclayers in layers.items():
for layer in mosaiclayers:
QgsProject.instance().addMapLayer(layer)
#TODO create groups
iface.messageBar().pushMessage("Planet Explorer",
f"Order '{self.order.name}' correctly downloaded and processed",
level=Qgis.Success, duration=5)
elif self.exception is not None:
QgsMessageLog.logMessage(f"Order '{self.order.name}' could not be downloaded.\n{self.exception}",
QGIS_LOG_SECTION_NAME, Qgis.Warning)
iface.messageBar().pushMessage("Planet Explorer",
f"Order '{self.order.name}' could not be downloaded. See log for details",
level=Qgis.Warning, duration=5)
def gdal_stack_images_vrt(input_imgs, output_vrt_file):
"""
A function which creates a GDAL VRT file from a set of input images by stacking the input images
in a multi-band output file.
:param input_imgs: A list of input images
:param output_vrt_file: The output file location for the VRT.
"""
try:
import tqdm
pbar = tqdm.tqdm(total=100)
callback = lambda *args, **kw: pbar.update()
except:
callback = gdal.TermProgress
build_vrt_opt = gdal.BuildVRTOptions(separate=True)
gdal.BuildVRT(output_vrt_file,
input_imgs,
options=build_vrt_opt,
callback=callback)
def stack_rasters(rasters, out, rescale=False, rescale_min=0, rescale_max=1):
if rescale:
rescaled = []
for r in rasters:
logger.info("Rescaling {}".format(r))
rescaled_name = r'/vsimem/{}_rescale.vrt'.format(Path(r).stem)
rescale_raster(str(r),
rescaled_name,
out_min=rescale_min,
out_max=rescale_max)
rescaled.append(rescaled_name)
rasters = rescaled
logger.info('Building stacked VRT...')
temp = r'/vsimem/stack.vrt'
gdal.BuildVRT(temp, rasters, separate=True)
logger.info('Writing to: {}'.format(out))
out_ds = gdal.Translate(out, temp)
return out_ds
def main(in_files, out_file):
vrt_file = os.path.join(tempfile.gettempdir(), '%s_vrt.vrt' % os.path.splitext(os.path.basename(out_file))[0])
if os.path.exists(vrt_file):
os.remove(vrt_file)
gdal.BuildVRT(vrt_file, str(in_files).split(','), srcNodata = srcNodata, VRTNodata = VRTNodata)
progress(0.25)
out_driver = gdal.GetDriverByName("GTiff")
if os.path.exists(out_file):
out_driver.Delete(out_file)
time.sleep(1)
out_sds = out_driver.CreateCopy(out_file, gdal.Open(vrt_file))
progress(0.85)
out_sds = None
os.remove(vrt_file)
progress(1)
def test_gdalbuildvrt_lib_2():
tab = [0]
ds = gdal.BuildVRT('',
'../gcore/data/byte.tif',
callback=mycallback,
callback_data=tab)
if ds is None:
return 'fail'
if ds.GetRasterBand(1).Checksum() != 4672:
gdaltest.post_reason('Bad checksum')
return 'fail'
if tab[0] != 1.0:
gdaltest.post_reason('Bad percentage')
return 'fail'
ds = None
return 'success'
def create_vrt_from_stac_assets(assets: dict, assets_name_to_use: list,
vrt_output_file: str):
"""Function to create GDAL VRT from STAC Assets
Args:
assets (dict): STAC-Assets dict (with href key)
assets_name_to_use (list): List of assets to be insert in vrt
vrt_output_file (str): Complete path and filename
Returns:
None
"""
from osgeo import gdal
# create href elements (vsicurl to allow gdal driver get elements without download complete image)
assets_href = [
"/vsicurl/" + assets[asset_name]['href']
for asset_name in assets_name_to_use
]
vrt_options = gdal.BuildVRTOptions(separate=True)
vrt_gdal = gdal.BuildVRT(vrt_output_file, assets_href, options=vrt_options)
vrt_gdal = None
def generate_mosaic(input_directory, output_directory, variable):
#output_tif = output_dest+f'AK_5m_original_mosaic_{region}_region.tif'
dem_fps = [str(file) for file in Path(input_directory).glob('*.tif')]
#input_files = filepath+'*.tif' #[str(file) for file in Path(filepath).glob('*.tif')]
for year in range(2002, 2020):
output_vrt = output_directory + f'MODIS_{variable}_{year}_by_climate_region_and_elevation_band.vrt'
year_list = [i for i in dem_fps if str(i[-8:-4]) == str(year)]
print(year_list)
cmd = gdal.BuildVRT(output_vrt,
dem_fps,
outputSRS='EPSG:3338',
allowProjectionDifference=True,
srcNodata=0)
#make a tif from the vrt file
#print('making the tif file')
#ds = gdal.Translate(output_tif,output_vrt,format='GTiff',outputSRS=epsg)
#close the datasets
#ds = None
cmd = None
def test_gdalbuildvrt_lib_separate_nodata_3():
src1_ds = gdal.GetDriverByName('MEM').Create('', 1000, 1000)
src1_ds.SetGeoTransform([2,0.001,0,49,0,-0.001])
src1_ds.GetRasterBand(1).SetNoDataValue(1)
src2_ds = gdal.GetDriverByName('MEM').Create('', 1000, 1000)
src2_ds.SetGeoTransform([2,0.001,0,49,0,-0.001])
src2_ds.GetRasterBand(1).SetNoDataValue(2)
gdal.BuildVRT('/vsimem/out.vrt', [src1_ds, src2_ds], separate=True, srcNodata='3 4', VRTNodata='5 6')
f = gdal.VSIFOpenL('/vsimem/out.vrt', 'rb')
data = gdal.VSIFReadL(1, 10000, f)
gdal.VSIFCloseL(f)
gdal.Unlink('/vsimem/out.vrt')
assert b'<NoDataValue>5</NoDataValue>' in data
assert b'<NODATA>3</NODATA>' in data
assert b'<NoDataValue>6</NoDataValue>' in data
assert b'<NODATA>4</NODATA>' in data
def make_hillshade(dem_paths):
# Make sure all dem_paths exist
# An obscure error is given if the files don't exist
for dem_path in dem_paths:
if not Path(dem_path).exists():
raise FileNotFoundError(dem_path)
tmpdir = tempfile.mkdtemp()
# Extract each dem_path into tmpdir
unzipped_paths = []
for dem_path in dem_paths:
zf = ZipFile(dem_path)
names = zf.namelist()
img_file = [x for x in names if x.endswith('.img')]
assert len(img_file) == 1, 'More than one img file in zip file'
img_file = img_file[0]
unzipped_path = Path(tmpdir) / img_file
with open(unzipped_path, 'wb') as f:
f.write(zf.read(img_file))
unzipped_paths.append(unzipped_path)
vrt_path = os.path.join(tmpdir, 'dem.vrt')
# Setting vrt to None is weird but required
# https://gis.stackexchange.com/a/314580
# https://gdal.org/tutorials/raster_api_tut.html#using-createcopy
vrt = gdal.BuildVRT(vrt_path, unzipped_paths)
vrt = None
# Check that vrt_path actually was created
if not Path(vrt_path).exists():
raise ValueError('Unable to create virtual raster')
return vrt_path
dem_paths
pass
def _mosaic(input_rasters, target_srs, resample, resolution, dtype, nodata,
gdal_multithread):
vrt_tempname = f"/vsimem/{uuid4()}.vrt"
vrt = gdal.BuildVRT(vrt_tempname, input_rasters)
assert vrt
vrt.FlushCache()
vrt = None
log.debug("Created VRT")
wrp_tempname = f"/vsimem/{uuid4()}.tif"
wopt = gdal.WarpOptions(
dstSRS=target_srs,
outputType=dtype,
resampleAlg=resample,
xRes=resolution[0],
yRes=resolution[1],
srcNodata=nodata,
dstNodata=nodata,
multithread=gdal_multithread,
)
wrp = gdal.Warp(wrp_tempname, vrt_tempname, options=wopt)
assert wrp
log.debug("Created WRP")
yield wrp
log.debug("Performing Cleanup")
wrp = None
vrt = None
rc1 = gdal.Unlink(vrt_tempname)
rc2 = gdal.Unlink(wrp_tempname)
if rc1 != 0 or rc2 != 0:
log.warning(
"Received return codes [%s, %s] while removing MemRasters",
rc1, rc2)
def run_merge(args):
"""Merge downloaded blocks into one VRT file"""
input_ds = gdal.Open(args.input)
print('Input dataset size', input_ds.RasterXSize, 'x', input_ds.RasterYSize)
bounds = get_bounds(input_ds)
print('Dataset bounds', bounds)
conn = get_db(args)
cursor = conn.cursor()
complete_block_names = []
failed_block_names = []
for row in cursor.execute("SELECT file_name, file_hash FROM task WHERE complete "):
file_name = os.path.join(args.output, row['file_name'])
file_ok, msg = verify_file(args, file_name, row['file_hash'])
if not file_ok:
print('\033[91m{:32} {}\033[0m'.format(row['file_name'], msg))
failed_block_names.append([row['file_name']])
else:
print('{:32} {} {}!'.format(row['file_name'], row['file_hash'], msg))
complete_block_names.append(file_name)
print('Found {} downloaded blocks'.format(len(complete_block_names)))
if failed_block_names:
print('\033[91mFound {} invalid blocks. Rerun download.\033[0m'
.format(len(failed_block_names)))
cursor.executemany('UPDATE task SET '
'complete = 0, '
'file_url = Null, file_hash = Null '
'WHERE file_name = ?',
failed_block_names)
conn.commit()
return 1
gdal.BuildVRT(os.path.join(args.output, 'merge.img'),
complete_block_names,
outputBounds=bounds,
callback=gdal.TermProgress)
return 0
def monthly_mosaic_lst(product, month, year, day_night, tilesH, tilesV,
save_path, usr, pwd, ncores):
# Creating a tempdir to store the monthly geotiff per tile
mosaic_name = (product + '.A' + str(year) + str(month).zfill(2) + '.' +
day_night)
tp = tempfile.mkdtemp()
temp_path_geotiff = os.path.join(tp, 'GEOTIFF', mosaic_name)
os.makedirs(temp_path_geotiff, exist_ok=True)
#
# One monthly composite is produced for each tile (on one or several cores)
Parallel(n_jobs=ncores, backend='threading')(
delayed(dl_month_hdf)(product=product,
day_night=day_night,
tileH=tilesH[i],
tileV=tilesV[i],
year=year,
month=month,
usr=usr,
pwd=pwd,
temp_path_geotiff=temp_path_geotiff)
for i in range(len(tilesV)))
#
# Build a virtual raster mosaicing all the GeoTiff in temp_path_geotiff
# VRT are saved in a temporary dir
path_vrt = tempfile.mkdtemp()
os.makedirs(path_vrt, exist_ok=True)
VRT = gdal.BuildVRT(os.path.join(path_vrt, mosaic_name + ".vrt"),
glob.glob(os.path.join(temp_path_geotiff, "*.tif")))
# Save the mosaic as a GeoTiff
save_raster(value=VRT.ReadAsArray(),
transfo=VRT.GetGeoTransform(),
projection=VRT.GetProjection(),
save_path=os.path.join(save_path, mosaic_name + '.tif'),
in_na_value=0)
VRT = None
# Removing the temporary geotiff
rmtree(temp_path_geotiff, ignore_errors=True)
# Removing the VRT temporary directory
rmtree(path_vrt, ignore_errors=True)
def createnativemosaic(filelist,
nativemosaic,
resample='lanczos',
mosaictype='vrt'):
"""
input is an array of file addresses, a location to place output,
and a resampling method.
writes a VRT to the specified location
"""
print("building native CRS mosaic")
#GDAL understands the /vsis3/ driver, niot s3 urls:
# https://www.gdal.org/gdal_virtual_file_systems.html
if 's3://' in nativemosaic:
nativemosaic = nativemosaic.replace('s3://', '/vsis3/')
vrtoptions = gdal.BuildVRTOptions(resampleAlg=resample, addAlpha=True)
myvrt = gdal.BuildVRT(nativemosaic, filelist, options=vrtoptions)
myvrt.FlushCache()
myvrt = None
return (nativemosaic)
def compileVrt(scene, product):
out_pathname = None
# generate file list
filelist = fio.getFileList('*', scene)
if len(filelist) > 0:
# need to guarantee consistent band ordering
sorted_list = []
# for each band
bands = product.getElementsByTagName('band')
for band in bands:
if (band.hasAttribute("filename")):
for obj in filelist:
# add entry to sort list if configuration matches argument
if fnmatch.fnmatch(os.path.basename(obj),
str(band.attributes["filename"].value)):
sorted_list.append(obj)
break
# build vrt on validation of successful sort
if len(sorted_list) == len(bands):
out_pathname = scene + '/' + product.attributes["name"].value + '.vrt'
updateImages(sorted_list)
vrt = gdal.BuildVRT(out_pathname,
sorted_list,
options=gdal.BuildVRTOptions(separate=True))
vrt = None
return out_pathname
def combine_same_utm():
'''Combines all files within one UTM zone in on vrt-file (per day)'''
print("-> Start combining same UTM zone...")
# Create Out folder
TEMP_FOLDERS["utm"] = create_folder(OUT_FOLDER, "04_combined_utm")
# iterate over each day
for day in listdir(TEMP_FOLDERS["unzipped"]):
# Create one folder per day
folder_day = create_folder(TEMP_FOLDERS["utm"], day)
# Sort files after their UTM zone
zones = {}
for file in listdir("{0}/{1}".format(TEMP_FOLDERS["combined_bands"],
day)):
file_path = "{0}/{1}/{2}".format(TEMP_FOLDERS["combined_bands"],
day, file)
cur_zone = file[1:3]
if cur_zone not in zones:
zones[cur_zone] = [file_path]
else:
zones[cur_zone].append(file_path)
# Iterate over UTM zones in dict
for zone in zones.keys():
# Build out_path
out_filename = "T{0}_{1}.vrt".format(zone, day)
out_path = "{0}/{1}".format(folder_day, out_filename)
# Merge all vrt-files inside one UTM zone
gdal.BuildVRT(out_path, zones[zone])
print(" - Combined UTM for {0}".format(out_filename))
print("-> Finished combining same UTM zone.")
def buildvrt_utility(self, task, zoom_levels):
if not (task["parameters"]["fileNames"] and task["parameters"]["originDirectory"]):
raise VRTError("jobData didn't have source files data, for {0}"
.format(utilities.task_format_log(task)))
vrt_config = {
'VRTNodata': self.__config["gdal"]["vrt"]["no_data"],
'outputSRS': self.__config["gdal"]["vrt"]["output_srs"],
'resampleAlg': self.__config["gdal"]["vrt"]["resample_algo"],
'addAlpha': self.__config["gdal"]["vrt"]["add_alpha"]
}
self.log.info("Starting process GDAL-BUILD-VRT on {0} and zoom-levels: {1}"
.format(utilities.task_format_log(task), zoom_levels))
mount_path = self.__config['source_mount']
files = [path.join(mount_path, task["parameters"]['originDirectory'], file) for file in task["parameters"]['fileNames']]
vrt_result = gdal.BuildVRT(self.vrt_file_location(task["parameters"]['discreteId']), files, **vrt_config)
if vrt_result != None:
vrt_result.FlushCache()
vrt_result = None
else:
raise VRTError("Could not create VRT File")
def gdalbuildvrt(src, dst, options=None, void=True):
"""
a simple wrapper for :osgeo:func:`gdal.BuildVRT`
Parameters
----------
src: str, list, :osgeo:class:`ogr.DataSource` or :osgeo:class:`gdal.Dataset`
the input data set(s)
dst: str
the output data set
options: dict
additional parameters passed to :osgeo:func:`gdal.BuildVRT`; see :osgeo:func:`gdal.BuildVRTOptions`
void: bool
just write the results and don't return anything? If not, the spatial object is returned
Returns
-------
"""
options = {} if options is None else options
if 'outputBounds' in options.keys() and gdal.__version__ < '2.4.0':
warnings.warn(
'\ncreating VRT files with subsetted extent is very likely to cause problems. '
'Please use GDAL version >= 2.4.0, which fixed the problem.\n'
'see here for a description of the problem:\n'
' https://gis.stackexchange.com/questions/314333/'
'sampling-error-using-gdalwarp-on-a-subsetted-vrt\n'
'and here for the release note of GDAL 2.4.0:\n'
' https://trac.osgeo.org/gdal/wiki/Release/2.4.0-News')
out = gdal.BuildVRT(dst, src, options=gdal.BuildVRTOptions(**options))
out.FlushCache()
if void:
out = None
else:
return out
def create_vrt(path_save,
list_path_raster,
list_band_name=None,
src_nodata=None,
dst_nodata=None):
"""
Write raster from image (can use with gdal and rasterio raster).
Parameters
----------
path_save: str
Virtual's save path.
list_band_name: list of string (optional), default None
List of the name of each band. Otherwise, blank band names.
src_nodata: int (optional), default None
Nodata value of the source raster.
dst_nodata: int (optional), default None
Nodata value of the virtual raster.
Examples
--------
>>> create_vrt(path_save="ice.tif", list_path_raster=["raster1.tif", "raster2.tif"], list_band_name=["ice", "lnw"], src_nodata=0, dst_nodata=0)
Returns
-------
None
"""
options = gdal.BuildVRTOptions(separate=True,
srcNodata=src_nodata,
VRTNodata=dst_nodata)
outds = gdal.BuildVRT(path_save, list_path_raster, options=options)
if list_band_name is not None:
for idx, band_name in enumerate(list_band_name):
outds.GetRasterBand(idx + 1).SetDescription(band_name)
outds.FlushCache()
del outds