def transformed_template(self, template, substitutions):
vrt_templ = Template(open(template).read())
vrt_xml = vrt_templ.substitute(substitutions)
vrt = tempfile.mktemp('.vrt', 'ogr_', '/vsimem')
# Create in-memory file
gdal.FileFromMemBuffer(vrt, vrt_xml)
return vrt
def shapefilereader(target):
"""Function to convert zipped shapefiles from the web or on disk into geopandas dataframes
Parameters
----------
target : str
string representing path to file on disk or url to download the zipped shapefile.
Returns
-------
Geopandas dataframe
Pandas dataframe with geospatial features and operations.
"""
# Detect whether we are using a web-based shapefile or local disk
r = re.compile('^(http|https)://', re.I)
if r.search(target):
download = True
request = requests.get(target)
target = '/vsimem/{}.zip'.format(
uuid.uuid4().hex) #gdal/ogr requires a .zip extension
gdal.FileFromMemBuffer(target, bytes(request.content))
else:
download = False
with fiona.Collection(target, vsi='zip') as f:
return gpd.GeoDataFrame.from_features(f, crs=f.crs)
def read_tiff(self, buffer, create_alpha=True):
""" Reads image using GDAL from a buffer, and returns a CV2 image.
"""
mem_map_name = '/vsimem/{}'.format(uuid.uuid4().get_hex())
gdal_img = None
try:
gdal.FileFromMemBuffer(mem_map_name, buffer)
gdal_img = gdal.Open(mem_map_name)
channels = [
gdal_img.GetRasterBand(i).ReadAsArray()
for i in range(1, gdal_img.RasterCount + 1)
]
if len(channels) >= 3: # opencv is bgr not rgb.
red_channel = channels[0]
channels[0] = channels[2]
channels[2] = red_channel
if len(channels) < 4 and create_alpha:
self.no_data_value = gdal_img.GetRasterBand(1).GetNoDataValue()
channels.append(
numpy.float32(
gdal_img.GetRasterBand(1).GetMaskBand().ReadAsArray()))
return cv.fromarray(cv2.merge(channels))
finally:
gdal_img = None
gdal.Unlink(mem_map_name) # Cleanup.
def vrt_read_7():
filename = "/vsimem/vrt_read_7.vrt"
content = """<VRTDataset rasterXSize="20" rasterYSize="20">
<VRTRasterBand dataType="Byte" band="1">
<SimpleSource>
<SourceFilename relativeToVRT="1">%s</SourceFilename>
<SourceBand>1</SourceBand>
<SrcRect xOff="0" yOff="0" xSize="20" ySize="20" />
<DstRect xOff="0" yOff="0" xSize="20" ySize="20" />
</SimpleSource>
</VRTRasterBand>
</VRTDataset>""" % filename
gdal.FileFromMemBuffer(filename, content)
gdal.PushErrorHandler('CPLQuietErrorHandler')
ds = gdal.Open(filename)
gdal.PopErrorHandler()
error_msg = gdal.GetLastErrorMsg()
gdal.Unlink(filename)
if ds is not None:
return 'fail'
if error_msg != 'GDALOpen() called with too many recursion levels':
return 'fail'
return 'success'
def tif_enve_to_poly_s3(s3tif):
# include these three lines if it is an s3 path not an s3 object
# bucket_name = s3path.split(r's3://')[1].split(r'/')[0]
# key = s3path.split(r'{}/'.format(bucket_name))[1]
# s3tif = s3.Object(bucket_name=bucket_name, key=key)
image_data = BytesIO(s3tif.get()['Body'].read())
tif_inmem = "/vsimem/data.tif" #Virtual Folder to Store Data
gdal.FileFromMemBuffer(tif_inmem, image_data.read())
src = gdal.Open(tif_inmem)
ulx, xres, xskew, uly, yskew, yres = src.GetGeoTransform()
lrx = ulx + (src.RasterXSize * xres)
lry = uly + (src.RasterYSize * yres)
# Create polygon from bounding box
# Create ring
ring = ogr.Geometry(ogr.wkbLinearRing)
ring.AddPoint(ulx, uly)
ring.AddPoint(ulx, lry)
ring.AddPoint(lrx, lry)
ring.AddPoint(lrx, uly)
ring.AddPoint(ulx, uly)
# Create polygon
poly = ogr.Geometry(ogr.wkbPolygon)
poly.AddGeometry(ring)
return poly
def read_geometry_with_gdal(self, response):
gdal.FileFromMemBuffer('/vsimem/temp', response.content)
dataset = gdal.ogr.Open('/vsimem/temp')
layer = dataset.GetLayer(0)
for _ in range(layer.GetFeatureCount()):
yield layer.GetNextFeature()
layer = None
gdal.Unlink('/vsimem/temp')
def read_from_s3(self) -> 'gdal objects':
assert not self._is_local, 'Tiff must be on s3 to use this function'
s3Obj = s3.Object(self._bucket, self._prefix)
image_data = BytesIO(s3Obj.get()['Body'].read())
tif_inmem = "/vsimem/data.tif" #Virtual Folder to Store Data
gdal.FileFromMemBuffer(tif_inmem, image_data.read())
src = gdal.Open(tif_inmem)
return src.GetRasterBand(1), src.GetGeoTransform(), src
def open(self):
"""
see gist.github.com/jleinonen/5781308 gdal_mmap.py
"""
mmap_name = '/vsimem/' + uuid4().hex #vsimem is special GDAL string
gdal.FileFromMemBuffer(mmap_name, self.buf)
return gdal.Open(mmap_name)
def show_url(url, *args, **kwargs):
try:
urlfd = urlopen(url, timeout=15)
mmapfn = "/vsimem/" + uuid4().hex
gdal.FileFromMemBuffer(mmapfn, urlfd.read())
show_fd(gdal.Open(mmapfn), *args, **kwargs)
except URLError as e:
print(e)
finally:
gdal.Unlink(mmapfn)
def queryRaster(conn,
rastertable,
tiletable,
tileid,
date,
rtype,
level,
bands,
verbose=False):
curs = conn.cursor()
# convert band names to band numbers using config file
sql = """
select
ST_astiff(ST_UNION(ST_CLIP(r.rast, t.geom)),ARRAY{bands})
from
{rastertable} r, {tiletable} t
where
t.id={tileid} and
ST_INTERSECTS(r.rast,t.geom) and
r.type='{rtype}' and
r.level='{level}' and
date='{date}'
""".format(rastertable=rastertable,
tiletable=tiletable,
tileid=tileid,
rtype=rtype,
level=level,
date=date.strftime("%Y-%m-%d"),
bands=bands)
if verbose: print sql
# following https://gis.stackexchange.com/questions/130139/downloading-raster-data-into-python-from-postgis-using-psycopg2
# Use a virtual memory file, which is named like this
vsipath = '/vsimem/from_postgis'
# Download raster data into Python as GeoTIFF, and make a virtual file for GDAL
curs.execute(sql)
gdal.FileFromMemBuffer(vsipath, bytes(curs.fetchone()[0]))
# Read first band of raster with GDAL
ds = gdal.Open(vsipath)
arrays = []
for b in range(len(bands)):
band = ds.GetRasterBand(b + 1)
arrays.append(band.ReadAsArray())
# Close and clean up virtual memory file
ds = band = None
gdal.Unlink(vsipath)
curs.close()
return np.stack(arrays, axis=2)
def download_image(self, api_key, base_url, parcel_id, product_id,
raster_id, source, img_format, epsg):
"""
Downloads, georeferences and transforms (if necessary) the raster image from the API with the given parameters.
:param api_key:
:param base_url:
:param parcel_id:
:param product_id:
:param raster_id:
:param source:
:param img_format:
:param epsg:
:return: The memory map string of the transformed GDAL dataset.
"""
img = self.get_raster(base_url,
api_key,
parcel_id,
product_id,
source,
raster_id,
img_format=img_format)
mmap_name = "/vsimem/" + uuid4().get_hex()
gdal.FileFromMemBuffer(mmap_name, img)
dataset = gdal.Open(mmap_name)
#print("original raster: {0}".format(mmap_name))
NDV, xsize, ysize, GeoT, Projection, DataType = self.get_gdal_metadata(
dataset)
#print(NDV, xsize, ysize, GeoT, Projection, DataType)
# PNG has to be referenced, Geotiff has the projection info already
if img_format == 'png':
bbox = self.get_raster_bbox(base_url, api_key, parcel_id,
product_id)
# original is WGS84 (because of bbox)
self.georeference_raster(dataset,
bbox,
xsize,
ysize,
epsg_code=4326)
#reproject raster
# TODO only if EPSG codes differ!
mmap_name = self.transform_raster(dataset, dst_epsg_code=epsg)
#print("transformed raster: {0}".format(mmap_name))
return mmap_name
def getTifData(s3path: s3.Object) -> gdal_data:
"""Read a raster from S3 into memory and get attributes"""
image_data = BytesIO(s3path.get()['Body'].read())
tif_inmem = "/vsimem/data.tif" #Virtual Folder to Store Data
gdal.FileFromMemBuffer(tif_inmem, image_data.read())
print(s3path)
src = gdal.Open(tif_inmem)
rb, gt = src.GetRasterBand(1), src.GetGeoTransform()
return rb, gt, src
def get_raster(query):
cursor = connection.cursor()
cursor.execute(sql=query)
vsipath = '/vsimem/from_postgis'
memview = cursor.fetchone()[0]
gdal.FileFromMemBuffer(vsipath, bytes(memview))
ds = gdal.Open(vsipath)
band = ds.GetRasterBand(1)
data = band.ReadAsArray()
ds = band = None
gdal.Unlink(vsipath)
return data
def load_float32_image(buffer):
try:
gdal.FileFromMemBuffer('/vsimem/temp', buffer)
ds = gdal.Open('/vsimem/temp')
channel = ds.GetRasterBand(1).ReadAsArray()
ds = None #cleanup
gdal.Unlink('/vsimem/temp') #cleanup
return channel
except Exception, e:
ds = None #cleanup
gdal.Unlink('/vsimem/temp') #cleanup
raise e
def buff2rast(buff):
# Use a virtual memory file, which is named like this
vsipath = '/vsimem/from_postgis'
gdal.FileFromMemBuffer(vsipath, bytes(buff))
# Read first band of raster with GDAL
ds = gdal.Open(vsipath)
band = ds.GetRasterBand(1)
gdal.Unlink(vsipath)
return np.flipud(band.ReadAsArray())
def open_image(url):
image_data = open_http_query(url)
if not image_data:
return None
mmap_name = "/vsimem/" + uuid4().get_hex()
gdal.FileFromMemBuffer(mmap_name, image_data.read())
gdal_dataset = gdal.Open(mmap_name)
image = gdal_dataset.GetRasterBand(1).ReadAsArray()
gdal_dataset = None
gdal.Unlink(mmap_name)
return image
def getTifData_S3(s3path):
"""Read a raster from S3 into memory and get attributes"""
s3 = boto3.resource("s3")
if isinstance(s3path, str):
bucket_name = s3path.split(r"s3://")[1].split(r"/")[0]
key = s3path.split(r"{}/".format(bucket_name))[1]
s3tif = s3.Object(bucket_name=bucket_name, key=key)
image_data = BytesIO(s3tif.get()["Body"].read())
else:
image_data = BytesIO(s3path.get()["Body"].read())
tif_inmem = "/vsimem/data.tif" # Virtual Folder to Store Data
gdal.FileFromMemBuffer(tif_inmem, image_data.read())
src = gdal.Open(tif_inmem)
rb, gt = src.GetRasterBand(1), src.GetGeoTransform()
null_value = rb.GetNoDataValue()
return rb, gt, src, null_value
def writeTile(self, tile, image):
if tile.z != self._zoom:
self._initZoomLayer(tile.z)
data = QByteArray()
buff = QBuffer(data)
image.save(buff, 'PNG')
mmap_name = '/vsimem/' + uuid4().hex
gdal.FileFromMemBuffer(mmap_name, data.data())
gdal_dataset = gdal.Open(mmap_name)
data = gdal_dataset.ReadRaster(0, 0, self.tile_width, self.tile_height)
gdal_dataset = None
gdal.Unlink(mmap_name)
xoff = (tile.x - self._first_tile.x) * self.tile_width
yoff = (tile.y - self._first_tile.y) * self.tile_height
self._zoomDs.WriteRaster(xoff, yoff, self.tile_width, self.tile_height, data)
def ogr_georss_15():
if not gdaltest.georss_read_support:
return 'skip'
try:
gdal.FileFromMemBuffer
except:
return 'skip'
content = """<?xml version="1.0" encoding="UTF-8"?>
<rss version="2.0" xmlns:georss="http://www.georss.org/georss" xmlns:gml="http://www.opengis.net/gml">
<channel>
<link>http://mylink.com</link>
<title>channel title</title>
<item>
<guid isPermaLink="false">0</guid>
<pubDate>Thu, 2 Apr 2009 23:03:00 +0000</pubDate>
<title>item title</title>
<georss:point>49 2</georss:point>
</item>
</channel>
</rss>"""
# Create in-memory file
gdal.FileFromMemBuffer('/vsimem/georssinmem', content)
ds = ogr.Open('/vsimem/georssinmem')
lyr = ds.GetLayer(0)
feat = lyr.GetNextFeature()
if feat.GetFieldAsString('title') != 'item title':
print(('Expected %s. Got %s' % ('item title', feat.GetFieldAsString('title'))))
return 'fail'
feat.Destroy()
ds.Destroy()
# Release memory associated to the in-memory file
gdal.Unlink('/vsimem/georssinmem')
return 'success'
def read_tiff(self, buffer, create_alpha=True):
""" Reads image using GDAL from a buffer, and returns a CV2 image.
"""
offset = float(getattr(self.context, 'offset', 0.0)) or 0
mem_map_name = '/vsimem/{}'.format(uuid.uuid4().hex)
gdal_img = None
try:
gdal.FileFromMemBuffer(mem_map_name, buffer)
gdal_img = gdal.Open(mem_map_name)
channels = [
gdal_img.GetRasterBand(i).ReadAsArray()
for i in range(1, gdal_img.RasterCount + 1)
]
if len(channels) >= 3: # opencv is bgr not rgb.
red_channel = channels[0]
channels[0] = channels[2]
# Offset is z-offset to the elevation value
# If it's set, we are reading a DEM tiff, which stores its elevation data in channels[2]
# We don't want to add an offset to a no-data value
no_data_value = None if not offset else gdal_img.GetRasterBand(
1).GetNoDataValue()
add_offset_if_data = numpy.vectorize(
lambda x: x + offset
if offset and x != no_data_value else x,
otypes=[numpy.float32])
# If there's an offset, run add_offset_if_data on numpy array, else just assign it to the proper channel
channels[2] = add_offset_if_data(
red_channel) if offset else red_channel
if len(channels) < 4 and create_alpha:
self.no_data_value = gdal_img.GetRasterBand(1).GetNoDataValue()
channels.append(
numpy.float32(
gdal_img.GetRasterBand(1).GetMaskBand().ReadAsArray()))
return cv2.merge(channels)
finally:
gdal_img = None
gdal.Unlink(mem_map_name) # Cleanup.
def xyz_3():
content = """Y X Z
0 0 65
0 1 66
1 0 67
1 1 68
2 0 69
2 1 70
"""
gdal.FileFromMemBuffer('/vsimem/grid.xyz', content)
ds = gdal.Open('/vsimem/grid.xyz')
buf = ds.ReadRaster(0,2,2,1)
bytes = struct.unpack('B' * 2, buf)
if bytes != (69, 70):
print(buf)
return 'fail'
buf = ds.ReadRaster(0,1,2,1)
bytes = struct.unpack('B' * 2, buf)
if bytes != (67, 68):
print(buf)
return 'fail'
buf = ds.ReadRaster(0,0,2,1)
bytes = struct.unpack('B' * 2, buf)
if bytes != (65, 66):
print(buf)
return 'fail'
buf = ds.ReadRaster(0,2,2,1)
bytes = struct.unpack('B' * 2, buf)
if bytes != (69, 70):
print(buf)
return 'fail'
ds = None
gdal.Unlink('/vsimem/grid.xyz')
return 'success'
def plot(self, schema_table, cmap=None, density=False):
if density:
ds = self.mk_density(schema_table)
else:
self.cur.execute("""
SELECT ST_AsGDALRaster(ST_Union(rast), 'GTiff') FROM %s ;""" %
(schema_table))
vsipath = '/vsimem/from_postgis'
gdal.FileFromMemBuffer(vsipath, bytes(self.cur.fetchone()[0]))
ds = gdal.Open(vsipath)
for k, v in getinfo(ds).items():
print(k, ":", v)
nb = ds.RasterCount
for i in range(nb):
b = ds.GetRasterBand(i + 1)
print(band_info(b))
def read_one_band(b):
arr = b.ReadAsArray()
# need to flup upside down
arr = arr[::-1]
arr = np.ma.masked_values(arr, b.GetNoDataValue())
return arr
if nb == 1:
b = ds.GetRasterBand(1)
arr = read_one_band(b)
if isinstance(cmap, str):
cmap = clr_to_cmap(cmap, arr)
elif density:
cmap = plt.get_cmap('YlOrRd')
elif nb == 3:
arr = [read_one_band(ds.GetRasterBand(_ + 1)) for _ in range(3)]
arr = np.dstack(arr)
cmap = None
else:
raise RuntimeError('only 1 or 3 band raster suppoerted')
info = getinfo(ds)
plt.figure(figsize=(12, 6))
llcrnrlon = info['Origin'][0]
urcrnrlat = info['Origin'][1]
urcrnrlon = (info['Origin'][0] +
info['Pixel Size'][0] * info['Size'][0])
llcrnrlat = (info['Origin'][1] +
info['Pixel Size'][1] * info['Size'][1])
if urcrnrlon > 180:
if urcrnrlon - 180 < .1 * info['Pixel Size'][0]:
urcrnrlon = 180
else:
raise RuntimeError('urrcrnerlon: %s' % urcrnrlon)
if llcrnrlat < -90:
if llcrnrlat - 90 < abs(.1 * info['Pixel Size'][1]):
llcrnrlat = -90
else:
raise RuntimeError('urrcrnerlon: %s' % llcrnrlat)
m = Basemap(
# need to flip upside down
llcrnrlon=llcrnrlon,
urcrnrlat=urcrnrlat,
urcrnrlon=urcrnrlon,
llcrnrlat=llcrnrlat)
m.drawcoastlines()
print(arr.shape)
m.imshow(arr, cmap=cmap)
plt.show()
def _merge_tiles(tile_iter,
bbox,
size,
resolution,
t_srs='EPSG:3857',
t_format='GTiff',
t_name='out.tif',
mode='RGB',
gdal_translate_bin='/usr/bin/gdal_translate',
gdalwarp_bin='/usr/bin/gdalwarp'):
"""merges tiles given by tile_iter.
tile_iter: generator returning dict: {'data': ..., 'geotransform': ...}
bbox: bbox of all tiles delivered by tile_iter
size: size of output image. must be tuple of 2
t_srs: srs of output image
t_format: GDAL Raster Format descriptor
t_name: name of output image
"""
input_mem_buffer = '/vsimem/temp'
mode = 'RGB' if t_format == 'JPEG' else mode
gdal.AllRegister()
gdal_file_info = file_info()
fh, result_filename = mkstemp(suffix='.tif')
os.close(fh)
log.info('Created %s' % result_filename)
# find first tile to determine number of bands
first_tile = tile_iter.next()
gdal.FileFromMemBuffer(input_mem_buffer,
convert_to_bands(first_tile['data'], mode).read())
if gdal_file_info.init_from_name(
input_mem_buffer) == 1: # add else for error
dest = create_target_image(size[0],
size[1],
bbox,
resolution,
bands=gdal_file_info.bands,
band_type=gdal_file_info.band_type,
t_name=result_filename)
log.info(
'Created destination image %s. Size: %s x %s; Resolution: %s; Bands: %s, BoundingBox: %s'
% (result_filename, size[0], size[1], resolution,
gdal_file_info.bands, str(bbox)))
copy_into_wrapper(dest, gdal_file_info, first_tile['geotransform'])
gdal.Unlink(input_mem_buffer)
for tile in tile_iter:
gdal.FileFromMemBuffer(input_mem_buffer,
convert_to_bands(tile['data'], mode).read())
if gdal_file_info.init_from_name(
input_mem_buffer) == 1: # add else for error
copy_into_wrapper(dest, gdal_file_info, tile['geotransform'])
gdal.Unlink(input_mem_buffer)
#close created image otherwise gdalwarp/gdal_translate produce black images
del dest
if t_srs != 'EPSG:3857':
fh, reproject_filename = mkstemp(suffix='.tif')
os.close(fh)
log.info('Created %s' % reproject_filename)
reproject_image(gdalwarp_bin, result_filename, 'EPSG:3857',
reproject_filename, t_srs)
# os.remove(result_filename)
log.info('Removed %s' % result_filename)
result_filename = reproject_filename
if t_format != 'GTiff':
fh, convert_filename = mkstemp(suffix=splitext(t_name)[1])
os.close(fh)
log.info('Created %s' % convert_filename)
convert_image(gdal_translate_bin, result_filename,
convert_filename, t_format)
# os.remove(result_filename)
log.info('Removed %s' % result_filename)
result_filename = convert_filename
copy_files(result_filename, t_name)
def kmlsuperoverlay_4():
vrt_xml = """<VRTDataset rasterXSize="800" rasterYSize="400">
<SRS>GEOGCS["WGS 84",DATUM["WGS_1984",SPHEROID["WGS 84",6378137,298.257223563,AUTHORITY["EPSG","7030"]],AUTHORITY["EPSG","6326"]],PRIMEM["Greenwich",0],UNIT["degree",0.0174532925199433],AUTHORITY["EPSG","4326"]]</SRS>
<GeoTransform> -1.8000000000000000e+02, 4.5000000000000001e-01, 0.0000000000000000e+00, 9.0000000000000000e+01, 0.0000000000000000e+00, -4.5000000000000001e-01</GeoTransform>
<Metadata>
<MDI key="AREA_OR_POINT">Area</MDI>
</Metadata>
<Metadata domain="IMAGE_STRUCTURE">
<MDI key="INTERLEAVE">BAND</MDI>
</Metadata>
<VRTRasterBand dataType="Byte" band="1">
<Metadata />
<ColorInterp>Red</ColorInterp>
<SimpleSource>
<SourceFilename relativeToVRT="0">data/small_world.tif</SourceFilename>
<SourceBand>1</SourceBand>
<SourceProperties RasterXSize="400" RasterYSize="200" DataType="Byte" BlockXSize="400" BlockYSize="20" />
<SrcRect xOff="0" yOff="0" xSize="400" ySize="200" />
<DstRect xOff="0" yOff="0" xSize="800" ySize="400" />
</SimpleSource>
</VRTRasterBand>
<VRTRasterBand dataType="Byte" band="2">
<Metadata />
<ColorInterp>Green</ColorInterp>
<SimpleSource>
<SourceFilename relativeToVRT="0">data/small_world.tif</SourceFilename>
<SourceBand>2</SourceBand>
<SourceProperties RasterXSize="400" RasterYSize="200" DataType="Byte" BlockXSize="400" BlockYSize="20" />
<SrcRect xOff="0" yOff="0" xSize="400" ySize="200" />
<DstRect xOff="0" yOff="0" xSize="800" ySize="400" />
</SimpleSource>
</VRTRasterBand>
<VRTRasterBand dataType="Byte" band="3">
<Metadata />
<ColorInterp>Blue</ColorInterp>
<SimpleSource>
<SourceFilename relativeToVRT="0">data/small_world.tif</SourceFilename>
<SourceBand>3</SourceBand>
<SourceProperties RasterXSize="400" RasterYSize="200" DataType="Byte" BlockXSize="400" BlockYSize="20" />
<SrcRect xOff="0" yOff="0" xSize="400" ySize="200" />
<DstRect xOff="0" yOff="0" xSize="800" ySize="400" />
</SimpleSource>
</VRTRasterBand>
</VRTDataset>"""
gdal.FileFromMemBuffer("/vsimem/src.vrt", vrt_xml)
src_ds = gdal.Open("/vsimem/src.vrt")
ds = gdal.GetDriverByName('KMLSUPEROVERLAY').CreateCopy('/vsimem/kmlsuperoverlay_4.kmz', src_ds, options = ['FORMAT=PNG'])
if ds.GetRasterBand(1).GetOverviewCount() != 1:
gdaltest.post_reason('fail')
ds = None
src_ds = None
gdal.Unlink("/vsimem/src.vrt")
gdal.Unlink("/vsimem/kmlsuperoverlay_4.kmz")
return 'fail'
if ds.GetRasterBand(1).GetOverview(0).Checksum() != 30111:
gdaltest.post_reason('fail')
ds = None
src_ds = None
gdal.Unlink("/vsimem/src.vrt")
gdal.Unlink("/vsimem/kmlsuperoverlay_4.kmz")
return 'fail'
if ds.GetRasterBand(1).Checksum() != src_ds.GetRasterBand(1).Checksum():
gdaltest.post_reason('fail')
ds = None
src_ds = None
gdal.Unlink("/vsimem/src.vrt")
gdal.Unlink("/vsimem/kmlsuperoverlay_4.kmz")
return 'fail'
ds = None
src_ds = None
gdal.Unlink("/vsimem/src.vrt")
gdal.Unlink("/vsimem/kmlsuperoverlay_4.kmz")
return 'success'
def costRASTER_shortPATHs(outS):
# Functions
def ind_to_Xcoord(ind_x): # X-Coordinate from X-PixelIndex
coord_x = ind_x * cr_gt[1] + cr_gt[0] + cr_gt[1] / 2
return (coord_x)
def ind_to_Ycoord(ind_y): # Y-Coordinate from Y-PixelIndex
coord_y = ind_y * cr_gt[5] + cr_gt[3] + cr_gt[5] / 2
return (coord_y)
# PostgreSQL DB-Connection
conn = psycopg2.connect(
"host=??? port=??? dbname=??? user=??? password=???")
cursor = conn.cursor()
# http://www.postgis.net/docs/manual-dev/postgis_gdal_enabled_drivers.html
cursor.execute("""SET postgis.gdal_enabled_drivers = 'ENABLE_ALL';""")
conn.commit()
#
cursor.execute("""CREATE SCHEMA IF NOT EXISTS """ + str(outS) + """;""")
conn.commit()
pas = 10 # Number of PAS: '******' - PatchArrangementScenarios
for x in range(pas):
cursor.execute(
"""SELECT start, start_xy, aim, aim_xy, distance FROM pas.pas""" +
str(x) + """_ed;""")
pas_ed = cursor.fetchall()
pas_ed = [list(y) for y in pas_ed]
pas_ed = [[
y[0],
[
float(y[1].split(' ', 1)[0][6:]),
float(y[1].split(' ', 1)[1][:-1])
], y[2],
[
float(y[3].split(' ', 1)[0][6:]),
float(y[3].split(' ', 1)[1][:-1])
], y[4]
] for y in pas_ed]
p_con = [[y[2], y[0]] for y in pas_ed
], [[y[0], y[2]] for y in pas_ed
] # Patches connected: 'p_con' - Patches CONnected
for xx in range(
int(len(p_con[0]) / 2)
): # Remove equal, opposite Connections from 'pas_ed', e.g. 4-6 & 6-4
y = p_con[0][xx]
for z in range(len(p_con[0])):
yy = p_con[1][z]
if yy == y:
p_con[0].pop(z)
p_con[1].pop(z)
pas_ed.pop(z)
break
lus = 10 # Number of LUS: 'lus' - LandUseScenarios
for xx in range(lus):
for xxx in ('nlmr', 'nlmrc', 'nlmre'): # NLM types
vsipath = '/vsimem/from_postgis'
cursor.execute(
"""SELECT ST_AsGDALRaster(rast, 'GTiff') FROM lus_""" +
str(outS[4:]) + """.""" + str(xxx) + str(xx) + """_rs;""")
gdal.FileFromMemBuffer(vsipath, bytes(cursor.fetchone()[0]))
cost_raster = gdal.Open(vsipath)
gdal.Unlink(vsipath)
cr_array = cost_raster.ReadAsArray()
cr_array = cr_array * 25.0 # DC raster * DC
cr_gt = cost_raster.GetGeoTransform()
cursor.execute("""DROP TABLE IF EXISTS """ + str(outS) +
""".""" + str(xxx) + str(xx) + """_pas""" +
str(x) + """;""")
cursor.execute(
"""CREATE TABLE """ + str(outS) + """.""" + str(xxx) +
str(xx) + """_pas""" + str(x) +
""" (start bigint, aim bigint, geom geometry, costs double precision);"""
)
for xxxx in range(
len(pas_ed)
): # Processing START, AIM, GEOMETRY, COSTS per MPC
start_x = int((pas_ed[xxxx][1][0] - cr_gt[0]) /
cr_gt[1]) # START: X Coordinate to Index
start_y = int((pas_ed[xxxx][1][1] - cr_gt[3]) /
cr_gt[5]) # START: Y Coordinate to Index
aim_x = int((pas_ed[xxxx][3][0] - cr_gt[0]) /
cr_gt[1]) # AIM: X Coordinate to Index
aim_y = int((pas_ed[xxxx][3][1] - cr_gt[3]) /
cr_gt[5]) # AIM: Y Coordinate to Index
indices, weight = route_through_array(cr_array,
[start_x, start_y],
[aim_x, aim_y],
fully_connected=True)
indices = np.array(indices).T # PixelIndex
coor = []
mpc_nodes = ""
for xxxxx in range(len(indices[0])):
coor.append([
ind_to_Xcoord(indices[0][xxxxx]),
ind_to_Ycoord(indices[1][xxxxx])
]) # PixelIndex to Coordinate
mpc_nodes = mpc_nodes + "ST_MakePoint(" + str(
coor[xxxxx][0]) + ', ' + str(coor[xxxxx][1]) + "),"
mpc_nodes = mpc_nodes[:-1]
cursor.execute("""INSERT INTO """ + str(outS) + """.""" +
str(xxx) + str(xx) + """_pas""" + str(x) +
""" (start, aim, geom, costs) VALUES (""" +
str(pas_ed[xxxx][0]) + """, """ +
str(pas_ed[xxxx][2]) +
""", ST_SetSRID(ST_MakeLine(ARRAY[""" +
str(mpc_nodes) + """]), 25832), """ +
str(weight) + """);""")
cursor.execute("""UPDATE """ + str(outS) + """.""" +
str(xxx) + str(xx) + """_pas""" + str(x) +
""" SET costs = 50 WHERE costs = 0;""")
conn.commit()
cursor.close()
conn.close()
def generate_tiles(self):
"""
Function to generate the dynamic tiles (either merging multiple web tile
sources and/or extracting data from a local GIS data source
"""
# print('Content-Type: text/html\n')
start = time.time()
tz = int(self.tz)
tx = int(self.tx)
ty = int(self.ty)
# In case of inverted y coordinate
if self.invert_y:
ty2 = ty
else:
if type(ty) is int:
ty2 = (2**tz) - ty - 1
else:
ty2 = ty
tilefilename = os.path.join(self.cachedir, str(tz), str(tx),
"%s.%s" % (ty, self.tileext))
# Tile name used if tile is cached
if self.cachedir != '':
if os.path.exists(tilefilename):
im = Image.open(tilefilename)
if major == 2:
f = StringIO()
elif major == 3:
f = BytesIO()
im.save(f, "PNG")
f.seek(0)
return f.read()
print('Getting Raster ' + str(time.time() - start) + ' s')
if self.proj == 'geo':
s_srs = "+proj=merc +a=6378137 +b=6378137 +lat_ts=0.0 +lon_0=0.0 +x_0=0.0 +y_0=0 +k=1.0 +units=m +nadgrids=@null +wktext +no_defs"
t_srs = "+proj=latlong +datum=wgs84 +no_defs"
if self.resample == 'near':
self.ResampleAlg = gdal.GRA_NearestNeighbour
elif self.resample == 'bilinear':
self.ResampleAlg = gdal.GRA_Bilinear
if self.profile == 'mercator':
self.mercator = GlobalMercator()
self.tileswne = self.mercator.TileLatLonBounds
self.tilewsen_merc = self.mercator.TileBounds
south, west, north, east = self.tileswne(tx, ty, tz)
w, s, e, n = self.tilewsen_merc(tx, ty, tz)
raster_url = self.url
raster_url = raster_url.replace('{$x}', str(tx))
raster_url = raster_url.replace('{$y}', str(ty2))
raster_url = raster_url.replace('{$invY}', str(ty2))
raster_url = raster_url.replace('{$z}', str(tz))
try:
if major == 2:
f = StringIO(urllib.urlopen(raster_url).read())
elif major == 3:
f = BytesIO(urlopen(raster_url).read())
im = Image.open(f).convert('RGBA')
if major == 2:
f = StringIO()
elif major == 3:
f = BytesIO()
im.save(f, "PNG")
f.seek(0)
except:
im = Image.new('RGBA', (100, 100))
if major == 2:
f = StringIO()
elif major == 3:
f = BytesIO()
im.save(f, "PNG")
f.seek(0)
return f.read()
content = f.read()
gdal.FileFromMemBuffer('/vsimem/inmem', content)
src_ds = gdal.Open('/vsimem/inmem')
src_srs = osr.SpatialReference()
src_srs.ImportFromProj4(s_srs)
src_wkt = src_srs.ExportToWkt()
dst_srs = osr.SpatialReference()
dst_srs.ImportFromProj4(t_srs)
dst_wkt = dst_srs.ExportToWkt()
nx = src_ds.RasterXSize
ny = src_ds.RasterYSize
nb = src_ds.RasterCount
gt = [w, (e - w) / nx, 0, n, 0, (s - n) / ny]
src_ds.SetGeoTransform(gt)
reproj_ds = gdal.AutoCreateWarpedVRT(src_ds, src_wkt, dst_wkt,
self.ResampleAlg, 0)
if nb == 1:
im = self.arrayToImage(reproj_ds.GetRasterBand(1).ReadAsArray())
elif nb == 3:
r = self.arrayToImage(reproj_ds.GetRasterBand(1).ReadAsArray())
g = self.arrayToImage(reproj_ds.GetRasterBand(2).ReadAsArray())
b = self.arrayToImage(reproj_ds.GetRasterBand(3).ReadAsArray())
im = Image.merge("RGB", (r, g, b))
elif nb == 4:
r = self.arrayToImage(reproj_ds.GetRasterBand(1).ReadAsArray())
g = self.arrayToImage(reproj_ds.GetRasterBand(2).ReadAsArray())
b = self.arrayToImage(reproj_ds.GetRasterBand(3).ReadAsArray())
a = self.arrayToImage(reproj_ds.GetRasterBand(4).ReadAsArray())
im = Image.merge("RGBA", (r, g, b, a))
else:
error('Images must have 1, 3, or 4 bands')
if major == 2:
f = StringIO()
elif major == 3:
f = BytesIO()
im.save(f, "PNG")
print('Saving image ' + str(time.time() - start) + ' s')
# If specified, save a copy of the cached image
if self.cachedir != '':
print('Saving copy ' + str(time.time() - start) + ' s')
if not os.path.exists(os.path.dirname(tilefilename)):
os.makedirs(os.path.dirname(tilefilename))
im.save(tilefilename, "PNG")
gdal.Unlink('/vsimem/tiffinmem')
del (src_ds)
del (reproj_ds)
f.seek(0)
return f.read()
if not os.path.exists(buffer_name):
if geodb:
gdf = gpd.read_file(geodb, layer=roi, driver='FileGDB')
else:
gdf = gdp.read_file(roi)
gdf_proj = gdf.to_crs({'init': 'epsg:{}'.format(out_proj)})
gdf_buffer = gdf_proj.buffer(300 * 1000) #300 km in METERS
gdf_buffer.to_file(buffer_name, driver='GeoJSON')
#Open file in memory, crop and reproject
verboseprint("Starting read raster and clip by shapefile")
s2 = time.time()
tar = tarfile.open(fname, 'r:gz')
member = tar.getnames()[0] #this is the ave_rade9h.tif file
img_data = tar.extractfile(member)
gdal.FileFromMemBuffer('/vsimem/tiffinmem', img_data.read())
dataset = gdal.Open('/vsimem/tiffinmem', gdal.GA_ReadOnly)
gt_orig = dataset.GetGeoTransform() #for getting original raster resolution
x_orig_res, y_orig_res = gt_orig[1], -gt_orig[5]
#Reproject and clip raster in memory
OutTile = gdal.Warp('/vsimem/clipped_reprojected.tif',
dataset,
cutlineDSName=buffer_name,
xRes=450,
yRes=450,
srcSRS='EPSG:4326',
dstSRS='EPSG:' + str(out_proj),
cropToCutline=True,
dstNodata=-9999)
cellSIZE = raster_MD[4]
vsipath = '/vsimem/from_postgis'
cursor.execute("""SET postgis.gdal_enabled_drivers = 'ENABLE_ALL';""")
ds = band = None
gdal.Unlink(vsipath)
# ???
cursor.execute(
"""SELECT ST_AsGDALRaster(rast, 'GTiff') FROM stream_network.rlp_stream_rast_testarea_50x50;"""
)
gdal.FileFromMemBuffer(vsipath, bytes(cursor.fetchone()[0]))
ds = gdal.Open(vsipath)
band = ds.GetRasterBand(1)
STREAM01_ARRAY = band.ReadAsArray()
ds = band = None
gdal.Unlink(vsipath)
cursor.close()
conn.close()
### Variables definition
# Number of rastres produced per NLM algorithm
def read_zipped_tif(tif_name:str, zip_data:any):
"""Read a zipped tif into memory using gdal"""
tif_inmem = "/vsimem/data.tif" #Virtual Folder to Store Data
gdal.FileFromMemBuffer(tif_inmem, zip_data.open(tif_name).read())
src = gdal.Open(tif_inmem)
return src.GetRasterBand(1), src.GetGeoTransform(), src
def read_request(request, filename):
gdal.FileFromMemBuffer(filename, request.content)
return gdal.Open(filename, gdalconst.GA_ReadOnly)
