def write(geo_array,
file_name,
dtype=None,
nodata=None,
proj=None,
gdal_driver='GTiff',
overwrite=False):
if not overwrite and os.path.isfile(file_name):
raise IOError('file `{}` already exists.'.format(file_name))
if dtype is None:
dtype = geo_array.array.dtype
raster = gdal.GetDriverByName(gdal_driver).Create(
utf8_path=file_name,
xsize=geo_array.width,
ysize=geo_array.height,
bands=geo_array.bands,
eType=gdal_array.flip_code(numpy.dtype(dtype)))
if not raster:
raise IOError('an error occurred while creating gdal dataset.')
# resolve projection
if proj is None:
proj = geo_array.proj_origin
if proj is None:
proj = DEFAULT_PROJ
raster.SetProjection(proj)
raster.SetGeoTransform(geo_array.geotransform.to_list())
# convert buffer dtype to informed dtype parameter as to write data to raster.
buffer = geo_array.array.astype(dtype, copy=True)
byte_string = buffer.tostring()
# get buffer strides in order to finda data elements position in `byte_string`
height_size, width_size, bands_size = buffer.strides
raster.WriteRaster(xoff=0,
yoff=0,
xsize=geo_array.width,
ysize=geo_array.height,
buf_string=byte_string,
buf_xsize=geo_array.width,
buf_ysize=geo_array.height,
buf_type=gdal_array.flip_code(dtype),
buf_pixel_space=width_size,
buf_line_space=height_size,
buf_band_space=bands_size)
# 2-do: capture gdal errors and treat them appropriately.
raster.FlushCache()
if nodata is None:
nodata = geo_array.nodata
if nodata is not None:
for i in range(1, raster.RasterCount + 1):
raster.GetRasterBand(i).SetNoDataValue(nodata)
for i in range(1, raster.RasterCount + 1):
raster.GetRasterBand(i).ComputeStatistics(False)
raster.SetMetadata(geo_array.metadata)
def test_write_array_to_file(shape: Tuple[int],
dst_filename: Optional[PathLikeOrStr] = None):
np_dtype = np.float32
a = np.random.rand(*shape)
a = np.array(a, dtype=np_dtype)
if dst_filename is None:
dst_filename = tempfile.mktemp(suffix='.tif')
ds = write_array_to_file(dst_filename, a)
if len(shape) == 3:
a0 = a[0]
raster_count = shape[0]
y_size, x_size = shape[1:]
else:
a0 = a
raster_count = 1
y_size, x_size = shape
assert ds.RasterCount == raster_count
assert ds.RasterXSize == x_size
assert ds.RasterYSize == y_size
bnd: gdal.Band = ds.GetRasterBand(1)
assert bnd.DataType == gdal_array.flip_code(a.dtype)
b = bnd.ReadAsArray()
assert_almost_equal(a0, b)
ds = None
gdal.Unlink(dst_filename)
def command(source_path, raster_path, target_path, attribute, partial):
""" Main """
if partial is None:
source_features = common.Source(source_path)
else:
source_features = common.Source(source_path).select(partial)
raster = gdal.Open(raster_path)
geo_transform = GeoTransform(raster.GetGeoTransform())
sr = osr.SpatialReference(raster.GetProjection())
band = raster.GetRasterBand(1)
dtype = gdal_array.flip_code(band.DataType)
no_data_value = np.array(band.GetNoDataValue(), dtype).item()
# prepare statistics gathering
target = common.Target(
path=target_path,
template_path=source_path,
attributes=[attribute],
)
for source_feature in source_features:
geometry = source_feature.geometry()
geometry.TransformTo(sr)
xoff, yoff = geo_transform.get_centroid_indices(geometry)
value = band.ReadAsArray(xoff, yoff, 1, 1).item()
attributes = {attribute: None if value == no_data_value else value}
target.append(geometry=geometry,
attributes=attributes)
return 0
def replace(source_path, target_path):
source_dataset = gdal.Open(source_path)
source_values = source_dataset.ReadAsArray()
source_band = source_dataset.GetRasterBand(1)
source_no_data_value = source_band.GetNoDataValue()
data_type = source_band.DataType
dtype = gdal_array.flip_code(data_type)
target_no_data_value = np.finfo(dtype).max.item()
target_values = np.where(
source_values == source_no_data_value,
target_no_data_value,
source_values,
)
target_dataset = DRIVER_MEM.CreateCopy('', source_dataset)
target_band = target_dataset.GetRasterBand(1)
target_band.WriteArray(target_values)
target_band.SetNoDataValue(target_no_data_value)
options = ['compress=deflate', 'tiled=yes']
gdal.GetDriverByName(str('gtiff')).CreateCopy(
target_path, target_dataset, options=options,
)
def write_array_to_file(dst_filename: PathLikeOrStr,
a: MaybeSequence[np.ndarray],
gdal_dtype=None) -> gdal.Dataset:
driver_name = GetOutputDriverFor(dst_filename, is_raster=True)
driver = gdal.GetDriverByName(driver_name)
a_shape = a[0].shape
if len(a_shape) == 1:
# 2d array, singleband raster
a = [a]
bands_count = 1
elif len(a_shape) == 2:
# 3d array, multiband raster
bands_count = a.shape[0]
else:
raise Exception('Array should have 2 or 3 dimensions')
y_size, x_size = a[0].shape
if gdal_dtype is None:
np_dtype = a[0].dtype
gdal_dtype = gdal_array.flip_code(np_dtype)
ds = driver.Create(dst_filename, x_size, y_size, bands_count, gdal_dtype)
if ds is None:
raise Exception(f'failed to create: {dst_filename}')
for bnd_num in range(bands_count):
bnd = ds.GetRasterBand(bnd_num + 1)
if gdal_array.BandWriteArray(bnd, a[bnd_num], xoff=0, yoff=0) != 0:
raise Exception('I/O error')
return ds
def point_from_dataset(dataset, point):
x, y = point
p, a, b, q, c, d = dataset.GetGeoTransform()
minv = np.linalg.inv([[a, b], [c, d]])
u, v = np.dot(minv, [x - p, y - q])
band_list = range(1, dataset.RasterCount + 1)
data = dataset.ReadRaster(int(u), int(v), 1, 1, band_list=band_list)
data_type = gdal_array.flip_code(dataset.GetRasterBand(1).DataType)
return np.fromstring(data, dtype=data_type)
def get_data_for_polygon(self, wkb, crs, size):
"""
Return a numpy array for the data.
"""
# Quick out if polygon bounds match polygon
geometry = vectors.Geometry(wkb)
envelope = geometry.envelope
extent = geometry.extent
nodatavalue = self.manager.no_data_value
datatype = self.manager.data_type
# Initialize resulting array to nodatavalue
dtype = gdal_array.flip_code(datatype)
array = np.ones(
(1, size[1], size[0]),
dtype=dtype,
) * dtype(nodatavalue)
# Create dataset and use it to retrieve data from the store
array_dict = dict(
array=array,
no_data_value=nodatavalue,
projection=projections.get_wkt(crs),
geo_transform=rasters.get_geotransform(
extent=extent, width=size[0], height=size[1],
),
)
dataset = rasters.dict2dataset(array_dict)
self.warpinto(dataset)
dataset.FlushCache()
# Cut when necessary
if not envelope.Equals(wkb):
source = OGR_MEM_DRIVER.CreateDataSource('')
sr = projections.get_spatial_reference(crs)
layer = source.CreateLayer(b'', sr)
defn = layer.GetLayerDefn()
feature = ogr.Feature(defn)
feature.SetGeometry(envelope.Difference(wkb))
layer.CreateFeature(feature)
gdal.RasterizeLayer(dataset, (1,), layer,
burn_values=(nodatavalue,))
dataset.FlushCache()
return np.ma.masked_equal(array, nodatavalue, copy=False)
def read_as_shared_array(dataset):
"""
Return numpy array.
Puts the array data in shared memory for use with multiprocessing.
"""
# Derive array properties from the dataset
dtype = np.dtype(gdal_array.flip_code(
dataset.GetRasterBand(1).DataType,
))
shape = (dataset.RasterCount,
dataset.RasterYSize,
dataset.RasterXSize)
size = shape[0] * shape[1] * shape[2] * dtype.itemsize
# Create shared memory array and read into it
return dataset.ReadAsArray(buf_obj=np.frombuffer(
multiprocessing.RawArray('b', size), dtype
).reshape(*shape))
def test_gdal2xyz_py_1():
""" test get_ovr_idx, create_flat_raster """
pytest.importorskip('numpy')
size = (3, 3)
origin = (500_000, 0)
pixel_size = (10, -10)
nodata_value = 255
band_count = 2
dt = gdal.GDT_Byte
np_dt = flip_code(dt)
ds = create_flat_raster(filename='',
size=size,
origin=origin,
pixel_size=pixel_size,
nodata_value=nodata_value,
fill_value=nodata_value,
band_count=band_count,
dt=dt)
src_nodata = nodata_value
np.random.seed()
for bnd_idx in range(band_count):
bnd = ds.GetRasterBand(bnd_idx + 1)
data = (np.random.random_sample(size) * 255).astype(np_dt)
data[1, 1] = src_nodata
bnd.WriteArray(data, 0, 0)
dst_nodata = 254
for pre_allocate_np_arrays, skip_nodata in product((True, False),
(True, False)):
geo_x, geo_y, data, nodata = \
gdal2xyz.gdal2xyz(
ds, None, return_np_arrays=True,
src_nodata=src_nodata, dst_nodata=dst_nodata,
skip_nodata=skip_nodata, pre_allocate_np_arrays=pre_allocate_np_arrays)
_pixels, _lines, data2 = gdallocationinfo.gdallocationinfo(
ds,
x=geo_x,
y=geo_y,
resample_alg=gdal.GRIORA_NearestNeighbour,
srs=gdallocationinfo.LocationInfoSRS.SameAsDS_SRS)
data2[data2 == src_nodata] = dst_nodata
assert np.all(np.equal(data, data2))
def point_from_dataset(animation, framenr, point):
x, y = point
p, a, b, q, c, d = animation.get_geotransform()
minv = np.linalg.inv([[a, b], [c, d]])
u, v = np.dot(minv, [x - p, y - q])
u = int(u)
v = int(v)
if not ((0 <= u < animation.cols) and (0 <= v < animation.rows)):
# Outside animation
return None
dataset = gdal.Open(animation.get_dataset_path(framenr))
if dataset is None:
return None
data = dataset.ReadRaster(u, v, 1, 1, band_list=[1])
data_type = gdal_array.flip_code(dataset.GetRasterBand(1).DataType)
return np.fromstring(data, dtype=data_type)[0]
def read_block(self, block):
""" Read a block as a dataset. """
# read the data
u1, u2, v1, v2 = self.get_offsets(block)
band_list = range(1, self.raster_count + 1)
array = np.fromstring(self.dataset.ReadRaster(
u1, v1, u2 - u1, v2 - v1, band_list=band_list,
), dtype=gdal_array.flip_code(self.data_type))
array.shape = self.raster_count, v2 - v1, u2 - u1
# dataset properties
p, a, b, q, c, d = self.geo_transform
geo_transform = p + a * u1 + b * v1, a, b, q + c * u1 + d * v1, c, d
dataset = dict2dataset(dict(
array=array,
geo_transform=geo_transform,
projection=self.projection,
no_data_value=self.no_data_value,
))
return dict(dataset=dataset, array=array)
def point_from_dataset(animation, framenr, point):
x, y = point
p, a, b, q, c, d = animation.get_geotransform()
minv = np.linalg.inv([[a, b], [c, d]])
u, v = np.dot(minv, [x - p, y - q])
u = int(u)
v = int(v)
if not ((0 <= u < animation.cols) and (0 <= v < animation.rows)):
# Outside animation
return None
dataset = gdal.Open(animation.get_dataset_path(framenr))
if dataset is None:
return None
data = dataset.ReadRaster(u, v, 1, 1, band_list=[1])
data_type = gdal_array.flip_code(dataset.GetRasterBand(1).DataType)
return np.fromstring(data, dtype=data_type)[0]
def get_dtype(dataset):
""" Return the numpy dtype. """
return np.dtype(gdal_array.flip_code(
dataset.GetRasterBand(1).DataType,
))
def lutx(lut, input, output, format):
"""
Reads in input image, applies lut and outputs image
"""
# Read in input image
gdal.AllRegister()
src_ds = gdal.Open(input, GA_ReadOnly)
if src_ds is None:
print 'Error: could not open {0}'.format(input)
rows = src_ds.RasterYSize
cols = src_ds.RasterXSize
# Read in data
band = src_ds.GetRasterBand(1)
dtype = band.DataType
data = band.ReadAsArray(0, 0, cols,
rows).astype(gdal_array.flip_code(dtype))
# Determine requiredo output datatype
out_dt = np.byte
if np.min(lut.values()) < 0:
# Must be signed int
if np.max(np.abs(lut.values())) < 2**15:
# NOTE: put np.int8 as np.int16 since GDAL has no int8
out_dt = np.int16
elif np.max(np.abs(lut.values())) < 2**31:
out_dt = np.int32
elif np.max(np.abs(lut.values())) < 2**63:
out_dt = np.int64
else:
print 'Required output data type is unknown'
sys.exit(1)
else:
# Can be unsigned
if np.max(lut.values()) < 2**8:
out_dt = np.uint8
elif np.max(lut.values()) < 2**16:
out_dt = np.uint16
elif np.max(lut.values()) < 2**32:
out_dt = np.uint32
elif np.max(lut.values()) < 2**64:
out_dt = np.uint64
else:
print 'Required output data type is unknown'
sys.exit(1)
if DEBUG:
print 'NumPy data type: %s' % str(out_dt)
print 'GDAL data type: %s' % str(
gdal.GetDataTypeName(gdal_array.flip_code(out_dt)))
# Copy data for output
lutdata = data.copy().astype(out_dt)
# Apply lut
for key, value in lut.iteritems():
np.place(lutdata, data == key, value)
# Write to output
driver = gdal.GetDriverByName(format)
dst_ds = driver.Create(output, src_ds.RasterXSize, src_ds.RasterYSize, 1,
gdal_array.flip_code(out_dt))
dst_ds.SetProjection(src_ds.GetProjection())
dst_ds.SetGeoTransform(src_ds.GetGeoTransform())
dst_ds.GetRasterBand(1).WriteArray(lutdata)
# Close
src_ds = None
dst_ds = None
print 'Wrote output to file {0}'.format(output)
def save_raster(outfile, data, info=None):
"""open a GDAL-readable raster file
returns a pair of (metadata,array)
"""
proj4txt = (info and info['grid'].srs())
geoxf = (info and info['grid'].geoxf())
meta = None # XXX fixme
dirname,fname = os.path.split(os.path.abspath(outfile))
if not os.path.isdir(dirname):
os.makedirs(dirname)
if len(data.shape) == 3:
ysize,xsize,nbands = data.shape
else:
ysize,xsize = data.shape
nbands = 1
# get the gdal datatype from the numpy datatype
gdtype = gdal_array.flip_code(data.dtype.type)
# do not add COMPRESS=DEFLATE here
# generate all TIFFs in blocks for faster random access
blocksize = 128
opts = [
'TILED=YES',
'BLOCKXSIZE=%d' % blocksize,
'BLOCKYSIZE=%d' % blocksize
]
#print 'SAVE %r %r' % (outfile, (data.shape, blocksize, opts))
gtiff = gdal.GetDriverByName('GTiff')
#print 'GDTY %r %r %r' % (data.dtype, gdtype, gdalconst.GDT_Byte)
ds = gtiff.Create(str(outfile), xsize, ysize, nbands,
gdtype, opts)
if proj4txt is not None:
srs = osr.SpatialReference()
srs.ImportFromProj4(proj4txt)
ds.SetProjection(srs.ExportToWkt())
if geoxf is not None:
ds.SetGeoTransform(geoxf)
if meta is not None:
ds.SetMetadata(meta)
for i in range(nbands):
rband = ds.GetRasterBand(i+1)
if nbands == 1:
rband.SetRasterColorInterpretation(gdalconst.GCI_GrayIndex)
elif i == 0:
rband.SetRasterColorInterpretation(gdalconst.GCI_RedBand)
elif i == 1:
rband.SetRasterColorInterpretation(gdalconst.GCI_GreenBand)
elif i == 2:
rband.SetRasterColorInterpretation(gdalconst.GCI_BlueBand)
elif i == 3:
rband.SetRasterColorInterpretation(gdalconst.GCI_AlphaBand)
#rband.SetNoDataValue(xxx):
bdata = data
if nbands > 1:
bdata = data[:,:,i]
# transform data to unsigned byte?
print 'DATA %r: %f < x < %f' % (outfile, N.min(bdata), N.max(bdata))
rband.WriteArray(bdata)
def lutx(lut, input, output, format):
"""
Reads in input image, applies lut and outputs image
"""
# Read in input image
gdal.AllRegister()
src_ds = gdal.Open(input, GA_ReadOnly)
if src_ds is None:
print 'Error: could not open {0}'.format(input)
rows = src_ds.RasterYSize
cols = src_ds.RasterXSize
# Read in data
band = src_ds.GetRasterBand(1)
dtype = band.DataType
data = band.ReadAsArray(0, 0, cols, rows).astype(
gdal_array.flip_code(dtype))
# Determine requiredo output datatype
out_dt = np.byte
if np.min(lut.values()) < 0:
# Must be signed int
if np.max(np.abs(lut.values())) < 2 ** 15:
# NOTE: put np.int8 as np.int16 since GDAL has no int8
out_dt = np.int16
elif np.max(np.abs(lut.values())) < 2 ** 31:
out_dt = np.int32
elif np.max(np.abs(lut.values())) < 2 ** 63:
out_dt = np.int64
else:
print 'Required output data type is unknown'
sys.exit(1)
else:
# Can be unsigned
if np.max(lut.values()) < 2 ** 8:
out_dt = np.uint8
elif np.max(lut.values()) < 2 ** 16:
out_dt = np.uint16
elif np.max(lut.values()) < 2 ** 32:
out_dt = np.uint32
elif np.max(lut.values()) < 2 ** 64:
out_dt = np.uint64
else:
print 'Required output data type is unknown'
sys.exit(1)
if DEBUG:
print 'NumPy data type: %s' % str(out_dt)
print 'GDAL data type: %s' % str(
gdal.GetDataTypeName(gdal_array.flip_code(out_dt)))
# Copy data for output
lutdata = data.copy().astype(out_dt)
# Apply lut
for key, value in lut.iteritems():
np.place(lutdata, data == key, value)
# Write to output
driver = gdal.GetDriverByName(format)
dst_ds = driver.Create(output,
src_ds.RasterXSize, src_ds.RasterYSize, 1,
gdal_array.flip_code(out_dt))
dst_ds.SetProjection(src_ds.GetProjection())
dst_ds.SetGeoTransform(src_ds.GetGeoTransform())
dst_ds.GetRasterBand(1).WriteArray(lutdata)
# Close
src_ds = None
dst_ds = None
print 'Wrote output to file {0}'.format(output)
def read(raster,
geom=None,
band_list=None,
dtype=None,
resample_alg=GRIORA_NearestNeighbour):
raster = _open_raster(raster)
if not isinstance(raster, gdal.Dataset):
raise ValueError('invalid informed `raster` argument.')
if raster.RasterCount < 1:
raise ValueError('informed `raster` has no bands')
# calculate the interested area
geotransform = GeoTransform(*raster.GetGeoTransform())
if geom is None:
x_off, y_off, x_size, y_size = 0, 0, raster.RasterXSize, raster.RasterYSize
else:
geom_raster = geotransform.transform(
geometry.box(0, 0, raster.RasterXSize, raster.RasterYSize))
intersection = geom_raster.intersection(geom)
if intersection.is_empty:
raise ValueError('`geom` do not intersects with `raster`.')
# get offset to read from raster
x_off, y_off, x_size, y_size = tuple(
map(int,
geotransform.get_inverse().transform(intersection).bounds))
x_size, y_size = x_size - x_off, y_size - y_off
# update resulting geotransform
x_ul, y_min, _, y_max = intersection.exterior.bounds
y_ul = y_max if geotransform.y_res < 0 else y_min
geotransform = GeoTransform(x_ul, geotransform.x_res,
geotransform.x_skew, y_ul,
geotransform.y_skew, geotransform.y_res)
bands = raster.RasterCount if band_list is None else len(band_list)
# prepare raster buffer: get buffer strides
if dtype is None:
dtype = gdal_array.flip_code(raster.GetRasterBand(1).DataType)
sizeof_dtype = numpy.dtype(dtype).itemsize
line_space, pixel_space, band_space = sizeof_dtype * x_size * bands, sizeof_dtype * bands, sizeof_dtype
# read to buffer
buffer = raster.ReadRaster(xoff=x_off,
yoff=y_off,
xsize=x_size,
ysize=y_size,
buf_type=gdal_array.flip_code(
numpy.dtype(dtype)),
buf_xsize=x_size,
buf_ysize=y_size,
band_list=band_list,
buf_pixel_space=pixel_space,
buf_line_space=line_space,
buf_band_space=band_space,
resample_alg=resample_alg)
array = numpy.fromstring(buffer, dtype=dtype)
array.shape = (y_size, x_size, -1)
return GeoArray(array,
geotransform,
proj_origin=raster.GetProjection(),
**raster.GetMetadata_Dict())
