def polygonize(input_file, output_file, proj):
"""
Polygonise a raster file
Parameters
----------
input_file: str Path to input raster
output_file: str Path to output vector
proj: str Projection
"""
with buzz.Dataset(sr_work=proj, sr_fallback="WGS84").close as ds:
ds.open_raster("raster", input_file)
if os.path.isfile(output_file):
os.remove(output_file)
fields = [{"name": "class", "type": np.int32}]
ds.create_vector("vector",
output_file,
"polygon",
driver="geojson",
fields=fields)
fp = ds["raster"].fp
mask = ds["raster"].get_data()
for class_idx in np.unique(mask):
if class_idx != 0:
polygons = fp.find_polygons(mask == class_idx)
if not polygons:
continue
for poly in polygons:
ds["vector"].insert_data(poly, {"class": class_idx})
def predict_from_file(rgb_path,
model,
downsampling_factor=1,
tile_size=256,
no_of_gpu=1,
batch_size=2):
ds_rgb = buzz.Dataset(allow_interpolation=True)
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
overlapx = int(tile_size / 2)
overlapy = int(tile_size / 2)
try:
predicted_probamap = predict_map(model, ds_rgb, fp, tile_size,
overlapx, overlapy, batch_size)
except Exception as e:
print(e)
print("Retrying prediction with reduced batch size")
predicted_probamap = predict_map(model, ds_rgb, fp, tile_size,
overlapx, overlapy, batch_size // 2)
return predicted_probamap, fp
def write_tif_from_fp(array,
tile_id,
build_dataset_dir,
output_dir,
prefix=""):
"""
Args:
array:
tile_id:
build_dataset_dir:
output_dir:
prefix:
Returns:
"""
assert ".tif" in tile_id, "wrong tile id should en with tif"
ds_tile = buzz.Dataset()
output_path = "{}{}_image{}".format(output_dir, prefix, tile_id)
fp_tile = get_fp(tile_id, build_dataset_dir)
with ds_tile.acreate_raster(
output_path,
fp_tile,
'float32',
channel_count=DICT_SHAPE[LABEL_DIR][-1]).close as cache:
cache.set_data(array)
return output_path
def generate_dsm(rsize=(16000, 16000),
resolution=0.03,
delta_z=(10, 110),
roughness=0.45,
nb_houses=5,
verbose=False):
"""generates a `.tif` file containing an artificial dsm.
the generated dsm does not have a projection, nor no_data values.
Parameters
----------
- rsize: (int, int)
desired raster size
- resolution: (float)
desired resolution in meters
- delta_z: (float, float)
span of elevations on generated dsm in meters
- roughness: float
value ranging from 0 to 1. 0 will generate a very smooth dsm (a plane). 0.5 generates
midly rough landscapes (imagine a valley in the Swiss Alps). 1 generates very sharp results.
- nb_houses: int
number of houses to be added on the dsm
- verbose: boolean
if True, some information about the generated dsm will be printed.
"""
w, l = rsize
min_z, max_z = delta_z
if verbose:
print("==== Metrics on generated dsm ====")
print(f" w, l = {w}, {l}")
print(f" resolution = {resolution}")
print(f" roughness = {roughness}")
print(f" min_z, max_z = {min_z}, {max_z}")
print(f" nb_houses = {nb_houses}")
dsm = diamond_square((l, w), min_z, max_z, roughness)
for _ in range(nb_houses):
_put_house_on_dsm(dsm, resolution, verbose)
tlx = np.random.uniform(42, 1337)
tly = np.random.uniform(32, 111)
fp = buzz.Footprint(tl=(tlx, tly),
size=(w * resolution, l * resolution),
rsize=(w, l))
ds = buzz.Dataset(allow_interpolation=False)
filename = f'{uuid.uuid4()}.tif'
if verbose:
print(f' {fp}')
print(' filename = ' + filename)
with ds.acreate_raster(filename,
fp,
dtype='float32',
channel_count=1,
sr=None).close as out:
out.set_data(dsm)
return filename
def main():
path = example_tools.create_random_elevation_gtiff()
ds = buzz.Dataset(allow_interpolation=True)
print('Classic opening')
# Features:
# - Disk reads are not tiled
# - Resampling operations are not tiled
with ds.aopen_raster(path).close as r:
test_raster(r)
return # The NEXT features are not yet implemented
print('Opening within scheduler')
# Features:
# - Disk reads are automatically tiled and parallelized
# - Resampling operations are automatically tiled and parallelized
# - `iter_data()` method is available
with ds.aopen_raster(path, async_=True).close as r:
# `async_=True` is equivalent to
# `async_={}`, and also equivalent to
# `async_={io_pool='io', resample_pool='cpu', max_resampling_size=512, max_read_size=512}`
test_raster(r)
# `Dataset.close()` closes all rasters, the scheduler, and the pools.
# If you let the garbage collector collect the `Dataset`, the rasters and
# the scheduler will be correctly closed, but the pools will leak memory.
ds.close()
os.remove(path)
def _reproj_and_clip_vector(srcpath, dstpath):
print("Transforming: {}".format(srcpath), flush=True)
if os.path.isfile(dstpath):
print(" {} already exits".format(dstpath), flush=True)
return
with buzz.Env(significant=10, allow_complex_footprint=True):
ds = buzz.Dataset(sr_work='WGS84', analyse_transformation=False)
ds.open_vector('src', srcpath, driver='GeoJSON')
ds.open_raster('raster', rgb_wgs84_path)
ds.create_vector(
'dst',
dstpath,
'polygon',
driver='ESRI Shapefile',
sr=EPSG29100,
)
# Iterate over all geoJSON geometries overlapping with raster
# Ignoring geoJSON fields
# Save all polygons to Shapefile
for geom in ds.src.iter_data(None, mask=ds.raster.fp, clip=True):
if isinstance(geom, shapely.geometry.Polygon):
ds.dst.insert_data(geom)
elif isinstance(geom, shapely.geometry.MultiPolygon):
for poly in geom.geoms:
ds.dst.insert_data(poly)
print("Transformed: {}\n to: {}".format(srcpath, dstpath), flush=True)
def get_fp(tile_id, build_dataset_dir):
""":param str the image id
:returns a buzzard fp"""
image_path = find_path(build_dataset_dir + XDIR + DICT_ORGA[XDIR][0],
tile_id)
ds = buzz.Dataset(allow_interpolation=True)
ds.open_raster('tile', image_path)
return ds.tile.fp
def load_tile_classif(input_dataset,list_id_tile,path_classif_tif,max_im):
""":param input_dataset : str path to the dataset which contains .tif
:param list_id_tile : list of tile id
:param path_classif_tif : str path to the landclass rejorj tiff, be careful that the projections match with the tiles tiff
:param max_im an int : if < len tile id reduced the size of the output to max_im
:returns a list of array which are land classification for each tile"""
assert os.path.isfile(path_classif_tif),"No tif at {}".format(path_classif_tif)
if max_im<len(list_id_tile):
list_id_tile=list_id_tile[:max_im]
batch_landclass=[]
ds = buzz.Dataset(allow_interpolation=True)
ds_tile=buzz.Dataset()
ds.open_raster('land_class',path_classif_tif)
for i,image_id in enumerate(list_id_tile):
path=find_path(input_dataset + XDIR + DICT_ORGA[XDIR][0], image_id)
with ds_tile.open_raster('tile',path).close:
fp_tile=ds_tile.tile.fp
tile_land_class=ds.land_class.get_data(fp=fp_tile)
batch_landclass+=[tile_land_class]
return batch_landclass
def tif2_path(fps):
"""Create a tif in SR2 with all single letter footprints from `fps` fixture burnt in it"""
path = '{}/{}.tif'.format(tempfile.gettempdir(), uuid.uuid4())
ds = buzz.Dataset()
with ds.acreate_raster(path, fps.AI, 'int32', 1, sr=SR2['wkt']).close as r:
for letter in string.ascii_uppercase[:9]:
fp = fps[letter]
arr = np.full(fp.shape, ord(letter), dtype=int)
r.set_data(arr, fp=fp)
yield path
gdal.GetDriverByName('GTiff').Delete(path)
def shp2_path(fps):
"""Create a shapefile in SR2 containing all single letter polygons from `fps` fixture"""
path = '{}/{}.shp'.format(tempfile.gettempdir(), uuid.uuid4())
ds = buzz.Dataset()
ds.create_vector('poly', path, 'polygon', sr=SR2['wkt'])
for letter in string.ascii_uppercase[:9]:
ds.poly.insert_data(fps[letter].poly)
ds.poly.close()
del ds
yield path
gdal.GetDriverByName('ESRI Shapefile').Delete(path)
def _save_output(self, data, output, nodata_mask, fp):
input_file = data["input_files"][0]
output_file = data["output_file"]
if self.ssh_server:
input_file = self.ssh_server.tmp_file(input_file)
output_file = self.ssh_server.tmp_file(output_file)
with buzz.Dataset().close as ds:
proj = ds.aopen_raster(input_file).proj4_virtual
if os.path.isfile(output_file):
ds.open_raster("output", output_file, mode="w")
if not ds.output.fp.poly.contains(fp.poly):
ds.output.close()
os.remove(output_file)
ds.create_raster(
"output",
output_file,
self.original_fp,
dtype=np.uint8,
channel_count=1,
sr=proj,
channels_schema={"nodata": self.n_classes},
)
else:
ds.create_raster(
"output",
output_file,
self.original_fp,
dtype=np.uint8,
channel_count=1,
sr=proj,
channels_schema={"nodata": self.n_classes},
)
if nodata_mask:
subslice = fp.slice_in(self.original_fp)
nodata_mask = nodata_mask[subslice]
output[nodata_mask] = self.n_classes
ds.output.set_data(output.astype(np.uint8), fp, channels=0)
ds.output.close()
if self.ssh_server:
self.ssh_server.put(data["output_file"])
if data["polygonize"]:
poylgon_file = (
data["polygonize"]
if not self.ssh_server
else self.ssh_server.tmp_file(data["polygonize"])
)
polygonize(output_file, poylgon_file, proj)
if self.ssh_server:
self.ssh_server.put(data["polygonize"])
def _test():
ds = buzz.Dataset()
assert len(ds) == len(ws) == 0
prox = ds.create_vector('test', random_path_shp, 'polygon')
ws.add(prox)
assert len(ds) == len(ws) == 1
prox = ds.acreate_vector('', 'polygon', driver='Memory')
ws.add(prox)
assert len(ds) == len(ws) == 2
ws.add(ds)
assert len(ws) == 3
def vec_to_list(input_vec, n_classes, fp, dist_map=False):
"""
Rasterize a vector file.
Assume that geometries have a field `class`.
Return a list of one hot encoding rasters for each class.
input_ortho is used to delimit the boundaries of the raster.
Only accept shapefiles and geojson as input for the vector.
--------------
Parameters:
dist_map: bool Dilate points and apply distance transform
"""
with buzz.Dataset(allow_none_geometry=True).close as ds:
ext = input_vec.split(".")[1]
if ext == "geojson":
driver = "geojson"
elif ext == "shp":
driver = "ESRI Shapefile"
else:
raise Exception(
"Wrong kind of input vector. Expect either geojson or shapefile."
)
ds.open_vector("poly", input_vec, driver=driver)
rasters = [
np.zeros(fp.shape, dtype=np.uint8) for i in range(n_classes)
]
# TODO iter_data warning
for geometry, class_idx in ds.poly.iter_data("class"):
if isinstance(geometry, shapely.geometry.point.Point):
burned_point = np.asarray(
fp.spatial_to_raster(
np.asarray(geometry.xy).transpose((1, 0)))[0])
if (0 < burned_point[1] < rasters[0].shape[0]
and 0 < burned_point[0] < rasters[0].shape[1]):
rasters[class_idx][burned_point[1], burned_point[0]] = 1
elif isinstance(geometry, shapely.geometry.polygon.Polygon):
burned_polygon = fp.burn_polygons(geometry)
rasters[class_idx][burned_polygon] = 1
kernel = (cv.getStructuringElement(cv.MORPH_ELLIPSE,
(10, 10)) if dist_map else np.ones(
(3, 3), dtype=np.uint8))
rasters = [cv.dilate(raster, kernel) for raster in rasters]
if dist_map:
rasters = [
cv.distanceTransform(raster.astype(np.uint8), cv.DIST_L2, 3)
for raster in rasters
]
rasters = [raster[np.newaxis] for raster in rasters]
return rasters
def tiling_fromgrid(path_tif, path_grid_geojson, output_dir):
grid = load_grid_geojson(path_grid_geojson)
ds = buzz.Dataset()
r = ds.open_raster("raster", path_tif)
v = ds.open_vector("vect", path_grid_geojson, driver='GeoJSON')
print(type(v))
print(v.get_keys())
for i, poly in enumerate(v.iter_data()):
# Compute the Footprint bounding `poly`
print(poly)
print(type(poly))
print(grid)
#print(poly["location"])
fp = r.fp.intersection(poly)
land_class_tile = r.get_data(fp=fp)
def test_value_error(path):
ds = buzz.Dataset()
v = ds.acreate_vector(path, type='polygon')
with pytest.raises(ValueError, match='geom_type'):
list(v.iter_data(geom_type=''))
with pytest.raises(TypeError, match='slicing'):
list(v.iter_data(slicing=0))
with pytest.raises(ValueError, match='clip'):
list(v.iter_data(clip=True))
with pytest.raises(TypeError, match='slicing'):
list(v.iter_geojson(slicing=0))
with pytest.raises(ValueError, match='clip'):
list(v.iter_geojson(clip=True))
with pytest.raises(TypeError, match='a'):
v.insert_data(42)
def test_run(path, driver, fps, test_fields, test_coords_insertion):
# Step 1 - Build file according to fixture parameters **********************
ds = buzz.Dataset()
if test_fields:
fields = FIELDS
else:
fields = []
geom_type = 'polygon'
v = ds.create_avector(path,
geom_type,
fields,
driver=driver,
sr=SRS[0]['wkt'])
def _build_test_data():
"""Create test data of random format"""
rng = np.random.RandomState(42)
for fpname, fp in fps.items():
geom = _geom_of_fp(rng, fp, test_coords_insertion)
# Keep `_fields_of_fp` invocation before `if` for rng side effect
fields = _fields_of_fp(rng, fp, fpname)
if test_fields:
yield geom, fields
else:
yield geom,
data = list(_build_test_data())
for dat in data:
v.insert_data(*dat)
if driver in {'GeoJson', 'DXF'}:
# Flushing to disk for geojson
v.deactivate()
v.activate()
# Step 2 - Test geometries read routines ***********************************
_test_geom_read(v, fps, data, test_fields)
# Step 3 - Test field read routines ****************************************
if test_fields:
_test_fields_read(v, data)
v.close()
def _test():
ds = buzz.Dataset()
assert len(ds) == len(ws) == 0
prox = ds.create_raster('test', random_path_tif, fps.A, float, 1)
ws.add(prox)
assert len(ds) == len(ws) == 1
prox = ds.acreate_raster('', fps.A, float, 1, driver='MEM')
ws.add(prox)
assert len(ds) == len(ws) == 2
prox = ds.awrap_numpy_raster(fps.A, np.zeros(fps.A.shape))
ws.add(prox)
assert len(ds) == len(ws) == 3
ws.add(ds)
assert len(ws) == 4
def test_iter_data_fields_behavior(path):
ds = buzz.Dataset()
with ds.acreate_vector(path, fields=[], type='point').remove as v:
pt0, pt1 = sg.Point(1, 1), sg.Point(2, 2)
v.insert_data(pt0)
v.insert_data(pt1)
assert list(v.iter_data(None)) == [pt0, pt1]
assert list(v.iter_data(-1)) == [pt0, pt1]
assert list(v.iter_data('')) == [(pt0, ), (pt1, )]
assert list(v.iter_data([])) == [(pt0, ), (pt1, )]
assert list(v.iter_data([-1])) == [(pt0, ), (pt1, )]
with ds.acreate_vector(path,
fields=[dict(name='toto', type=int)],
type='point').remove as v:
pt0, pt1 = sg.Point(1, 1), sg.Point(2, 2)
v.insert_data(pt0, [42])
v.insert_data(pt1, [43])
assert list(v.iter_data(None)) == [pt0, pt1]
assert list(v.iter_data(-1)) == [(pt0, 42), (pt1, 43)]
assert list(v.iter_data('')) == [(pt0, ), (pt1, )]
assert list(v.iter_data([])) == [(pt0, ), (pt1, )]
assert list(v.iter_data([-1])) == [(pt0, 42), (pt1, 43)]
def test_vector_concurrent():
def _work(i):
point, = r1.iter_data(None)
return point
ds = buzz.Dataset(max_active=4)
meta = dict(type='point', )
p = mp.pool.ThreadPool(4)
with ds.acreate_vector('/tmp/v1.shp', **meta).delete as r1:
pt = sg.Point([42, 45])
r1.insert_data(pt)
r1.deactivate()
points = list(p.map(_work, range(1000)))
assert all(p == pt for p in points)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (4, 0, 4)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (4, 0, 4, True)
p.terminate()
def test_(pools, test_prefix, cache_tiles, test_prefix2):
def _open(**kwargs):
d = dict(fp=fp,
dtype='float32',
channel_count=2,
compute_array=functools.partial(_meshgrid_raster_in,
reffp=fp),
cache_dir=test_prefix,
cache_tiles=cache_tiles,
**dict(
itertools.chain(
pools['merge'].items(),
pools['resample'].items(),
pools['computation'].items(),
pools['io'].items(),
)))
d.update(kwargs)
return ds.acreate_cached_raster_recipe(**d)
def _test_get():
arrs = r.get_data(band=-1)
assert arrs.shape == tuple(np.r_[fp.shape, 2])
x, y = arrs[..., 0], arrs[..., 1]
xref, yref = fp.meshgrid_raster
assert np.all(x == xref)
assert np.all(y == yref)
def _test_resampling(fp):
arr = r.get_data(band=-1, fp=fp)
ref = npr.get_data(band=-1, fp=fp)
assert np.allclose(arr, ref)
def _corrupt_files(files):
for path in files:
with open(path, 'wb') as stream:
stream.write(b'42')
print() # debug line
fp = buzz.Footprint(
rsize=(100, 100),
size=(100, 100),
tl=(1000, 1100),
)
compute_same_address_space = (type(
pools['computation']['computation_pool'])
in {str, mp.pool.ThreadPool,
type(None)})
with buzz.Dataset(allow_interpolation=1).close as ds:
# Create a numpy raster with the same data, useful to compare resampling
npr = ds.awrap_numpy_raster(
fp,
np.stack(fp.meshgrid_raster, axis=2).astype('float32'))
# Test lazyness of cache
r = _open()
files = glob.glob(os.path.join(test_prefix, '*.tif'))
assert len(files) == 0
# Test get_data results
_test_get()
files = glob.glob(os.path.join(test_prefix, '*.tif'))
assert len(files) > 0
mtimes0 = {f: os.stat(f).st_mtime for f in files}
# Test persistence of cache
# Test get_data results
r.close()
r = _open(compute_array=_should_not_be_called)
_test_get()
files = glob.glob(os.path.join(test_prefix, '*.tif'))
assert len(files) > 0
mtimes1 = {f: os.stat(f).st_mtime for f in files}
assert mtimes0 == mtimes1
# Test overwrite parameter
# Test get_data results
r.close()
r = _open(ow=True)
_test_get()
files = glob.glob(os.path.join(test_prefix, '*.tif'))
assert len(files) > 0
mtimes1 = {f: os.stat(f).st_mtime for f in files}
assert mtimes0.keys() == mtimes1.keys()
for k, t0 in mtimes0.items():
t1 = mtimes1[k]
assert t0 < t1
# Test remapping #1 - Interpolation - Fully Inside
fp_within_upscaled = fp.intersection(
fp, scale=fp.scale / 2) & fp.erode(fp.rsemiminoraxis // 4)
_test_resampling(fp_within_upscaled)
# Test remapping #2 - Interpolation - Fully Outside
_test_resampling(fp_within_upscaled.move(fp.br + fp.diagvec))
# Test remapping #3 - No Interpolation - Fully Outside
_test_resampling(fp.move(fp.br + fp.diagvec))
# Test remapping #4 - Interpolation - Both in and out
_test_resampling(
fp_within_upscaled.move(fp.br - fp_within_upscaled.diagvec / 2))
# Test remapping #5 - No Interpolation - Both in and out
_test_resampling(fp.move(fp.br - fp.pxvec * fp.rsemiminoraxis))
# Test remapping #6 - Interpolation - Fully Inside - Tiled
r.close()
r = _open(max_resampling_size=20)
_test_resampling(fp_within_upscaled)
# Concurrent queries that need a cache file checksum
r.close()
r = _open()
for it in [r.iter_data(fps=[fp], band=-1) for _ in range(10)]:
next(it)
# Concurrent queries that need a cache file missing, all but one computation aborted
# because already launched
r.close()
r = _open(ow=True)
for it in [r.iter_data(fps=[fp], band=-1) for _ in range(10)]:
next(it)
# Query garbage collected
it1 = r.iter_data(fps=[fp] * 2,
max_queue_size=1) # 2/2 ready, 1/2 sinked
it2 = r.iter_data(fps=[fp] * 1,
max_queue_size=1) # 1/1 ready, 0/1 sinked
it3 = r.iter_data(fps=[fp] * 2,
max_queue_size=1) # 1/2 ready, 0/2 sinked
next(it1)
time.sleep(1 / 2)
del it1, it2, it3
gc.collect()
time.sleep(1 / 2)
r.get_data() # This line will reraise any exception from scheduler
# Raster closing during query
it1 = r.iter_data(fps=[fp] * 2,
max_queue_size=1) # 2/2 ready, 1/2 sinked
it2 = r.iter_data(fps=[fp] * 1,
max_queue_size=1) # 1/1 ready, 0/1 sinked
it3 = r.iter_data(fps=[fp] * 2,
max_queue_size=1) # 1/2 ready, 0/2 sinked
next(it1)
time.sleep(1 / 2)
# Close Dataset instead of Raster, because Dataset.close is currently blocking
with buzz.Dataset(allow_interpolation=1).close as ds:
npr = ds.awrap_numpy_raster(
fp,
np.stack(fp.meshgrid_raster, axis=2).astype('float32'))
# Corrupted cache file
files = glob.glob(os.path.join(test_prefix, '*.tif'))
mtimes0 = {f: os.stat(f).st_mtime for f in files}
corrupted_path = files[0]
_corrupt_files([corrupted_path])
r = _open()
r.get_data()
mtimes1 = {f: os.stat(f).st_mtime for f in files}
assert mtimes0.keys() == mtimes1.keys()
for path in files:
if path == corrupted_path:
assert mtimes0[path] != mtimes1[path]
else:
assert mtimes0[path] == mtimes1[path]
with buzz.Dataset(allow_interpolation=1).close as ds:
npr = ds.awrap_numpy_raster(
fp,
np.stack(fp.meshgrid_raster, axis=2).astype('float32'))
# In iter_data, the first one(s) don't need cache, the next ones need cache file checking and then recomputation
_corrupt_files(glob.glob(os.path.join(test_prefix, '*.tif')))
r = _open()
fps = [
fp.move(fp.br + fp.diagvec), # Outside
] + [fp] * 12
arrs = list(r.iter_data(band=-1, fps=fps))
assert len(arrs) == 13
for tile, arr in zip(fps, arrs):
assert np.all(arr == npr.get_data(band=-1, fp=tile))
with buzz.Dataset(allow_interpolation=1).close as ds:
npr = ds.awrap_numpy_raster(
fp,
np.stack(fp.meshgrid_raster, axis=2).astype('float32'))
# Test channels order versus numpy raster
r = _open()
for channels in [
0,
1,
None,
slice(None),
[0, 1],
[1, 0],
[1, 0, 1],
]:
assert np.all(
r.get_data(channels=channels) == npr.get_data(
channels=channels))
with buzz.Dataset(allow_interpolation=1).close as ds:
# Derived and primitive rasters not computed
if compute_same_address_space:
ac0, ac1 = _AreaCounter(fp), _AreaCounter(fp)
else:
ac0, ac1 = None, None
r0 = _open(
compute_array=functools.partial(_base_computation,
area_counter=ac0,
reffp=fp),
ow=True,
)
r1 = _open(
compute_array=functools.partial(_derived_computation,
area_counter=ac1,
reffp=fp),
queue_data_per_primitive={
'prim': functools.partial(r0.queue_data, band=-1)
},
cache_dir=test_prefix2,
ow=True,
)
assert len(r0.primitives) == 0
assert len(r1.primitives) == 1
assert r1.primitives['prim'] is r0
r1.get_data()
if compute_same_address_space:
ac0.check_done()
ac1.check_done()
r0.close()
r1.close()
# Derived raster not computed
if compute_same_address_space:
ac0, ac1 = _AreaCounter(fp), _AreaCounter(fp)
else:
ac0, ac1 = None, None
r0 = _open(
compute_array=functools.partial(_base_computation,
area_counter=ac0,
reffp=fp),
ow=False,
)
r1 = _open(
compute_array=functools.partial(_derived_computation,
area_counter=ac1,
reffp=fp),
queue_data_per_primitive={
'prim': functools.partial(r0.queue_data, band=-1)
},
cache_dir=test_prefix2,
ow=True,
)
r1.get_data()
if compute_same_address_space:
ac0.check_not_done()
ac1.check_done()
r0.close()
r1.close()
# Primitive raster not computed
if compute_same_address_space:
ac0, ac1 = _AreaCounter(fp), _AreaCounter(fp)
else:
ac0, ac1 = None, None
r0 = _open(
compute_array=functools.partial(_base_computation,
area_counter=ac0,
reffp=fp),
ow=True,
)
r1 = _open(
compute_array=functools.partial(_derived_computation,
area_counter=ac1,
reffp=fp),
queue_data_per_primitive={
'prim': functools.partial(r0.queue_data, band=-1)
},
cache_dir=test_prefix2,
ow=False,
)
r1.get_data()
if compute_same_address_space:
ac0.check_not_done()
ac1.check_not_done()
r0.close()
r1.close()
# Test computation tiles
if compute_same_address_space:
ac0, ac1 = _AreaCounter(fp), _AreaCounter(fp)
else:
ac0, ac1 = None, None
r0 = _open(
compute_array=functools.partial(_base_computation,
area_counter=ac0,
reffp=fp),
computation_tiles=(11, 11),
ow=True,
)
r1 = _open(
compute_array=functools.partial(_derived_computation,
area_counter=ac1,
reffp=fp),
queue_data_per_primitive={
'prim': functools.partial(r0.queue_data, band=-1)
},
cache_dir=test_prefix2,
computation_tiles=(22, 22),
ow=True,
)
r1.get_data()
if compute_same_address_space:
ac0.check_done()
ac1.check_done()
r0.close()
r1.close()
# Several queries, one is dropped, the rest is still working
r0 = _open(
compute_array=functools.partial(_base_computation, reffp=fp),
ow=True,
)
r1 = _open(
compute_array=functools.partial(_derived_computation, reffp=fp),
queue_data_per_primitive={
'prim': functools.partial(r0.queue_data, band=-1)
},
cache_dir=test_prefix2,
ow=True,
)
t = r1.cache_tiles.flatten()
fps0 = t.tolist() * 2
fps1 = fps0[::-1]
fps2 = np.roll(t, t.size // 2).tolist() * 2
fps3 = fps2[::-1]
it0 = r1.iter_data(fps=fps0)
it1 = r1.iter_data(fps=fps1)
it2 = r1.iter_data(fps=fps2)
it3 = r1.iter_data(fps=fps3)
del it1
assert len(list(it3)) == t.size * 2
assert len(list(it0)) == t.size * 2
assert len(list(it2)) == t.size * 2
r0.close()
r1.close()
# Computation function crashes, we catch error in main thread
r = _open(ow=True, compute_array=_please_crash)
with pytest.raises(NecessaryCrash):
r.get_data()
def test_vector():
ds = buzz.Dataset(max_active=2)
meta = dict(type='point', )
def statuses(*args):
l = tuple([(ds._back.idle_count(prox._back.uid),
ds._back.used_count(prox._back.uid), prox.active_count,
prox.active) for prox in args])
return l
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 0, 0)
with ds.acreate_vector('/tmp/v1.shp', **meta).delete as r1:
r1.insert_data([0, 0])
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 0, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
# Iteration 1 - exn
it = r1.iter_data()
next(it)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 1, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 1, 1, True)
try:
next(it)
except StopIteration:
pass
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 0, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
# Iteration 2 - close
it = r1.iter_data()
next(it)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 1, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 1, 1, True)
it.close()
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
# Iteration 3 - del
it = r1.iter_data()
next(it)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 1, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 1, 1, True)
del it
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
# Iteration 4 - try source.close
it = r1.iter_data()
next(it)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 1, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 1, 1, True)
with pytest.raises(Exception, match='deactivate'):
r1.close()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 1, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 1, 1, True)
del it
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 0, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
# Iteration 5 - multi
it0 = r1.iter_data()
next(it0)
it1 = r1.iter_data()
next(it1)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 2, 2)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 2, 2, True)
it2 = r1.iter_data()
with pytest.raises(RuntimeError, match='simultaneous'):
next(it2)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 2, 2)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 2, 2, True)
r1.activate() # no effect
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 2, 2)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (0, 2, 2, True)
del it0
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 1, 2)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 1, 2, True)
del it1
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (2, 0, 2, True)
def test_raster(fps, random_path_tif):
def _asserts(should_exist, should_be_open, is_anonymous=False):
exist = os.path.isfile(random_path_tif)
assert should_exist == exist
if not is_anonymous:
is_open_key = 'test' in ds
is_open_prox = test in ds
assert is_open_key == is_open_prox
assert is_open_key == should_be_open
if is_open_key:
assert test is ds.test is ds['test']
else:
is_open = test in ds
assert is_open == should_be_open
if should_be_open and not is_anonymous:
assert ds['test'] == ds.test
if should_be_open:
assert len(ds) == 1
else:
assert len(ds) == 0
ds = buzz.Dataset()
# Raster test 1
test = None
_asserts(False, False)
test = ds.create_raster('test', random_path_tif, fps.A, float, 1)
_asserts(True, True)
ds.test.close()
_asserts(True, False)
test = ds.aopen_raster(random_path_tif)
_asserts(True, True, True)
test.close()
_asserts(True, False)
test = ds.open_raster('test', random_path_tif, mode='w')
_asserts(True, True)
test.remove()
_asserts(False, False)
# Raster test 2 - context/close
with ds.create_raster('test', random_path_tif, fps.A, float, 1).close as test:
_asserts(True, True)
_asserts(True, False)
with ds.open_raster('test', random_path_tif, mode='w').delete as test:
_asserts(True, True)
_asserts(False, False)
# Raster test 3 - context/close/anonymous
with ds.acreate_raster(random_path_tif, fps.A, float, 1).delete as test:
_asserts(True, True, True)
_asserts(False, False)
# Raster test 4 - context/delete
with ds.create_raster('test', random_path_tif, fps.A, float, 1).delete as test:
_asserts(True, True)
_asserts(False, False)
# Raster test 5 - MEM
with ds.create_raster('test', '', fps.A, float, 1, driver='MEM').close as test:
_asserts(False, True)
# Raster test 6 - numpy
with ds.wrap_numpy_raster('test', fps.A, np.zeros(fps.A.shape)).close as test:
_asserts(False, True)
# Raster test 7 - gc
del ds
ws = weakref.WeakSet()
def _test():
ds = buzz.Dataset()
assert len(ds) == len(ws) == 0
prox = ds.create_raster('test', random_path_tif, fps.A, float, 1)
ws.add(prox)
assert len(ds) == len(ws) == 1
prox = ds.acreate_raster('', fps.A, float, 1, driver='MEM')
ws.add(prox)
assert len(ds) == len(ws) == 2
prox = ds.awrap_numpy_raster(fps.A, np.zeros(fps.A.shape))
ws.add(prox)
assert len(ds) == len(ws) == 3
ws.add(ds)
assert len(ws) == 4
_test()
gc.collect()
assert len(ws) == 0
def test_mode1(fps, shp1_path, tif1_path, shp2_path, tif2_path, shp3_path,
tif3_path):
ds = buzz.Dataset()
ds.open_raster('tif1', tif1_path)
ds.open_vector('shp1', shp1_path)
ds.open_raster('tif2', tif2_path)
ds.open_vector('shp2', shp2_path)
ds.open_raster('tif3', tif3_path)
ds.open_vector('shp3', shp3_path)
# Test SR equality
assert sreq(
ds.tif1.wkt_virtual,
ds.tif1.wkt_stored,
ds.tif1.proj4_virtual,
ds.tif1.proj4_stored,
ds.shp1.wkt_virtual,
ds.shp1.wkt_stored,
ds.shp1.proj4_virtual,
ds.shp1.proj4_stored,
)
assert sreq(
ds.tif2.wkt_virtual,
ds.tif2.wkt_stored,
ds.tif2.proj4_virtual,
ds.tif2.proj4_stored,
ds.shp2.wkt_virtual,
ds.shp2.wkt_stored,
ds.shp2.proj4_virtual,
ds.shp2.proj4_stored,
)
assert not sreq(ds.tif1.wkt_stored, ds.tif2.wkt_stored)
assert (None == ds.wkt == ds.proj4 == ds.tif3.wkt_virtual ==
ds.tif3.wkt_stored == ds.tif3.proj4_virtual == ds.tif3.proj4_stored
== ds.shp3.wkt_virtual == ds.shp3.wkt_stored ==
ds.shp3.proj4_virtual == ds.shp3.proj4_stored)
# Test footprints equality
assert fpeq(
fps.AI,
# tif/shp 1
ds.tif1.fp,
ds.tif1.fp_origin,
buzz.Footprint.of_extent(ds.shp1.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds[[0, 2, 1, 3]], fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
# tif/shp 2
ds.tif2.fp,
ds.tif2.fp_origin,
buzz.Footprint.of_extent(ds.shp2.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds[[0, 2, 1, 3]], fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
# tif/shp 3
ds.tif3.fp,
ds.tif3.fp_origin,
buzz.Footprint.of_extent(ds.shp3.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp3.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp3.bounds[[0, 2, 1, 3]], fps.AI.scale),
buzz.Footprint.of_extent(ds.shp3.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
)
# Test what's written in all 6 files
tif1 = ds.tif1.get_data()
tif2 = ds.tif2.get_data()
tif3 = ds.tif3.get_data()
assert np.all(tif1 == tif2)
assert np.all(tif1 == tif3)
for i, letter in enumerate(string.ascii_uppercase[:9]):
# tif/shp 1
shp1 = ds.shp1.get_data(i, None)
raster1_polys = ds.tif1.fp.find_polygons(tif1 == ord(letter))
assert len(raster1_polys) == 1
assert (shp1 ^ raster1_polys[0]).is_empty
# tif/shp 2
shp2 = ds.shp2.get_data(i, None)
raster2_polys = ds.tif2.fp.find_polygons(tif2 == ord(letter))
assert len(raster2_polys) == 1
assert (shp2 ^ raster2_polys[0]).is_empty
# tif/shp 3
shp3 = ds.shp3.get_data(i, None)
raster3_polys = ds.tif3.fp.find_polygons(tif3 == ord(letter))
assert len(raster3_polys) == 1
assert (shp3 ^ raster3_polys[0]).is_empty
def main():
return # None of the features shown here are implemented yet
ds = buzz.Dataset(allow_interpolation=True)
pixel_per_line = {
'mand_100px': 10,
'mand_10kpx': 100,
'mand_1mpx': 1_000,
'mand_100mpx': 10_000,
'mand_10gpx': 100_000,
'mand_1tpx': 1_000_000,
}
# Instanciate 6 fixed scale mandelbrot rasters
for key, rwidth in pixel_per_line.items():
# Create a Footprint that ranges from -2 to 2 on both x and y axes
fp = buzz.Footprint(
gt=(-2, 4 / rwidth, 0, -2, 0, 4 / rwidth),
rsize=(rwidth, rwidth),
)
ds.create_raster_recipe(
key,
fp=fp,
dtype='float32',
channel_count=1,
compute_array=mandelbrot_of_footprint,
automatic_remapping=True, # True is the default value
max_computation_size=128,
)
# Instanciate 1 flexible scale mandelbrot raster
# The fp parameter does not mean much when `automatic_remapping=False`, but
# it is still mandatory.
ds.create_raster_recipe(
'mand',
fp=ds.mand_10kpx.fp,
dtype='float32',
channel_count=1,
compute_array=mandelbrot_of_footprint,
automatic_remapping=False,
max_computation_size=128,
)
# Test 1 - Perform basic tests ****************************************** **
test_raster(ds.mand_100px)
test_raster(ds.mand_10kpx)
test_raster(ds.mand_1mpx)
# Test 2 - Play with resampling with the non-flexible scale rasters ***** **
fp100k = buzz.Footprint(
gt=(-2, 4 / 316, 0, -2, 0, 4 / 316),
rsize=(316, 316),
)
example_tools.show_several_images(
('10kpx', ds.mand_10kpx.fp, ds.mand_10kpx.get_data()),
('1mpx', ds.mand_1mpx.fp, ds.mand_1mpx.get_data()),
('10kpx to 100kpx', fp100k,
ds.mand_10kpx.get_data(fp=fp100k)), # upsample * 10
('1mpx to 100kpx', fp100k,
ds.mand_1mpx.get_data(fp=fp100k)), # downsample * 10
)
# Test 3 - Play with with the flexible scale raster ********************* **
example_tools.show_several_images(
('10kpx', ds.mand_10kpx.fp, ds.mand.get_data(fp=ds.mand_10kpx.fp)),
('1mpx', ds.mand_1mpx.fp, ds.mand.get_data(fp=ds.mand_1mpx.fp)),
)
# Test 4 - Zoom to a point ********************************************** **
focus = shapely.geometry.Point(-1.1172, -0.221103)
for key in pixel_per_line.items():
fp = ds[key].fp
fp = fp.dilate(250) & focus.buffer(fp.pxsizex * 250)
arr = ds[key].get_data(fp=fp)
title = f'{fp.rw}x{fp.rh} rect of the {key} image'
example_tools.show_several_images((title, fp, arr))
ds.close()
def test_vector(fps, random_path_shp):
def _asserts(should_exist, should_be_open, is_anonymous=False):
exist = os.path.isfile(random_path_shp)
assert should_exist == exist
if not is_anonymous:
is_open_key = 'test' in ds
is_open_prox = test in ds
assert is_open_key == is_open_prox
assert is_open_key == should_be_open
if is_open_key:
assert test is ds.test is ds['test']
else:
is_open = test in ds
assert is_open == should_be_open
if should_be_open and not is_anonymous:
assert ds['test'] == ds.test
if should_be_open:
assert len(ds) == 1
else:
assert len(ds) == 0
ds = buzz.Dataset()
# Vector test 1
test = None
_asserts(False, False)
test = ds.create_vector('test', random_path_shp, 'polygon')
_asserts(True, True)
ds.test.close()
_asserts(True, False)
test = ds.aopen_vector(random_path_shp)
_asserts(True, True, True)
test.close()
_asserts(True, False)
test = ds.open_vector('test', random_path_shp, mode='w')
_asserts(True, True)
test.remove()
_asserts(False, False)
# Vector test 2 - context/close
with ds.create_vector('test', random_path_shp, 'polygon').close as test:
_asserts(True, True)
_asserts(True, False)
with ds.open_vector('test', random_path_shp, mode='w').delete as test:
_asserts(True, True)
_asserts(False, False)
# Vector test 3 - context/close/anonymous
with ds.acreate_vector(random_path_shp, 'polygon').delete as test:
_asserts(True, True, True)
_asserts(False, False)
# Vector test 4 - context/delete
with ds.create_vector('test', random_path_shp, 'polygon').delete as test:
_asserts(True, True)
_asserts(False, False)
# Vector test 5 - MEM
with ds.create_vector('test', '', 'polygon', driver='Memory').close as test:
_asserts(False, True)
# Vector test 6 - gc
del ds
ws = weakref.WeakSet()
def _test():
ds = buzz.Dataset()
assert len(ds) == len(ws) == 0
prox = ds.create_vector('test', random_path_shp, 'polygon')
ws.add(prox)
assert len(ds) == len(ws) == 1
prox = ds.acreate_vector('', 'polygon', driver='Memory')
ws.add(prox)
assert len(ds) == len(ws) == 2
ws.add(ds)
assert len(ws) == 3
_test()
gc.collect()
assert len(ws) == 0
def main():
return # None of the features shown here are implemented yet
path = example_tools.create_random_elevation_gtiff()
ds = buzz.Dataset()
# Pool to parallelize:
# - `ds.slopes` computations
# - `ds.elevation` resamplings
cpu_pool = mp.pool.ThreadPool(mp.cpu_count())
# Pool to parallelize:
# - `ds.elevation` disk reads
io_pool = mp.pool.ThreadPool(4)
ds.open_raster(
'elevation',
path=path,
async_={'io_pool': io_pool, 'resample_pool': cpu_pool},
)
ds.create_raster_recipe(
'slopes',
computation_pool=cpu_pool,
# The next 6 lines can be replaced by **buzz.algo.slopes(ds.elevation)
fp=ds.elevation.fp,
dtype='float32',
channel_count=1,
compute_array=slopes_of_elevation,
queue_data_per_primitive={'dem': ds.elevation.queue_data},
convert_footprint_per_primitive={'dem': lambda fp: fp.dilate(1)},
)
# Test 1 - Perform basic tests ****************************************** **
# `test_raster` will request `slopes`'s' pixels. `elevation`'s' pixels will
# be requested in cascade and then used to compute the `slopes`.
test_raster(ds.slopes)
# Test 2 - Multiple iterations at the same time ************************* **
# Here the `elevation` raster is directly requested and also requested by
# the `slopes`, the Dataset's scheduler is made to handle simultaneous
# queries.
tiles = ds.elevation.fp.tile_count(2, 2).flatten()
dem_iterator = ds.elevation.iter_data(tiles)
slopes_iterator = ds.slopes.iter_data(tiles)
for tile, dem, slopes in zip(tiles, dem_iterator, slopes_iterator):
print(f'Showing dem and slopes at:\n {tile}')
example_tools.show_several_images(
('elevation (dem)', tile, dem),
('slopes', tile, slopes),
)
# Test 3 - Backpressure prevention ************************************** **
tiles = ds.slopes.tile_count(3, 3).flatten()
print('Creating a slopes iterator on 9 tiles')
it = ds.slopes.iter_data(tiles, max_queue_size=1)
print(' At most 5 dem arrays can be ready between `ds.elevation` and '
'`ds.slopes`')
print(' At most 1 slopes array can be ready out of the slopes iterator')
print('Sleeping several seconds to let the scheduler create 6 of the 9 '
'dem arrays, and 1 of the 9 slopes arrays.')
time.sleep(4)
with example_tools.Timer() as t:
arr = next(it)
print(f'Getting the first array took {t}, this was instant because it was '
'ready')
with example_tools.Timer() as t:
for _ in range(5):
next(it)
print(f'Getting the next 5 arrays took {t}, it was quick because the dems '
'were ready')
with example_tools.Timer() as t:
for _ in range(3):
next(it)
print(f'Getting the last 4 arrays took {t}, it was long because nothing was'
' ready')
# Cleanup *************************************************************** **
ds.close()
os.remove(path)
def test_mode4(fps, shp1_path, tif1_path, shp2_path, tif2_path, shp3_path,
tif3_path, random_path_shp, random_path_tif, env):
ds = buzz.Dataset(sr_work=SR1['wkt'], sr_forced=SR2['wkt'])
ds.open_raster('tif1', tif1_path)
ds.open_vector('shp1', shp1_path)
ds.open_raster('tif2', tif2_path)
ds.open_vector('shp2', shp2_path)
ds.open_raster('tif3', tif3_path)
ds.open_vector('shp3', shp3_path)
# Test file creation without spatial reference
with buzz.Env(allow_complex_footprint=True):
with ds.acreate_vector(random_path_shp, 'polygon', [],
sr=None).close as r:
assert r.wkt_stored == None
assert sreq(r.wkt_virtual, SR2['wkt'])
with ds.acreate_raster(random_path_tif,
fps.AI,
'int32',
1, {},
sr=None).close as v:
assert v.wkt_stored == None
assert sreq(v.wkt_virtual, SR2['wkt'])
# Test SR equality
assert sreq(
ds.wkt,
ds.proj4,
ds.tif1.wkt_stored,
ds.tif1.proj4_stored,
ds.shp1.wkt_stored,
ds.shp1.proj4_stored,
)
assert sreq(
ds.tif1.wkt_virtual,
ds.tif1.proj4_virtual,
ds.shp1.wkt_virtual,
ds.shp1.proj4_virtual,
ds.tif2.wkt_virtual,
ds.tif2.proj4_virtual,
ds.shp2.wkt_virtual,
ds.shp2.proj4_virtual,
ds.tif2.wkt_stored,
ds.tif2.proj4_stored,
ds.shp2.wkt_stored,
ds.shp2.proj4_stored,
ds.tif3.wkt_virtual,
ds.tif3.proj4_virtual,
ds.shp3.wkt_virtual,
ds.shp3.proj4_virtual,
)
assert not sreq(ds.tif1.wkt_stored, ds.tif2.wkt_stored)
assert (None == ds.tif3.wkt_stored == ds.tif3.proj4_stored ==
ds.shp3.wkt_stored == ds.shp3.proj4_stored)
# Test foorprints equality
assert fpeq(
fps.AI,
# tif/shp 1
ds.tif1.fp_origin,
buzz.Footprint.of_extent(ds.shp1.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
# tif/shp 2
ds.tif2.fp_origin,
buzz.Footprint.of_extent(ds.shp2.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
# tif/shp 3
ds.tif3.fp_origin,
buzz.Footprint.of_extent(ds.shp3.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp3.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
)
assert fpeq(
# tif/shp 1
ds.tif1.fp,
buzz.Footprint.of_extent(ds.shp1.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds[[0, 2, 1, 3]], fps.AI.scale),
# tif/shp 2
ds.tif2.fp,
buzz.Footprint.of_extent(ds.shp2.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds[[0, 2, 1, 3]], fps.AI.scale),
# tif/shp 3
ds.tif3.fp,
buzz.Footprint.of_extent(ds.shp3.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp3.bounds[[0, 2, 1, 3]], fps.AI.scale),
)
assert ds.tif1.fp != ds.tif1.fp_stored
# Test what's written in all 6 files
tif1 = ds.tif1.get_data()
tif2 = ds.tif2.get_data()
tif3 = ds.tif3.get_data()
assert np.all(tif1 == tif2)
assert np.all(tif1 == tif3)
def f(x, y, z=None):
return np.around(x, 6), np.around(y, 6)
for i, letter in enumerate(string.ascii_uppercase[:9]):
# tif/shp 1
shp1 = ds.shp1.get_data(i, None)
shp1 = shapely.ops.transform(f, shp1)
raster1_polys = ds.tif1.fp.find_polygons(tif1 == ord(letter))
assert len(raster1_polys) == 1
raster1_poly = raster1_polys[0]
raster1_poly = shapely.ops.transform(f, shp1)
assert (shp1 ^ raster1_poly).is_empty
# tif/shp 2
shp2 = ds.shp2.get_data(i, None)
shp2 = shapely.ops.transform(f, shp2)
raster2_polys = ds.tif2.fp.find_polygons(tif2 == ord(letter))
assert len(raster2_polys) == 1
raster2_poly = raster2_polys[0]
raster2_poly = shapely.ops.transform(f, shp2)
assert (shp2 ^ raster2_poly).is_empty
# tif/shp 3
shp3 = ds.shp3.get_data(i, None)
shp3 = shapely.ops.transform(f, shp3)
raster3_polys = ds.tif3.fp.find_polygons(tif3 == ord(letter))
assert len(raster3_polys) == 1
raster3_poly = raster3_polys[0]
raster3_poly = shapely.ops.transform(f, shp3)
assert (shp3 ^ raster3_poly).is_empty
def example():
ds = buzz.Dataset()
# Create a Footprint that ranges from -2 to 2 on both x and y axes
fp = buzz.Footprint(
gt=(-2, 4 / 10000, 0, -2, 0, 4 / 10000),
rsize=(10000, 10000),
)
cache_tiling = fp.tile((512, 512), boundary_effect='shrink')
computation_tiling = fp.tile((128, 128), boundary_effect='shrink')
cached_recipe_params = dict(
key='mand_100mpx',
fp=fp,
dtype='float32',
channel_count=1,
compute_array=mandelbrot_of_footprint,
cache_dir=CACHE_DIR,
cache_tiles=cache_tiling,
computation_tiles=computation_tiling,
)
ds.create_cached_raster_recipe(**cached_recipe_params)
# Test 1 - Timings before and after caching ***************************** **
print('Test 1 - Read mandelbrot 100mpx twice and compare timings')
fp = ds.mand_100mpx.fp
fp = fp & shapely.geometry.Point(-1.1172, -0.221103).buffer(
fp.pxsizex * 300)
print(f'Getting Footprint at {fp.c}...')
with example_tools.Timer() as t:
ds.mand_100mpx.get_data(fp=fp)
print(f' took {t}')
print(f'Getting Footprint at {fp.c}...')
with example_tools.Timer() as t:
ds.mand_100mpx.get_data(fp=fp)
print(f' took {t}')
print('Tiles in `{}` directory:\n- {}'.format(
CACHE_DIR,
'\n- '.join(example_tools.list_cache_files_path_in_dir(CACHE_DIR)),
))
print()
# Test 2 - Corrupt one cache file and try to reuse the file ************* **
print('Test 2 - Corrupt one cache file and try to reuse the file')
ds.mand_100mpx.close()
# Pick one cache file and append one byte to it
one_cache_tile_path = example_tools.list_cache_files_path_in_dir(
CACHE_DIR)[0]
print(f'Corrupting {one_cache_tile_path}...')
with open(one_cache_tile_path, 'ba') as f:
f.write(b'\0x42')
ds.create_cached_raster_recipe(**cached_recipe_params)
print(f'Getting Footprint at {fp.c}...')
with example_tools.Timer() as t:
arr = ds.mand_100mpx.get_data(fp=fp)
print(f' took {t}')
example_tools.show_several_images((
'part of mandelbrot 100 mega pixels',
fp,
arr,
))
return # The NEXT features are not yet implemented
# Test 4 - Colorize mandelbrot 100mpx ************************************ **
ds.create_raster_recipe(
'mand_red',
fp=fp,
dtype='uint8',
channel_count=3,
compute_array=colorize_mandelbrot,
queue_data_per_primitive={'mand': ds.mand_100mpx.queue_data},
computation_tiles=computation_tiling,
automatic_remapping=False,
)
example_tools.show_several_images((
'part of mandelbrot 10 mega pixels in red',
fp,
ds.mand_red.get_data(fp=fp),
))
def main():
print("All images shown here belong to ESA/Hubble. See spacetelescope.org.\n")
ds = buzz.Dataset(allow_interpolation=True)
open_zoomable_rasters(ds, 'andromeda', overwrite=True)
# Test 1 - Perform basic tests ****************************************** **
print()
print('Test 1 - Show andromeda with 3 resolutions')
test_raster(ds.andromeda_zoom0)
example_tools.show_several_images((
'andromeda_zoom0', ds.andromeda_zoom0.fp,
ds.andromeda_zoom0.get_data()
))
print()
test_raster(ds.andromeda_zoom1)
example_tools.show_several_images((
'andromeda_zoom1', ds.andromeda_zoom1.fp,
ds.andromeda_zoom1.get_data()
))
print()
test_raster(ds.andromeda_zoom2)
example_tools.show_several_images((
'andromeda_zoom2', ds.andromeda_zoom2.fp,
ds.andromeda_zoom2.get_data()
))
print()
# Test 2 - Test `get_data` timings ************************************** **
print()
print('Test 2 - Read andromeda 4 times and compare timings')
with example_tools.Timer() as t:
ds.andromeda_zoom5.get_data()
print(f'Getting andromeda_zoom5 took {t}, download was performed')
with example_tools.Timer() as t:
ds.andromeda_zoom5.get_data()
print(f'Getting andromeda_zoom5 took {t}, data was directly fetched from cache')
print('Closing and opening andromeda rasters again...')
ds.close()
ds = buzz.Dataset(allow_interpolation=True)
open_zoomable_rasters(ds, 'andromeda', overwrite=False)
with example_tools.Timer() as t:
ds.andromeda_zoom5.get_data()
print(f'Getting andromeda_zoom5 took {t}, cache files validity was checked'
' and data was fetched from cache')
with example_tools.Timer() as t:
ds.andromeda_zoom5.get_data()
print(f'Getting andromeda_zoom5 took {t}, data was directly fetched from cache')
example_tools.show_several_images((
'andromeda_zoom5', ds.andromeda_zoom5.fp,
ds.andromeda_zoom5.get_data()
))
# Test 3 **************************************************************** **
print()
print('Test 2 - Show monocerotis')
open_zoomable_rasters(ds, 'monocerotis', overwrite=False)
example_tools.show_several_images((
'monocerotis_zoom3', ds.monocerotis_zoom3.fp,
ds.monocerotis_zoom3.get_data()
))
def test_mode2(fps, shp1_path, tif1_path, shp2_path, tif2_path, shp3_path,
tif3_path, random_path_shp, random_path_tif, env):
ds = buzz.Dataset(sr_work=SR1['wkt'])
ds.open_raster('tif1', tif1_path)
ds.open_vector('shp1', shp1_path)
ds.open_raster('tif2', tif2_path)
ds.open_vector('shp2', shp2_path)
# Test file creation/opening without spatial reference
with buzz.Env(allow_complex_footprint=True):
with pytest.raises(ValueError, match='spatial refe'):
ds.acreate_vector(random_path_shp, 'polygon', [], sr=None)
with pytest.raises(ValueError, match='spatial refe'):
ds.acreate_raster(random_path_tif, fps.AI, 'int32', 1, {}, sr=None)
with pytest.raises(ValueError, match='spatial refe'):
ds.aopen_raster(tif3_path)
with pytest.raises(ValueError, match='spatial refe'):
ds.aopen_vector(shp3_path)
# Test SR equality
assert sreq(
ds.wkt,
ds.proj4,
ds.tif1.wkt_virtual,
ds.tif1.wkt_stored,
ds.tif1.proj4_virtual,
ds.tif1.proj4_stored,
ds.shp1.wkt_virtual,
ds.shp1.wkt_stored,
ds.shp1.proj4_virtual,
ds.shp1.proj4_stored,
)
assert sreq(
ds.tif2.wkt_virtual,
ds.tif2.proj4_virtual,
ds.shp2.wkt_virtual,
ds.shp2.proj4_virtual,
ds.tif2.wkt_stored,
ds.tif2.proj4_stored,
ds.shp2.wkt_stored,
ds.shp2.proj4_stored,
)
assert not sreq(ds.tif1.wkt_stored, ds.tif2.wkt_stored)
# Test foorprints equality
assert fpeq(
fps.AI,
# tif/shp 1
ds.tif1.fp,
ds.tif1.fp_origin,
buzz.Footprint.of_extent(ds.shp1.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds[[0, 2, 1, 3]], fps.AI.scale),
buzz.Footprint.of_extent(ds.shp1.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
# tif/shp 2
ds.tif2.fp_origin,
buzz.Footprint.of_extent(ds.shp2.extent_stored, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds_stored[[0, 2, 1, 3]],
fps.AI.scale),
)
assert fpeq(
ds.tif2.fp,
buzz.Footprint.of_extent(ds.shp2.extent, fps.AI.scale),
buzz.Footprint.of_extent(ds.shp2.bounds[[0, 2, 1, 3]], fps.AI.scale),
)
assert ds.tif2.fp != ds.tif1.fp
# Test file creation with/without conversion of footprint
with buzz.Env(allow_complex_footprint=True):
with ds.acreate_raster(random_path_tif,
fps.AI,
'int32',
1,
sr=SR1['wkt'],
ow=1).delete as r:
assert fpeq(fps.AI, r.fp, r.fp_origin)
with ds.acreate_raster(random_path_tif,
fps.AI,
'int32',
1,
sr=SR2['wkt'],
ow=1).delete as r:
assert fpeq(
fps.AI,
r.fp,
)
assert fps.AI != r.fp_origin
# Test what's written in all 4 files
tif1 = ds.tif1.get_data()
tif2 = ds.tif2.get_data()
assert np.all(tif1 == tif2)
def f(x, y, z=None):
return np.around(x, 6), np.around(y, 6)
for i, letter in enumerate(string.ascii_uppercase[:9]):
# tif/shp 1
shp1 = ds.shp1.get_data(i, None)
shp1 = shapely.ops.transform(f, shp1)
raster1_polys = ds.tif1.fp.find_polygons(tif1 == ord(letter))
assert len(raster1_polys) == 1
raster1_poly = raster1_polys[0]
raster1_poly = shapely.ops.transform(f, shp1)
assert (shp1 ^ raster1_poly).is_empty
# tif/shp 2
shp2 = ds.shp2.get_data(i, None)
shp2 = shapely.ops.transform(f, shp2)
raster2_polys = ds.tif2.fp.find_polygons(tif2 == ord(letter))
assert len(raster2_polys) == 1
raster2_poly = raster2_polys[0]
raster2_poly = shapely.ops.transform(f, shp2)
assert (shp2 ^ raster2_poly).is_empty
