def test_maxfd():
cap, _ = resource.getrlimit(resource.RLIMIT_NOFILE)
if cap > 2049:
pytest.skip('file descriptors cap is too high to be tested: {}'.format(cap))
# Test 1 without fifo
ds = buzz.DataSource(max_activated=np.inf)
suff = str(uuid.uuid4())
try:
with pytest.raises(Exception):
for i in range(cap + 1):
ds.create_vector(i, '/tmp/f{:04d}_{}.shp'.format(i, suff), 'point', [{'name': 'Hello', 'type': int}])
ds[i].insert_data((i, i + 1))
pytest.skip("Reached {} opened files but didn't crash, whatever...".format(i))
finally:
file_count = i
for i in range(file_count):
ds[i].delete()
# Test 2 with fifo
ds = buzz.DataSource(max_activated=5)
suff = str(uuid.uuid4())
for i in range(cap + 1):
ds.create_vector(i, '/tmp/f{:04d}_{}.shp'.format(i, suff), 'point', [{'name': 'Hello', 'type': int}])
ds[i].insert_data((i, i + 1))
assert (ds._queued_count, ds._locked_count) == (5, 0)
for i in range(cap + 1):
v = ds[i]
assert (len(v), v.get_data(0, geom_type='coordinates')) == (1, ((i, i + 1), None))
for i in range(cap + 1):
ds[i].delete()
def compute_dataset(image_size=448,
n_files=None,
downsampling_factor=1,
n_samples=2):
'''
Computes the augmented dataset of length (n_files*n_samples).
Parameters
----------
image_size : int
size of the input images for the neural network
n_files : int
number of images in the total inria dataset to consider (for debugging)
downsampling_factor : float
downsampling factor to lower the resolution
n_samples : int
number of cropped images to generate for each image in inria dataset
Returns
-------
X_train : np.ndarray
Input array for the neural network (n_files*samples rgb images)
Y_train : np.ndarray
Output array for the neural network (n_files*samples binary images)
'''
images_train = "AerialImageDataset/train/images/"
gt_train = "AerialImageDataset/train/gt/"
files = [f for f in listdir(images_train) if isfile(join(images_train, f))]
if n_files is None:
n_files = len(files)
X_train = np.zeros((n_files * n_samples, image_size, image_size, 3))
Y_train = np.zeros((n_files * n_samples, image_size, image_size, 1))
for i in tqdm.tqdm(range(n_files)):
# current image
file = files[i]
ds_rgb = buzz.DataSource(allow_interpolation=True)
ds_binary = buzz.DataSource(allow_interpolation=True)
rgb_path = images_train + file
binary_path = gt_train + file
ds_rgb.open_raster('rgb', rgb_path)
ds_binary.open_raster('rgb', binary_path)
# n_samples random crop of the image
for sample in range(n_samples):
fp = random_crop(ds_rgb,
crop_rsize=image_size,
factor=downsampling_factor)
rgb = ds_rgb.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8')
binary = ds_binary.rgb.get_data(band=(1), fp=fp).astype('uint8')
# Normalization of data
X_train[i * n_samples + sample] = rgb / 127.5 - 1.0
almost_binary_y = binary.reshape(
(image_size, image_size, 1)) / 127.5 - 1.0
Y_train[i * n_samples + sample] = almost_binary_y > 0
return X_train, Y_train
def compute_data(fp, *args):
if resample:
ds = buzz.DataSource(allow_interpolation=True)
else:
ds = buzz.DataSource(allow_interpolation=True)
with ds.open_araster(path).close as r:
# print(f'Reading {fp.rarea:12,}px of {path.split("/")[-1]:10} ')
arr = r.get_data(fp, band=-1)
# print(f' Read {fp.rarea:12,}px of {path.split("/")[-1]:10} ')
return arr
def test_reproj():
sr0 = SRS[0]
sr1 = SRS[3]
with buzz.Env(significant=8, allow_complex_footprint=1, warnings=0):
# Create `twos`, a recipe from `sr1` to `sr0`
# `fp` in sr0
# `ds.wkt` in sr0
# `twos.fp` in sr0
# `twos.fp_origin` in sr1
# `pxfun@fp` in sr1
fp = buzz.Footprint(
tl=(sr0['cx'], sr0['cy']),
size=(2, 2),
rsize=(20, 20),
)
ds = buzz.DataSource(sr0['wkt'])
def pxfun(fp):
assert (twos.fp_origin
& fp) == fp, 'fp should be aligned and within twos.fp'
return ones.get_data(fp=fp) * 2
twos = ds.create_recipe_araster(pxfun,
fp,
'float32',
sr=sr1['wkt'],
band_schema={'nodata': 42})
# Create `ones`, a raster from `sr1` to `sr1`
# `ds2.wkt` in sr1
# `ones.fp` in sr1
# `ones.fp_origin` in sr1
ds2 = buzz.DataSource(sr1['wkt'])
ones = ds2.create_araster('',
twos.fp_origin,
'float32',
1,
driver='MEM',
sr=sr1['wkt'],
band_schema={'nodata': 42})
ones.fill(1)
# Test that `sr1` performs reprojection of `fp` before sending it to `pxfun`
assert ones.fp == ones.fp_origin, 'ones has no reproj'
assert twos.fp_origin == ones.fp_origin
assert ones.dtype == twos.dtype
tiles = fp.tile_count(
3, 3, boundary_effect='shrink').flatten().tolist() + [fp]
for tile in tiles:
assert (twos.get_data(fp=tile) == 2).all()
twos.close()
ones.close()
def test_raster(path, driver, band_details, options, save_proj):
dtype, band_count, band_schema = band_details
ds = buzz.DataSource()
fp = buzz.Footprint(tl=(0, 10), size=(10, 10), rsize=(30, 30))
write = ds.create_araster(
path,
fp,
dtype,
band_count,
band_schema,
driver,
options,
SRS[0]['wkt'],
)
array = np.repeat(
np.sum(fp.meshgrid_raster, 0)[:, :, np.newaxis], band_count, -1)
write.set_data(array, band=np.arange(band_count) + 1)
write.close()
# open again and check
read = ds.open_raster('read', path, driver=driver)
array2 = read.get_data(band=np.arange(band_count) + 1)
assert np.all(array == array2)
assert len(read) == band_count
assert read.fp == fp
assert read.band_schema == band_schema
if save_proj:
assert buzz.srs.wkt_same(SRS[0]['wkt'], read.wkt_origin)
def _launch_test(fps, tif_options, rast_fp_name, path):
fp = fps[rast_fp_name]
print('Working with fp={}, with options={}, ~size={:.3} GB'.format(
fp,
tif_options,
4 * fp.rarea / 1024**3,
))
ds = buzz.DataSource()
dsm = ds.create_raster(
path,
fp,
np.float32,
1,
{'nodata': -32727},
key='dsm',
sr=SRS[0]['wkt'],
options=tif_options,
)
fp_l = [fps.A, fps.Q, fps.L]
tile_size = (50000, 1000) # small height, full width, to wrap file's bands
def _build_ar(shape):
ar = np.ones(shape, dtype='float32')
diag_indices = np.diag_indices_from(ar[:min(shape), :min(shape)])
ar[diag_indices] = dsm.nodata
return ar
for fp in fp_l:
print('set', fp)
assert fp.poly.within(dsm.fp.poly)
tiles = fp.tile(tile_size,
boundary_effect='shrink') # ~1GB with float32
for tile in tiles.flatten():
ar = _build_ar(tile.shape)
print(" ar size {}GB".format(ar.dtype.itemsize *
np.prod(ar.shape) / 1024**3))
dsm.set_data(ar, fp=tile)
dsm.close()
dsm = ds.open_raster('dsm', path)
for fp in fp_l:
print('check', fp)
assert fp.poly.within(dsm.fp.poly)
tiles = fp.tile(tile_size,
boundary_effect='shrink') # ~1GB with float32
for tile in tiles.flatten():
print(' ', tile)
ar = _build_ar(tile.shape)
ar2 = dsm.get_data(fp=tile)
print(" ar size {}GB".format(ar.dtype.itemsize *
np.prod(ar.shape) / 1024**3))
print(" ar2 size {}GB".format(ar2.dtype.itemsize *
np.prod(ar2.shape) / 1024**3))
same = ar == ar2
is_ok = same.all()
assert is_ok
dsm.close()
def _test_geom_read(path, driver, fps, data):
ds = buzz.DataSource()
v = ds.open_vector('v', path, driver=driver)
queries = _build_geom_read_queries(v, fps)
for gen, slicing, _, mask, clip in queries:
# Normalize input mask
if mask is None:
pass
elif isinstance(mask, buzz.Footprint):
mask = mask.poly
elif isinstance(mask, sg.Polygon):
pass
elif isinstance(mask, (tuple, np.ndarray)):
mask = sg.box(*np.asarray(mask)[[0, 2, 1, 3]])
else:
print('fail with type', type(mask))
assert False
# Normalize input geometry
geoms_ref = [dat[0] for dat in data]
geoms_ref = [_any_geom_to_shapely(geom) for geom in geoms_ref]
if mask is not None:
geoms_ref = [geom for geom in geoms_ref if not geom.disjoint(mask)]
if clip:
geoms_ref = [geom & mask for geom in geoms_ref]
geoms_ref = geoms_ref[slicing]
# Compare
geoms = list(gen)
geoms = [_any_geom_to_shapely(geom) for geom in geoms]
geoms_ref = [_any_geom_to_shapely(geom) for geom in geoms_ref]
assert len(geoms) == len(geoms_ref)
for geom, geom_ref in zip(geoms, geoms_ref):
assert (geom ^ geom_ref).is_empty
def get_data(self, input_fp):
if not hasattr(self._thread_storage, "ds"):
ds = buzz.DataSource(allow_interpolation=True)
self._thread_storage.ds = ds
else:
ds = self._thread_storage.ds
input_data = []
intersecting_tiles = []
def tile_info_gen():
for cache_tile, filename in zip(self._cache_tiles_fps.flat,
self._cache_tile_paths):
if cache_tile.share_area(input_fp):
yield cache_tile, filename
tile_info = list(tile_info_gen())
for fp, filename in tile_info:
self._req_q.put((fp, filename))
self._req_q.join()
for fp, filename in tile_info:
with self._lock:
input_data.append(self._dico[filename].copy())
intersecting_tiles.append(fp)
return self._merge_out_tiles(intersecting_tiles, input_data, input_fp)
def create_mask(rgb_path, shp_path, mask_path):
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('rgb', rgb_path)
ds.open_vector('shp', shp_path)
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize,
)
polygons = ds.shp.iter_data(None)
mask = fp.burn_polygons(polygons)
mask_tr = mask * 255
with ds.create_raster('mask',
mask_path,
ds.rgb.fp,
'uint8',
1,
band_schema=None,
sr=ds.rgb.proj4_virtual).close:
ds.mask.set_data(mask_tr, band=1)
return True
def test_vector_fifo2_2files():
ds = buzz.DataSource(max_activated=2)
# Test with a shapefile
assert (ds._queued_count, ds._locked_count) == (0, 0)
with ds.create_avector('/tmp/v1.shp', **V_META).delete as v1:
assert (ds._queued_count, ds._locked_count, v1.activated) == (1, 0, True)
with ds.create_avector('/tmp/v2.shp', **V_META).delete as v2:
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (2, 0, True, True)
v1.insert_data((42, 43), ['fra'])
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (2, 0, True, True)
v2.insert_data((44, 45), ['ger'])
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (2, 0, True, True)
assert (len(v1), v1.get_data(0, geom_type='coordinates')) == (1, ((42, 43), 'fra'))
assert (len(v2), v2.get_data(0, geom_type='coordinates')) == (1, ((44, 45), 'ger'))
# Attempt to iterate on both
it1 = v1.iter_data()
next(it1)
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (1, 1, True, True)
it2 = v2.iter_data()
next(it2)
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (0, 2, True, True)
del it2, it1
assert (ds._queued_count, ds._locked_count, v1.activated, v2.activated) == (2, 0, True, True)
def _reproj_and_clip_vector(srcpath, dstpath):
print("Transforming: {}".format(srcpath), flush=True)
if os.path.isfile(dstpath):
print(" {} already exits".format(dstpath), flush=True)
return
with buzz.Env(significant=10,
allow_complex_footprint=True,
warnings=False):
ds = buzz.DataSource(sr_work='WGS84', analyse_transformation=False)
ds.open_vector('src', srcpath, driver='GeoJSON')
ds.open_raster('raster', rgb_wgs84_path)
ds.create_vector(
'dst',
dstpath,
'polygon',
driver='ESRI Shapefile',
sr=EPSG29100,
)
# Iterate over all geoJSON geometries overlapping with raster
# Ignoring geoJSON fields
# Save all polygons to Shapefile
for geom in ds.src.iter_data(None, mask=ds.raster.fp, clip=True):
if isinstance(geom, shapely.geometry.Polygon):
ds.dst.insert_data(geom)
elif isinstance(geom, shapely.geometry.MultiPolygon):
for poly in geom.geoms:
ds.dst.insert_data(poly)
print("Transformed: {}\n to: {}".format(srcpath, dstpath), flush=True)
def predict_from_file(rgb_path,
model,
pre_process,
downsampling_factor=3,
tile_size=128):
'''
Predict binaryzed array and adapted footprint from a file_name
Parameters
----------
rgb_path : string
file name (with extension)
model : Model object
trained model for the prediction of image (tile_size * tile_size)
downsampling_factor : int
downsampling factor (to lower resolution)
tile_size : int
size of a tile (in pixel) i.e. size of the input images
for the neural network
'''
ds_rgb = buzz.DataSource(allow_interpolation=True)
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
predicted_binary = predict_map(model, tile_size, ds_rgb, fp, pre_process)
return predicted_binary, fp
def test_mode1(fps, shp1_path, tif1_path, shp2_path, tif2_path):
ds = buzz.DataSource()
ds.open_raster('rast1', tif1_path)
ds.open_vector('poly1', shp1_path)
ds.open_raster('rast2', tif2_path)
ds.open_vector('poly2', shp2_path)
# Test footprints equality
assert fpeq(
fps.AI,
ds.rast1.fp,
ds.rast1.fp_origin,
buzz.Footprint.of_extent(ds.poly1.extent, fps.AI.scale),
ds.rast2.fp,
ds.rast2.fp_origin,
buzz.Footprint.of_extent(ds.poly2.extent, fps.AI.scale),
)
# Test what's written in all 4 files
rast1 = ds.rast1.get_data()
rast2 = ds.rast2.get_data()
for i, letter in enumerate(string.ascii_uppercase[:9]):
poly1 = ds.poly1.get_data(i, None)
raster1_polys = ds.rast1.fp.find_polygons(rast1 == ord(letter))
assert len(raster1_polys) == 1
assert (poly1 ^ raster1_polys[0]).is_empty
poly2 = ds.poly2.get_data(i, None)
raster2_polys = ds.rast2.fp.find_polygons(rast2 == ord(letter))
assert len(raster2_polys) == 1
assert (poly2 ^ raster2_polys[0]).is_empty
def get_cl(self, reso, n, fp, dist, mod, vshift, intersect=False):
ds = buzz.DataSource()
with ds.open_araster(self.dsm_path(reso, n)).close as r:
if intersect:
fp = r.fp.dilate(fp.rlength // 2) & fp
dsm = r.get_data(fp=fp)
nodata_mask = dsm == r.nodata
dsm = dsm + vshift * mod * dist
# Smooth dsm
kernel = ndi.gaussian_filter(
np.pad([[1]], 3, 'constant').astype(float), 1.5)
kernel[kernel < 0.0005] = 0
dsm_smooth = ndi.convolve(dsm, kernel, mode='constant', cval=0)
# Undo smoothing near nodata
nodata_mask_smooth = ndi.convolve(
nodata_mask.astype('float32'), kernel, mode='constant',
cval=1) != 0
near_nodata_mask = nodata_mask ^ nodata_mask_smooth
dsm_smooth[near_nodata_mask] = dsm[near_nodata_mask]
arr = dsm_smooth // dist % mod
arr = np.dstack([arr == i for i in range(mod)])
arr[nodata_mask] = 0
return arr
def get_slopes(self, reso, n, fp, normalize=True, intersect=False):
ds = buzz.DataSource()
with ds.open_araster(self.dsm_path(reso, n)).close as r:
if intersect:
fp = r.fp.dilate(fp.rlength // 2) & fp
arr = r.get_data(fp=fp.dilate(1))
nodata_mask = arr == r.nodata
nodata_mask = ndi.binary_dilation(nodata_mask)
kernel = [
[0, 1, 0],
[1, 1, 1],
[0, 1, 0],
]
arru = ndi.maximum_filter(arr, None, kernel) - arr
arru = np.arctan(arru / fp.pxsizex)
arru = arru / np.pi * 180.
arru[nodata_mask] = 0
arru = arru[1:-1, 1:-1]
arrd = arr - ndi.minimum_filter(arr, None, kernel)
arrd = np.arctan(arrd / fp.pxsizex)
arrd = arrd / np.pi * 180.
arrd[nodata_mask] = 0
arrd = arrd[1:-1, 1:-1]
arr = np.dstack([arrd, arru])
if normalize:
arr = arr / 45 - 1
return arr
def acquisition_labels_stats(dat):
reso, n, path, test, tiles = dat
with buzz.Env(significant=9):
ds = buzz.DataSource()
ds.open_raster('labs', path)
numerators_acq = np.zeros(LABEL_COUNT, int)
denominators_acq = np.zeros(LABEL_COUNT, int)
for fp in tiles:
counts_tile = np.bincount(
ds.labs.get_data(fp=fp).flatten(), None, LABEL_COUNT)
numerators_acq += counts_tile
denominators_acq += (counts_tile > 0) * fp.rarea
numerators_acq = numerators_acq * reso**2 / 1e4
denominators_acq = denominators_acq * reso**2 / 1e4
numerators_acq = [
(('nume', catname), nbr)
for catname, nbr in zip(CATEGORIES, numerators_acq)
]
denominators_acq = [
(('deno', catname), nbr)
for catname, nbr in zip(CATEGORIES, denominators_acq)
]
return collections.OrderedDict([
('name', n),
('test', test),
] + numerators_acq + denominators_acq)
def predict_file(file,
model_nn,
images_train="AerialImageDataset/train/images/",
downsampling_factor=3,
tile_size=128):
'''
Predict binaryzed array and adapted footprint from a file_name
Parameters
----------
file : string
file name (with extension)
model_nn : Model object
trained model for the prediction of image (tile_size * tile_size)
images_train : string
folder name for whole input images
downsampling_factor : int
downsampling factor (to lower resolution)
tile_size : int
size of a tile (in pixel) i.e. size of the input images
for the neural network
'''
ds_rgb = buzz.DataSource(allow_interpolation=True)
rgb_path = images_train + file
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
predicted_binary = predict_map(model_nn, tile_size, ds_rgb, fp)
return predicted_binary, fp
def display_results(file,
gt_train="AerialImageDataset/train/gt/",
images_train="AerialImageDataset/train/images/",
polygons_path="geoJSON/",
downsampling_factor=1):
geojson_file = polygons_path + file.split('.')[0] + '.geojson'
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('rgb', images_train + file)
ds.open_vector('roofs', geojson_file, driver='geoJSON')
# Build a low resolution Footprint to perform quicker calculations
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize // downsampling_factor,
)
polygons = ds.roofs.iter_data(None)
rgb = ds.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8')
fig = plt.figure(figsize=(5. / fp.height * fp.width, 5))
plt.title('Roof boundary')
ax = fig.add_subplot(111)
ax.imshow(rgb, extent=[fp.lx, fp.rx, fp.by, fp.ty])
for polygon_roof in polygons:
ax.add_patch(
descartes.PolygonPatch(polygon_roof,
fill=False,
ec='#ff0000',
lw=3,
ls='--'))
plt.show()
def test_basic(fps):
ds = buzz.DataSource()
ones = ds.create_araster('', fps.AI, 'float32', 1, driver='MEM')
ones.fill(1)
def pxfun(fp):
return ones.get_data(fp=fp) * 2
# araster
twos = ds.create_recipe_araster(pxfun, fps.AI, 'float32')
assert ones.fp == twos.fp
assert ones.dtype == twos.dtype
for fp in fps.values():
assert (twos.get_data(fp=fp) == 2).all()
twos.close()
# raster
twos = ds.create_recipe_raster('hello', pxfun, fps.AI, 'float32')
assert ones.fp == twos.fp
assert ones.dtype == twos.dtype
for fp in fps.values():
assert (twos.get_data(fp=fp) == 2).all()
twos.close()
ones.close()
def test_run(path, driver, fps, test_fields, test_coords_insertion):
ds = buzz.DataSource()
if test_fields:
fields = FIELDS
else:
fields = []
geom_type = 'polygon'
v = ds.create_avector(path,
geom_type,
fields,
driver=driver,
sr=SRS[0]['wkt'])
def _build_data():
rng = np.random.RandomState(42)
for fpname, fp in fps.items():
geom = _geom_of_fp(rng, fp, test_coords_insertion)
fields = _fields_of_fp(rng, fp,
fpname) # Keep invocation before `if`
if test_fields:
yield geom, fields
else:
yield geom,
data = list(_build_data())
for dat in data:
v.insert_data(*dat)
v.close()
del ds
_test_geom_read(path, driver, fps, data)
if test_fields:
_test_fields_read(path, driver, data)
def test_pipeline(file, model_nn, images_train, gt_train, downsampling_factor,
tile_size):
'''
Test the whole pipeline from a file name to a binaryzed array and compare
it to the referenced binary image
'''
# predinction
predicted_binary, fp = predict_file(file, model_nn, images_train,
downsampling_factor, tile_size)
# Loading reference
binary_path = gt_train + file
ds_binary = buzz.DataSource(allow_interpolation=True)
ds_binary.open_raster('rgb', binary_path)
binary = ds_binary.rgb.get_data(band=(1), fp=fp)
# Plotting results
fig = plt.figure(figsize=(5. / fp.height * fp.width, 5))
plt.title('Predicted segmentation')
ax = fig.add_subplot(111)
ax.imshow(predicted_binary, extent=[fp.lx, fp.rx, fp.by, fp.ty])
plt.show()
fig = plt.figure(figsize=(5. / fp.height * fp.width, 5))
plt.title('True segmentation')
ax = fig.add_subplot(111)
ax.imshow(binary, extent=[fp.lx, fp.rx, fp.by, fp.ty])
plt.show()
def test_vector(path, driver, test_fields):
fields = [
dict(name='name', type=str),
dict(name='area', type='float32'),
dict(name='count', type='int32'),
]
ds = buzz.DataSource()
if test_fields:
out = ds.create_avector(path,
'polygon',
fields,
driver=driver,
sr=SRS[0]['wkt'])
else:
out = ds.create_avector(path,
'polygon', [],
driver=driver,
sr=SRS[0]['wkt'])
# insert features
features = [
[
sg.box(0, 0, 1, 1),
None,
None,
None,
],
[
sg.box(0, 0, 2, 2),
'ruelle',
42.5,
10,
],
[
sg.box(0, 0, 3, 3),
'avenue',
None,
9,
],
]
if test_fields:
out.insert_data(features[0][0])
out.insert_data(features[1][0], features[1][1:])
out.insert_data(features[2][0], {
defn['name']: val
for (defn, val) in zip(fields, features[2][1:])
})
else:
for data in features:
out.insert_data(data[0])
out.close()
# close/open and check
out = ds.open_vector('out', path, mode='r', driver=driver)
for input, output in zip(features, out.iter_data()):
if test_fields:
for v1, v2 in zip(input[1:], output[1:]):
assert v1 == v2 or (v1 in {'', 0, 0., None}
and v2 in {'', 0, 0., None})
def checksum_file(path):
"""
Computes the checksum of a file
"""
ds = buzz.DataSource()
with ds.open_araster(path).close as raster:
cs = checksum(raster.get_data(band=-1))
return cs
def test_raster():
ds = buzz.DataSource(max_activated=2)
fp = buzz.Footprint(
tl=(1, 1),
size=(10, 10),
rsize=(10, 10),
)
with ds.create_araster('/tmp/t1.tif', fp, float, 1).delete as r1:
assert (ds._queued_count, ds._locked_count, r1.activated) == (1, 0, True)
with ds.create_araster('/tmp/t2.tif', fp, float, 1).delete as r2:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True)
with ds.create_araster('/tmp/t3.tif', fp, float, 1).delete as r3:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test lru policy
r1.fill(1)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, False, True)
r2.fill(2)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, True, False)
r3.fill(3)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test raster proxy
def pxfn(fp):
return np.ones(fp.shape) * 42
with ds.create_recipe_araster(pxfn, fp, 'float32').close as r4:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.deactivate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.activate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
assert (r4.get_data() == 42).all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test MEM raster (should behave like raster proxy in this case)
with ds.create_araster('', fp, float, 1, driver='MEM').close as r4:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.deactivate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.activate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.fill(42)
assert (r4.get_data() == 42).all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test full activations / deactivations
with pytest.raises(RuntimeError, match='max_activated'):
ds.activate_all()
ds.deactivate_all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (0, 0, False, False, False)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (0, 0, False, False)
ds.activate_all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True)
def _test_fields_read(path, driver, data):
"""Test fields reading with iter_data
Not testing geojson, no testing get_* functions
Not actually testing fields value,
- testing that all test yield the same thing
- testing that all lengths are ok
"""
query_waysssss = [
[[], '', None],
[[0], 'rarea', ['rarea']],
[[1], 'fpname', ['fpname']],
[[2], 'sqrtarea', ['sqrtarea']],
[[0, 1], 'rarea,fpname', ['rarea', 'fpname'], [0, 'fpname'],
['rarea', 1]],
[[0, 2], 'rarea,sqrtarea', ['rarea', 'sqrtarea'], [0, 'sqrtarea'],
['rarea', 2]],
[[1, 0], 'fpname,rarea', ['fpname', 'rarea'], [1, 'rarea'],
['fpname', 0]],
[[1, 2], 'fpname,sqrtarea', ['fpname', 'sqrtarea'], [1, 'sqrtarea'],
['fpname', 2]],
[[2, 1], 'sqrtarea,fpname', ['sqrtarea', 'fpname'], [2, 'fpname'],
['sqrtarea', 1]],
[[2, 0], 'sqrtarea,rarea', ['sqrtarea', 'rarea'], [2, 'rarea'],
['sqrtarea', 0]],
[[0, 1, 2], -1, ['rarea', 'fpname', 'sqrtarea'],
[0, 'fpname', 'sqrtarea'], ['rarea', 1, 2]],
]
ds = buzz.DataSource()
v = ds.open_vector('v', path, driver=driver)
for query_ways in query_waysssss:
# All queries in `query_ways` request the same thing in a different way
indices = query_ways[0]
queries_results = []
for query in query_ways:
features = list(v.iter_data(query))
assert len(features) == len(data)
query_result = []
for feature in features:
# For each feature in the polygon
if len(indices) == 0:
feature = [feature]
fields = feature[1:]
assert len(fields) == len(indices)
query_result.append(fields) # Build list of fields
queries_results.append(
query_result) # Build list of list of fields
del query_result
assert len(queries_results) == len(query_ways)
_assert_all_list_of_fields_same(queries_results)
def get_rgb(self, reso, n, fp, normalize=True, intersect=False):
ds = buzz.DataSource()
with ds.open_araster(self.ortho_path(reso, n)).close as r:
if intersect:
fp = r.fp.dilate(fp.rlength // 2) & fp
arr = r.get_data(band=(1, 2, 3), fp=fp, nodata=0).astype('uint8')
if normalize:
arr = arr.astype('float32') * 2 / 255 - 1
return arr
def convert_to_polygons(self, predicted_binary, fp):
predicted_mask = (predicted_binary > self.threshold)*255
poly = fp.find_polygons(predicted_mask)
ds = buzz.DataSource(allow_interpolation=True)
self.geojson_path = args.results_dir + self.args.aoi_file[:-4] +"_ds"+str(self.downs)+str("_ths_")+str(self.threshold)+str("_inria_post_process")+'.geojson'
ds.create_vector('dst', self.geojson_path, 'polygon', driver='GeoJSON')
for i in tqdm(range(len(poly))):
ds.dst.insert_data(poly[i])
ds.dst.close()
def get_data(self, input_fp):
output_data = []
intersecting_tiles = []
if not hasattr(self._thread_storage, "ds"):
ds = buzz.DataSource(allow_interpolation=True)
self._thread_storage.ds = ds
else:
ds = self._thread_storage.ds
def tile_info_gen():
for cache_tile, filename in zip(self._cache_tiles_fps.flat,
self._cache_tile_paths):
if cache_tile.share_area(input_fp):
yield cache_tile, filename
for cache_tile, filepath in tile_info_gen():
file_exists = os.path.isfile(filepath)
if not file_exists:
with self._lock:
# checking again if the file exists (happens when entering the lock after waiting)
file_exists = os.path.isfile(filepath)
if not file_exists:
print("--> using GPU...")
prediction = self._double_tiled_structure.get_cache_data(
cache_tile)
print("<-- no more GPU")
out_proxy = ds.create_araster(
filepath,
cache_tile,
"float32",
LABEL_COUNT,
driver="GTiff",
sr=self._resampled_rgba.wkt_origin)
out_proxy.set_data(prediction, band=-1)
out_proxy.close()
if file_exists:
print("!! cache was calculated when i was waiting")
with ds.open_araster(filepath).close as src:
prediction = src.get_data(band=-1)
else:
with ds.open_araster(filepath).close as src:
prediction = src.get_data(band=-1)
intersecting_tiles.append(cache_tile)
output_data.append(prediction)
return self._merge_out_tiles(intersecting_tiles, output_data, input_fp)
def _get_tile_data(self, tile_path):
if not hasattr(self._thread_storage, "ds"):
ds = buzz.DataSource(allow_interpolation=True)
self._thread_storage.ds = ds
else:
ds = self._thread_storage.ds
with ds.open_araster(tile_path).close as raster:
out = raster.get_data(band=-1)
return out
def get_labels(self, n, fp, intersect=False):
if n in NAMES:
ds = buzz.DataSource()
with ds.open_araster(self.labels_path(n)).close as r:
if intersect:
fp = r.fp.dilate(fp.rlength // 2) & fp
labs = r.get_data(fp=fp, nodata=0).astype('uint8')
else:
labs = np.zeros(fp.shape, 'uint8')
return labs