def random_crop(self, ds, tile_size=128, factor=1):
# Cropping parameters
crop_factor = ds.rgb.fp.rsize / tile_size
crop_size = ds.rgb.fp.size / crop_factor
# Original footprint
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize,
)
min = np.random.randint(fp.tl[0], fp.tl[0] + fp.size[0] -
factor * crop_size[0]) #x
max = np.random.randint(fp.tl[1] - fp.size[1] + factor * crop_size[1],
fp.tl[1]) #y
# print(min > fp.tlx, max < fp.tly)
# print("fp.tlx", fp.tlx, "fp.tly", fp.tly, "min , max", min, max)
# New random footprint
tl = np.array([min, max])
fp = buzz.Footprint(
tl=tl,
size=crop_size * factor,
rsize=[tile_size, tile_size],
)
return fp
def test_non_trivial_accessors(fps):
assert eq(fps.AI.semimajoraxis, fps.AH.semimajoraxis, fps.AG.semimajoraxis, )
assert eq(fps.BH.semiminoraxis, fps.BE.semiminoraxis, fps.B.semiminoraxis, )
assert eq(
fps.A.length + fps.B.length + fps.D.length + fps.E.length,
fps.AE.length + fps.A.length * 2
)
assert eq(fps.AI.rsemimajoraxis, fps.AH.rsemimajoraxis, fps.AG.rsemimajoraxis, )
assert eq(fps.BH.rsemiminoraxis, fps.BE.rsemiminoraxis, fps.B.rsemiminoraxis, )
assert eq(fps.AI.rarea, np.prod(fps.AI.rsize), np.prod(fps.AI.size / fps.AI.pxsize))
assert eq(fps.AI.rlength,
# sum of sides minus 4
fps.AI.rsizex * 2 + fps.AI.rsizey * 2 - 4,
# sum of smaller rlength plus delta
fps.AC.rlength + fps.AD.rsizey * 2,)
fp = buzz.Footprint(gt=fps.AI.gt, rsize=(2, 10))
assert eq(fp.rsemiminoraxis, 1)
assert eq(fp.rlength, fp.rsemimajoraxis * 4)
fp = buzz.Footprint(gt=fps.AI.gt, rsize=(1, 10))
assert eq(fp.rsemiminoraxis, 1, tol=1)
assert eq(fp.rlength, fp.rsemimajoraxis * 2)
fp = buzz.Footprint(gt=fps.AI.gt, rsize=(1, 1))
assert eq(fp.rsemiminoraxis, 1, fp.rsemimajoraxis, tol=1)
assert eq(fp.rlength, 1)
def create_work_tiles(fp, work_overlap_fraction):
"""Create the tiling of `fp` that is compatible with the model `m` and all its inputs.
Constraint 1: Should be aligned with all model's input files
Constraint 2: Two neighboring tiles should have the same global aligment in their poolings
to avoid huge boundary effect when stitching tiles.
Parameters
----------
fp: Footprint
The heatmap footprint to compute
m: Model
work_overlap_fraction: float
Fraction of a tile to overlap with its neighbor
Returns
-------
work_tiles: ndarray of Footprint of shape (H, W)
Tiling with overlap and modulo alignment
"""
# Phase 0 - Recreate `fp` such that it can serve as a basis for tiling
fp0 = fp
fp1 = fp0.intersection(fp0, scale=1.28, alignment=(0, 0))
fp2 = buzz.Footprint(gt=fp1.gt, rsize=(512, 512))
fp3 = fp2.intersection(fp2, scale=0.64)
fp3 = fp3.erode((fp3.rw - 500) / 2)
assert np.allclose(fp2.c, fp3.c)
fp4 = fp3.move(tl=fp3.tl -
np.ceil((fp3.tl - fp0.tl) / fp2.scale) * fp2.scale)
fp5 = buzz.Footprint(
gt=fp4.gt,
rsize=fp4.rsize + np.around((fp0.br - fp4.br) / fp0.pxvec),
)
assert np.allclose(fp5.br, fp0.br)
fp = fp5
# Phase 1 - Calculate the stride between neighboring tiles ********************************** **
work_rsize = np.array([500, 500])
work_modulo = 8.
x = work_rsize * (1 - work_overlap_fraction)
x = x / work_modulo
x = np.around(x)
x = x * work_modulo
x = x.clip(work_modulo, work_rsize // work_modulo * work_modulo)
x = x.astype(int)
aligned_rsize = x
# Phase 2 - Instanciate tiles *************************************************************** **
aligned_tiles = fp.tile(aligned_rsize)
work_tiles = np.vectorize(lambda tile: buzz.Footprint(
gt=tile.gt, rsize=work_rsize))(aligned_tiles)
return work_tiles
def create_mask(rgb_path, shp_path, mask_path):
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('rgb', rgb_path)
ds.open_vector('shp', shp_path)
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize,
)
polygons = ds.shp.iter_data(None)
mask = fp.burn_polygons(polygons)
mask_tr = mask * 255
with ds.create_raster('mask',
mask_path,
ds.rgb.fp,
'uint8',
1,
band_schema=None,
sr=ds.rgb.proj4_virtual).close:
ds.mask.set_data(mask_tr, band=1)
return True
def generate_dsm(rsize=(16000, 16000),
resolution=0.03,
delta_z=(10, 110),
roughness=0.45,
nb_houses=5,
verbose=False):
"""generates a `.tif` file containing an artificial dsm.
the generated dsm does not have a projection, nor no_data values.
Parameters
----------
- rsize: (int, int)
desired raster size
- resolution: (float)
desired resolution in meters
- delta_z: (float, float)
span of elevations on generated dsm in meters
- roughness: float
value ranging from 0 to 1. 0 will generate a very smooth dsm (a plane). 0.5 generates
midly rough landscapes (imagine a valley in the Swiss Alps). 1 generates very sharp results.
- nb_houses: int
number of houses to be added on the dsm
- verbose: boolean
if True, some information about the generated dsm will be printed.
"""
w, l = rsize
min_z, max_z = delta_z
if verbose:
print("==== Metrics on generated dsm ====")
print(f" w, l = {w}, {l}")
print(f" resolution = {resolution}")
print(f" roughness = {roughness}")
print(f" min_z, max_z = {min_z}, {max_z}")
print(f" nb_houses = {nb_houses}")
dsm = diamond_square((l, w), min_z, max_z, roughness)
for _ in range(nb_houses):
_put_house_on_dsm(dsm, resolution, verbose)
tlx = np.random.uniform(42, 1337)
tly = np.random.uniform(32, 111)
fp = buzz.Footprint(tl=(tlx, tly),
size=(w * resolution, l * resolution),
rsize=(w, l))
ds = buzz.Dataset(allow_interpolation=False)
filename = f'{uuid.uuid4()}.tif'
if verbose:
print(f' {fp}')
print(' filename = ' + filename)
with ds.acreate_raster(filename,
fp,
dtype='float32',
channel_count=1,
sr=None).close as out:
out.set_data(dsm)
return filename
def make_tif2(path, reso=(1., -1.), rsize=(10, 10), tl=(0., 10.),
proj=SRS[0]['wkt'], band_count=1, dtype=gdal.GDT_Float32,
nodata=-32000, nodata_border_size=(0, 0, 0, 0)):
"""Create a tiff files"""
reso = np.asarray(reso)
fp = buzz.Footprint(tl=tl, rsize=rsize, size=np.abs(reso * rsize))
x, y = fp.meshgrid_raster
a = x + y
if nodata_border_size != 0:
l, r, t, b = nodata_border_size
if t != 0:
a[None:t, None:None] = nodata
if b != 0:
a[-b:None, None:None] = nodata
if l != 0:
a[None:None, None:l] = nodata
if r != 0:
a[None:None, -r:None] = nodata
LOGGER.info('TIFF ARRAY:%s\n', a)
gdal.UseExceptions()
driver = gdal.GetDriverByName('GTiff')
dataset = driver.Create(path, int(rsize[0]), int(rsize[1]), band_count, dtype)
dataset.SetGeoTransform(fp.gt)
dataset.SetProjection(proj)
for i in range(band_count):
dataset.GetRasterBand(i + 1).WriteArray(a)
dataset.GetRasterBand(i + 1).SetNoDataValue(nodata)
dataset.FlushCache()
def test_concurrent_cached():
computation_pool = mp.pool.ThreadPool(1)
io_pool = mp.pool.ThreadPool(1)
footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10))
obs0 = RasterStateOberver()
obs1 = RasterStateOberver()
obs2 = RasterStateOberver()
def compute_data(fp, *args):
return np.zeros(fp.shape)
simple_raster = Raster(footprint=footprint,
computation_function=compute_data,
computation_pool=computation_pool,
io_pool=io_pool,
cached=True,
cache_dir='./test_cache/0/',
cache_fps=footprint.tile_count(
3, 3, boundary_effect='shrink'),
overwrite=True,
debug_callbacks=[obs0.callback])
dependent_raster_1 = Raster(
footprint=footprint,
computation_function=compute_data,
computation_pool=computation_pool,
io_pool=io_pool,
primitives={"prim": simple_raster.get_multi_data_queue},
to_collect_of_to_compute=lambda fp: {"prim": fp},
cached=True,
cache_dir='./test_cache/1/',
cache_fps=footprint.tile_count(3, 3, boundary_effect='shrink'),
overwrite=True,
debug_callbacks=[obs1.callback])
dependent_raster_2 = Raster(
footprint=footprint,
computation_function=compute_data,
computation_pool=computation_pool,
io_pool=io_pool,
primitives={"prim": simple_raster.get_multi_data_queue},
to_collect_of_to_compute=lambda fp: {"prim": fp},
cached=True,
cache_dir='./test_cache/2/',
cache_fps=footprint.tile_count(3, 3, boundary_effect='shrink'),
overwrite=True,
debug_callbacks=[obs2.callback])
arrays1 = list(
dependent_raster_1.get_multi_data(footprint.tile_count(5, 5).flat))
arrays2 = list(
dependent_raster_2.get_multi_data(
reversed(list(footprint.tile_count(5, 5).flat))))
assert np.all(arrays1[0] == 0)
assert np.all(arrays2[0] == 0)
print("base", obs0.df)
print("1", obs1.df)
print("2", obs2.df)
def test_simple_cached():
computation_pool = mp.pool.ThreadPool(1)
io_pool = mp.pool.ThreadPool(1)
footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10))
obs = RasterStateOberver()
def compute_data(fp, *args):
return np.zeros(fp.shape)
simple_raster = Raster(footprint=footprint,
computation_function=compute_data,
computation_pool=computation_pool,
io_pool=io_pool,
cached=True,
cache_dir='./test_cache/',
cache_fps=footprint.tile_count(
3, 3, boundary_effect='shrink'),
overwrite=True,
debug_callbacks=[
obs.callback,
])
array = simple_raster.get_data(footprint)
assert np.all(array == 0)
def make_tif(path,
tloffset=(0, 0),
reso=(0.25, -0.25),
rsize=(20, 10),
proj=SRS[0]['wkt'],
channel_count=1,
dtype=gdal.GDT_Float32):
"""Create a tiff files and return info about it"""
tl = ROOT_TL + tloffset
reso = np.asarray(reso)
fp = buzz.Footprint(tl=tl, rsize=rsize, size=np.abs(reso * rsize))
x, y = fp.meshgrid_spatial
x = np.abs(x) - abs(ROOT_TL[0])
y = abs(ROOT_TL[1]) - np.abs(y)
x *= 15
y *= 15
a = x / 2 + y / 2
a = np.around(a).astype('float32')
driver = gdal.GetDriverByName('GTiff')
dataset = driver.Create(path, rsize[0], rsize[1], channel_count, dtype)
dataset.SetGeoTransform(fp.gt)
dataset.SetProjection(proj)
for i in range(channel_count):
dataset.GetRasterBand(i + 1).WriteArray(a)
dataset.GetRasterBand(i + 1).SetNoDataValue(-32000.)
dataset.FlushCache()
return path, fp, a
def test_move(fps1px):
fps = fps1px
with buzz.Env(warnings=False, allow_complex_footprint=True): # Test the `warnings` deprecation
assert fpeq(
fps.B,
fps.A.move(fps.B.tl),
fps.B.move(fps.B.tl),
fps.C.move(fps.B.tl),
fps.A.move(fps.B.tl, fps.B.tr),
fps.B.move(fps.B.tl, fps.B.tr),
fps.C.move(fps.B.tl, fps.B.tr),
fps.A.move(fps.B.tl, fps.B.tr, fps.B.br),
fps.B.move(fps.B.tl, fps.B.tr, fps.B.br),
fps.C.move(fps.B.tl, fps.B.tr, fps.B.br),
)
aff = (
Affine.translation(*fps.A.bl) * Affine.rotation(45) * Affine.scale(2**0.5, 2**0.5 * -2)
)
assert fpeq(
buzz.Footprint(gt=aff.to_gdal(), rsize=(1, 1)),
fps.A.move(fps.A.bl, fps.A.tr, fps.I.tr),
fps.B.move(fps.A.bl, fps.A.tr, fps.I.tr),
fps.C.move(fps.A.bl, fps.A.tr, fps.I.tr),
)
with pytest.raises(ValueError, match='angle'):
fps.C.move(fps.A.bl, fps.A.tr, fps.I.c)
def predict_from_file(rgb_path,
model,
downsampling_factor=1,
tile_size=256,
no_of_gpu=1,
batch_size=2):
ds_rgb = buzz.Dataset(allow_interpolation=True)
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
overlapx = int(tile_size / 2)
overlapy = int(tile_size / 2)
try:
predicted_probamap = predict_map(model, ds_rgb, fp, tile_size,
overlapx, overlapy, batch_size)
except Exception as e:
print(e)
print("Retrying prediction with reduced batch size")
predicted_probamap = predict_map(model, ds_rgb, fp, tile_size,
overlapx, overlapy, batch_size // 2)
return predicted_probamap, fp
def display_results(file,
gt_train="AerialImageDataset/train/gt/",
images_train="AerialImageDataset/train/images/",
polygons_path="geoJSON/",
downsampling_factor=1):
geojson_file = polygons_path + file.split('.')[0] + '.geojson'
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('rgb', images_train + file)
ds.open_vector('roofs', geojson_file, driver='geoJSON')
# Build a low resolution Footprint to perform quicker calculations
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize // downsampling_factor,
)
polygons = ds.roofs.iter_data(None)
rgb = ds.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8')
fig = plt.figure(figsize=(5. / fp.height * fp.width, 5))
plt.title('Roof boundary')
ax = fig.add_subplot(111)
ax.imshow(rgb, extent=[fp.lx, fp.rx, fp.by, fp.ty])
for polygon_roof in polygons:
ax.add_patch(
descartes.PolygonPatch(polygon_roof,
fill=False,
ec='#ff0000',
lw=3,
ls='--'))
plt.show()
def predict_file(file,
model_nn,
images_train="AerialImageDataset/train/images/",
downsampling_factor=3,
tile_size=128):
'''
Predict binaryzed array and adapted footprint from a file_name
Parameters
----------
file : string
file name (with extension)
model_nn : Model object
trained model for the prediction of image (tile_size * tile_size)
images_train : string
folder name for whole input images
downsampling_factor : int
downsampling factor (to lower resolution)
tile_size : int
size of a tile (in pixel) i.e. size of the input images
for the neural network
'''
ds_rgb = buzz.DataSource(allow_interpolation=True)
rgb_path = images_train + file
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
predicted_binary = predict_map(model_nn, tile_size, ds_rgb, fp)
return predicted_binary, fp
def test_gc():
ws = weakref.WeakSet()
fp = buzz.Footprint(
tl=(1, 1),
size=(10, 10),
rsize=(10, 10),
)
def pxfn(fp):
return np.ones(fp.shape)
def _test():
ds = buzz.DataSource()
prox = ds.create_araster('', fp, float, 1, driver='MEM')
ds.create_vector('vec', '', 'point', driver='Memory')
ws.add(ds)
ws.add(prox)
_test()
gc.collect()
assert len(ws) == 0
def _test():
ds = buzz.DataSource()
prox = ds.create_recipe_araster(pxfn, fp, float)
ws.add(ds)
ws.add(prox)
_test()
gc.collect()
assert len(ws) == 0
def test_raster(path, driver, band_details, options, save_proj):
dtype, band_count, band_schema = band_details
ds = buzz.DataSource()
fp = buzz.Footprint(tl=(0, 10), size=(10, 10), rsize=(30, 30))
write = ds.create_araster(
path,
fp,
dtype,
band_count,
band_schema,
driver,
options,
SRS[0]['wkt'],
)
array = np.repeat(
np.sum(fp.meshgrid_raster, 0)[:, :, np.newaxis], band_count, -1)
write.set_data(array, band=np.arange(band_count) + 1)
write.close()
# open again and check
read = ds.open_raster('read', path, driver=driver)
array2 = read.get_data(band=np.arange(band_count) + 1)
assert np.all(array == array2)
assert len(read) == band_count
assert read.fp == fp
assert read.band_schema == band_schema
if save_proj:
assert buzz.srs.wkt_same(SRS[0]['wkt'], read.wkt_origin)
def test_coordinates_accessors_spatial_corners(fps):
buzz.Footprint(gt=fps.A.gt, rsize=fps.A.rsize)
assert eq(
fps.E.tl,
fps.B.bl,
fps.A.br,
fps.D.tr,
)
assert eq(
fps.E.bl,
fps.D.br,
fps.G.tr,
fps.H.tl,
)
assert eq(
fps.E.br,
fps.H.tr,
fps.I.tl,
fps.F.bl,
)
assert eq(
fps.E.tr,
fps.F.tl,
fps.C.bl,
fps.B.br,
)
for letter in LETTERS:
those_tl = [v.tl for k, v in fps.items() if k.startswith(letter)]
assert eq(*those_tl)
those_br = [v.br for k, v in fps.items() if k.endswith(letter)]
assert eq(*those_br)
def predict_from_file(rgb_path,
model,
pre_process,
downsampling_factor=3,
tile_size=128):
'''
Predict binaryzed array and adapted footprint from a file_name
Parameters
----------
rgb_path : string
file name (with extension)
model : Model object
trained model for the prediction of image (tile_size * tile_size)
downsampling_factor : int
downsampling factor (to lower resolution)
tile_size : int
size of a tile (in pixel) i.e. size of the input images
for the neural network
'''
ds_rgb = buzz.DataSource(allow_interpolation=True)
ds_rgb.open_raster('rgb', rgb_path)
fp = buzz.Footprint(
tl=ds_rgb.rgb.fp.tl,
size=ds_rgb.rgb.fp.size,
rsize=ds_rgb.rgb.fp.rsize / downsampling_factor,
) #unsampling
predicted_binary = predict_map(model, tile_size, ds_rgb, fp, pre_process)
return predicted_binary, fp
def test_raster():
ds = buzz.DataSource(max_activated=2)
fp = buzz.Footprint(
tl=(1, 1),
size=(10, 10),
rsize=(10, 10),
)
with ds.create_araster('/tmp/t1.tif', fp, float, 1).delete as r1:
assert (ds._queued_count, ds._locked_count, r1.activated) == (1, 0, True)
with ds.create_araster('/tmp/t2.tif', fp, float, 1).delete as r2:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True)
with ds.create_araster('/tmp/t3.tif', fp, float, 1).delete as r3:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test lru policy
r1.fill(1)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, False, True)
r2.fill(2)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, True, False)
r3.fill(3)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test raster proxy
def pxfn(fp):
return np.ones(fp.shape) * 42
with ds.create_recipe_araster(pxfn, fp, 'float32').close as r4:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.deactivate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.activate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
assert (r4.get_data() == 42).all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test MEM raster (should behave like raster proxy in this case)
with ds.create_araster('', fp, float, 1, driver='MEM').close as r4:
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.deactivate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.activate()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True)
r4.fill(42)
assert (r4.get_data() == 42).all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True)
# Test full activations / deactivations
with pytest.raises(RuntimeError, match='max_activated'):
ds.activate_all()
ds.deactivate_all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (0, 0, False, False, False)
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (0, 0, False, False)
ds.activate_all()
assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True)
def _foorprint_of_char(c):
ys, xs = np.where(chars_grid == c)
minx = xs.min()
maxx = xs.max()
miny = ys.min()
maxy = ys.max()
rsize = maxx - minx + 1, maxy - miny + 1
fp = buzz.Footprint(tl=(minx, -miny), rsize=rsize, size=rsize)
return fp
def _footprint_per_zoom():
for zoom in range(zoom_count):
pxsizex = 2**(zoom_count - zoom - 1)
scale = np.asarray([pxsizex, -pxsizex])
pxvec = scale
rsize = np.floor(full_rsize / pxsizex).astype(int)
size = rsize * pxsizex
tl = images_center - (rsize / 2 * pxvec)
yield buzz.Footprint(tl=tl, size=size, rsize=rsize)
def test_pickling():
def pxfn(fp):
"""Pixel function for recipe. `ds` lives in the closure"""
return np.ones(fp.shape) * id(ds)
def slave():
print('slave', dd.get_worker())
"""Slave process routine. `ds` and `oldid` live in the closure and are pickled by cloudpickle in Client.sumbit"""
assert id(ds) != oldid, 'this test makes sense if `ds` was pickled'
assert 'v1' in ds
assert 'v2' not in ds
assert 'r1' in ds
assert 'r2' not in ds
assert 'r3' in ds
assert (ds._queued_count, ds._locked_count, ds.v1.activated, ds.r1.activated, ds.r3.activated) == (0, 0, False, False, True)
assert ds.v1.get_data(0)[1] == str(oldid)
assert (ds.r1.get_data() == oldid).all()
assert (ds.r3.get_data() == id(ds)).all(), '`slave` and `pxfn` should share the same `ds` obj'
ds.v1.insert_data((0, 1), ['42'])
ds.r1.fill(42)
assert ds.v1.get_data(1)[1] == '42'
assert (ds.r1.get_data() == 42).all()
ds.deactivate_all()
ds = buzz.DataSource(max_activated=2)
oldid = id(ds)
fp = buzz.Footprint(
tl=(1, 1),
size=(10, 10),
rsize=(10, 10),
)
clust = dd.LocalCluster(n_workers=1, threads_per_worker=1, scheduler_port=0)
print()
print(clust)
cl = dd.Client(clust)
print(cl)
with ds.create_vector('v1', '/tmp/v1.shp', **V_META).delete:
with ds.create_raster('r1', '/tmp/t1.shp', fp, float, 1).delete:
ds.create_raster('r2', '', fp, float, 1, driver='MEM')
ds.create_recipe_raster('r3', pxfn, fp, float)
ds.create_vector('v2',**MEMV_META)
ds.v1.insert_data((0, 1), [str(oldid)])
ds.v2.insert_data((0, 1), [str(oldid)])
ds.r1.fill(oldid)
ds.r2.fill(oldid)
ds.deactivate_all()
cl.submit(slave).result()
assert ds.v1.get_data(1)[1] == '42'
assert (ds.r1.get_data() == 42).all()
def test_reproj():
sr0 = SRS[0]
sr1 = SRS[3]
with buzz.Env(significant=8, allow_complex_footprint=1, warnings=0):
# Create `twos`, a recipe from `sr1` to `sr0`
# `fp` in sr0
# `ds.wkt` in sr0
# `twos.fp` in sr0
# `twos.fp_origin` in sr1
# `pxfun@fp` in sr1
fp = buzz.Footprint(
tl=(sr0['cx'], sr0['cy']),
size=(2, 2),
rsize=(20, 20),
)
ds = buzz.DataSource(sr0['wkt'])
def pxfun(fp):
assert (twos.fp_origin
& fp) == fp, 'fp should be aligned and within twos.fp'
return ones.get_data(fp=fp) * 2
twos = ds.create_recipe_araster(pxfun,
fp,
'float32',
sr=sr1['wkt'],
band_schema={'nodata': 42})
# Create `ones`, a raster from `sr1` to `sr1`
# `ds2.wkt` in sr1
# `ones.fp` in sr1
# `ones.fp_origin` in sr1
ds2 = buzz.DataSource(sr1['wkt'])
ones = ds2.create_araster('',
twos.fp_origin,
'float32',
1,
driver='MEM',
sr=sr1['wkt'],
band_schema={'nodata': 42})
ones.fill(1)
# Test that `sr1` performs reprojection of `fp` before sending it to `pxfun`
assert ones.fp == ones.fp_origin, 'ones has no reproj'
assert twos.fp_origin == ones.fp_origin
assert ones.dtype == twos.dtype
tiles = fp.tile_count(
3, 3, boundary_effect='shrink').flatten().tolist() + [fp]
for tile in tiles:
assert (twos.get_data(fp=tile) == 2).all()
twos.close()
ones.close()
def random_crop(ds, crop_rsize=700, factor=1, rotation=0):
'''
Returns a random crop of a geotiff buzzard datasource
Parameters
----------
ds : DataSource
Input buzzard DataSource
crop_size : int
Size in pixel of the cropped final image
factor : int
Downsampling factor
rotation : float
Rotation angle (rad)
Return
------
fp : Footprint
Cropped footprint
'''
# Cropping parameters
crop_factor = ds.rgb.fp.rsize / crop_rsize
crop_size = ds.rgb.fp.size / crop_factor
# Original footprint
fp = buzz.Footprint(
tl=ds.rgb.fp.tl,
size=ds.rgb.fp.size,
rsize=ds.rgb.fp.rsize,
)
# New random footprint
tl = np.array([
np.random.randint(fp.tl[0],
fp.tl[0] + fp.size[0] - factor * crop_size[0]), #x
np.random.randint(fp.tl[1] - fp.size[1] + factor * crop_size[1],
fp.tl[1]) #y
])
fp = buzz.Footprint(
tl=tl,
size=crop_size * factor,
rsize=[crop_rsize, crop_rsize],
)
return fp
def output_fp_to_input_fp(fp, scale, rsize):
"""
Creates a footprint of rsize from fp using dilation and scale
Used to compute a keras model input footprint from output
(e.g.: deduce the big RGB and slopes extents from the smaller output heatmap)
"""
out = buzz.Footprint(tl=fp.tl,
size=fp.size,
rsize=np.around(fp.size / scale))
padding = (rsize - out.rsizex) / 2
assert padding == int(padding)
out = out.dilate(padding)
return out
def test_save_polynoms(file):
'''
test polynom identification of saving for one file
'''
# Loading reference
binary_path = gt_train + file
ds_binary = buzz.DataSource(allow_interpolation=True)
ds_binary.open_raster('rgb', binary_path)
fp = buzz.Footprint(
tl=ds_binary.rgb.fp.tl,
size=ds_binary.rgb.fp.size,
rsize=ds_binary.rgb.fp.rsize / downsampling_factor,
) #unsampling
binary = ds_binary.rgb.get_data(band=(1), fp=fp)
save_polynoms(file, binary, fp)
def _simulate_sparsity(self, gt, old_annot, diff_gt_pred, sparsity):
"""
3 cases:
- Training (simulate at random)
- Test initialization (no annotations)
- Test after at least one inference (simulate using the wrong prediction map)
"""
if sparsity == 0 or sparsity is None:
sparse_gt = np.zeros_like(gt)
else:
if self.train:
flat_gt = gt.reshape((-1))
tot_pixs = flat_gt.shape[0]
probs = np.ones((tot_pixs), dtype=np.float32)
if self.weights is not None:
for i in range(self.n_classes):
probs[flat_gt == i] = self.weights[i]
probs /= np.sum(probs) # normalize
sparse_points = random.choice(np.prod(gt.shape),
sparsity,
replace=False,
p=probs)
sparse_gt = np.zeros_like(flat_gt)
sparse_gt[sparse_points] = 1
sparse_gt = sparse_gt.reshape(*gt.shape)
else:
sparse_gt = old_annot.copy()
fp = buzz.Footprint(gt=(0, 0.1, 0, 10, 0, -0.1),
rsize=gt.swapaxes(1, 0).shape)
polygons = fp.find_polygons(diff_gt_pred)
if not len(polygons):
return sparse_gt
order = np.argsort([p.area for p in polygons
])[max(np.random.randint(-8, 0),
-len(polygons))]
polygon = polygons[order]
center = fp.spatial_to_raster(
np.asarray(polygon.representative_point().xy).transpose(
(1, 0)))[0]
loc = [
min(sparse_gt.shape[0] - 1, max(0, center[1])),
min(sparse_gt.shape[1] - 1, max(0, center[0])),
]
sparse_gt[loc[0], loc[1]] = 1
return sparse_gt
def test_simple_raster():
computation_pool = mp.pool.ThreadPool(1)
io_pool = mp.pool.ThreadPool(1)
footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10))
obs = RasterStateOberver()
def compute_data(fp, *args):
return np.zeros(fp.shape)
simple_raster = Raster(footprint=footprint,
computation_function=compute_data,
computation_pool=computation_pool,
io_pool=io_pool,
debug_callbacks=[obs.callback])
array = simple_raster.get_data(footprint)
assert np.all(array == 0)
print(obs.df)
def poly_to_binary(gt_path, geojson_path, downsampling_factor):
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('binary', gt_path)
ds.open_vector('polygons', geojson_path, driver='geoJSON')
fp = buzz.Footprint(
tl=ds.binary.fp.tl,
size=ds.binary.fp.size,
rsize=ds.binary.fp.rsize / downsampling_factor,
)
binary = ds.binary.get_data(band=(1), fp=fp).astype('uint8')
binary_predict = np.zeros_like(binary)
for poly in ds.polygons.iter_data(None):
mark_poly = fp.burn_polygons(poly)
binary_predict[mark_poly] = 1
return binary, binary_predict
def poly_to_binary(file,
gt_train="AerialImageDataset/train/gt/",
polygons_path="geoJSON/",
downsampling_factor=4):
ds = buzz.DataSource(allow_interpolation=True)
ds.open_raster('binary', gt_train + file)
geojson_file = polygons_path + file.split('.')[0] + '.geojson'
ds.open_vector('polygons', geojson_file, driver='geoJSON')
fp = buzz.Footprint(
tl=ds.binary.fp.tl,
size=ds.binary.fp.size,
rsize=ds.binary.fp.rsize / downsampling_factor,
)
binary = ds.binary.get_data(band=(1), fp=fp).astype('uint8')
binary_predict = np.zeros_like(binary)
for poly in ds.polygons.iter_data(None):
mark_poly = fp.burn_polygons(poly)
binary_predict[mark_poly] = 1
return binary, binary_predict
def test_raster():
ds = buzz.DataSource(max_activated=2)
meta = dict(
fp=buzz.Footprint(
tl=(1, 1),
size=(10, 10),
rsize=(10, 10),
),
dtype=float,
band_count=1,
)
def statuses(*args):
l = tuple([(ds._back.idle_count(prox._back.uid),
ds._back.used_count(prox._back.uid), prox.active_count,
prox.active) for prox in args])
return l
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 0, 0)
with ds.acreate_raster('/tmp/t1.tif', **meta).delete as r1:
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 0, 1)
assert (ds._back.idle_count(r1._back.uid),
ds._back.used_count(r1._back.uid), r1.active_count,
r1.active) == (1, 0, 1, True)
with ds.acreate_raster('/tmp/t2.tif', **meta).delete as r2:
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2) == ((1, 0, 1, True), (1, 0, 1, True))
with ds.acreate_raster('/tmp/t3.tif', **meta).delete as r3:
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2,
r3) == ((0, 0, 0, False), (1, 0, 1, True),
(1, 0, 1, True))
r1.activate()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2,
r3) == ((1, 0, 1, True), (0, 0, 0, False),
(1, 0, 1, True))
r1.deactivate()
r2.activate()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2,
r3) == ((0, 0, 0, False), (1, 0, 1, True),
(1, 0, 1, True))
with pytest.raises(RuntimeError, match='max_activated'):
ds.activate_all()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2,
r3) == ((0, 0, 0, False), (1, 0, 1, True),
(1, 0, 1, True))
ds.deactivate_all()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 0, 0)
assert statuses(r1, r2,
r3) == ((0, 0, 0, False), (0, 0, 0, False),
(0, 0, 0, False))
r1.fill(42)
r2.fill(42)
r3.fill(42)
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2,
r3) == ((0, 0, 0, False), (1, 0, 1, True),
(1, 0, 1, True))
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (1, 0, 1)
assert statuses(r1, r2) == ((0, 0, 0, False), (1, 0, 1, True))
ds.deactivate_all()
ds.activate_all()
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (2, 0, 2)
assert statuses(r1, r2) == ((1, 0, 1, True), (1, 0, 1, True))
return
assert (ds._back.idle_count(), ds._back.used_count(),
ds.active_count) == (0, 0, 0)
