def random_crop(self, ds, tile_size=128, factor=1): # Cropping parameters crop_factor = ds.rgb.fp.rsize / tile_size crop_size = ds.rgb.fp.size / crop_factor # Original footprint fp = buzz.Footprint( tl=ds.rgb.fp.tl, size=ds.rgb.fp.size, rsize=ds.rgb.fp.rsize, ) min = np.random.randint(fp.tl[0], fp.tl[0] + fp.size[0] - factor * crop_size[0]) #x max = np.random.randint(fp.tl[1] - fp.size[1] + factor * crop_size[1], fp.tl[1]) #y # print(min > fp.tlx, max < fp.tly) # print("fp.tlx", fp.tlx, "fp.tly", fp.tly, "min , max", min, max) # New random footprint tl = np.array([min, max]) fp = buzz.Footprint( tl=tl, size=crop_size * factor, rsize=[tile_size, tile_size], ) return fp
def test_non_trivial_accessors(fps): assert eq(fps.AI.semimajoraxis, fps.AH.semimajoraxis, fps.AG.semimajoraxis, ) assert eq(fps.BH.semiminoraxis, fps.BE.semiminoraxis, fps.B.semiminoraxis, ) assert eq( fps.A.length + fps.B.length + fps.D.length + fps.E.length, fps.AE.length + fps.A.length * 2 ) assert eq(fps.AI.rsemimajoraxis, fps.AH.rsemimajoraxis, fps.AG.rsemimajoraxis, ) assert eq(fps.BH.rsemiminoraxis, fps.BE.rsemiminoraxis, fps.B.rsemiminoraxis, ) assert eq(fps.AI.rarea, np.prod(fps.AI.rsize), np.prod(fps.AI.size / fps.AI.pxsize)) assert eq(fps.AI.rlength, # sum of sides minus 4 fps.AI.rsizex * 2 + fps.AI.rsizey * 2 - 4, # sum of smaller rlength plus delta fps.AC.rlength + fps.AD.rsizey * 2,) fp = buzz.Footprint(gt=fps.AI.gt, rsize=(2, 10)) assert eq(fp.rsemiminoraxis, 1) assert eq(fp.rlength, fp.rsemimajoraxis * 4) fp = buzz.Footprint(gt=fps.AI.gt, rsize=(1, 10)) assert eq(fp.rsemiminoraxis, 1, tol=1) assert eq(fp.rlength, fp.rsemimajoraxis * 2) fp = buzz.Footprint(gt=fps.AI.gt, rsize=(1, 1)) assert eq(fp.rsemiminoraxis, 1, fp.rsemimajoraxis, tol=1) assert eq(fp.rlength, 1)
def create_work_tiles(fp, work_overlap_fraction): """Create the tiling of `fp` that is compatible with the model `m` and all its inputs. Constraint 1: Should be aligned with all model's input files Constraint 2: Two neighboring tiles should have the same global aligment in their poolings to avoid huge boundary effect when stitching tiles. Parameters ---------- fp: Footprint The heatmap footprint to compute m: Model work_overlap_fraction: float Fraction of a tile to overlap with its neighbor Returns ------- work_tiles: ndarray of Footprint of shape (H, W) Tiling with overlap and modulo alignment """ # Phase 0 - Recreate `fp` such that it can serve as a basis for tiling fp0 = fp fp1 = fp0.intersection(fp0, scale=1.28, alignment=(0, 0)) fp2 = buzz.Footprint(gt=fp1.gt, rsize=(512, 512)) fp3 = fp2.intersection(fp2, scale=0.64) fp3 = fp3.erode((fp3.rw - 500) / 2) assert np.allclose(fp2.c, fp3.c) fp4 = fp3.move(tl=fp3.tl - np.ceil((fp3.tl - fp0.tl) / fp2.scale) * fp2.scale) fp5 = buzz.Footprint( gt=fp4.gt, rsize=fp4.rsize + np.around((fp0.br - fp4.br) / fp0.pxvec), ) assert np.allclose(fp5.br, fp0.br) fp = fp5 # Phase 1 - Calculate the stride between neighboring tiles ********************************** ** work_rsize = np.array([500, 500]) work_modulo = 8. x = work_rsize * (1 - work_overlap_fraction) x = x / work_modulo x = np.around(x) x = x * work_modulo x = x.clip(work_modulo, work_rsize // work_modulo * work_modulo) x = x.astype(int) aligned_rsize = x # Phase 2 - Instanciate tiles *************************************************************** ** aligned_tiles = fp.tile(aligned_rsize) work_tiles = np.vectorize(lambda tile: buzz.Footprint( gt=tile.gt, rsize=work_rsize))(aligned_tiles) return work_tiles
def create_mask(rgb_path, shp_path, mask_path): ds = buzz.DataSource(allow_interpolation=True) ds.open_raster('rgb', rgb_path) ds.open_vector('shp', shp_path) fp = buzz.Footprint( tl=ds.rgb.fp.tl, size=ds.rgb.fp.size, rsize=ds.rgb.fp.rsize, ) polygons = ds.shp.iter_data(None) mask = fp.burn_polygons(polygons) mask_tr = mask * 255 with ds.create_raster('mask', mask_path, ds.rgb.fp, 'uint8', 1, band_schema=None, sr=ds.rgb.proj4_virtual).close: ds.mask.set_data(mask_tr, band=1) return True
def generate_dsm(rsize=(16000, 16000), resolution=0.03, delta_z=(10, 110), roughness=0.45, nb_houses=5, verbose=False): """generates a `.tif` file containing an artificial dsm. the generated dsm does not have a projection, nor no_data values. Parameters ---------- - rsize: (int, int) desired raster size - resolution: (float) desired resolution in meters - delta_z: (float, float) span of elevations on generated dsm in meters - roughness: float value ranging from 0 to 1. 0 will generate a very smooth dsm (a plane). 0.5 generates midly rough landscapes (imagine a valley in the Swiss Alps). 1 generates very sharp results. - nb_houses: int number of houses to be added on the dsm - verbose: boolean if True, some information about the generated dsm will be printed. """ w, l = rsize min_z, max_z = delta_z if verbose: print("==== Metrics on generated dsm ====") print(f" w, l = {w}, {l}") print(f" resolution = {resolution}") print(f" roughness = {roughness}") print(f" min_z, max_z = {min_z}, {max_z}") print(f" nb_houses = {nb_houses}") dsm = diamond_square((l, w), min_z, max_z, roughness) for _ in range(nb_houses): _put_house_on_dsm(dsm, resolution, verbose) tlx = np.random.uniform(42, 1337) tly = np.random.uniform(32, 111) fp = buzz.Footprint(tl=(tlx, tly), size=(w * resolution, l * resolution), rsize=(w, l)) ds = buzz.Dataset(allow_interpolation=False) filename = f'{uuid.uuid4()}.tif' if verbose: print(f' {fp}') print(' filename = ' + filename) with ds.acreate_raster(filename, fp, dtype='float32', channel_count=1, sr=None).close as out: out.set_data(dsm) return filename
def make_tif2(path, reso=(1., -1.), rsize=(10, 10), tl=(0., 10.), proj=SRS[0]['wkt'], band_count=1, dtype=gdal.GDT_Float32, nodata=-32000, nodata_border_size=(0, 0, 0, 0)): """Create a tiff files""" reso = np.asarray(reso) fp = buzz.Footprint(tl=tl, rsize=rsize, size=np.abs(reso * rsize)) x, y = fp.meshgrid_raster a = x + y if nodata_border_size != 0: l, r, t, b = nodata_border_size if t != 0: a[None:t, None:None] = nodata if b != 0: a[-b:None, None:None] = nodata if l != 0: a[None:None, None:l] = nodata if r != 0: a[None:None, -r:None] = nodata LOGGER.info('TIFF ARRAY:%s\n', a) gdal.UseExceptions() driver = gdal.GetDriverByName('GTiff') dataset = driver.Create(path, int(rsize[0]), int(rsize[1]), band_count, dtype) dataset.SetGeoTransform(fp.gt) dataset.SetProjection(proj) for i in range(band_count): dataset.GetRasterBand(i + 1).WriteArray(a) dataset.GetRasterBand(i + 1).SetNoDataValue(nodata) dataset.FlushCache()
def test_concurrent_cached(): computation_pool = mp.pool.ThreadPool(1) io_pool = mp.pool.ThreadPool(1) footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10)) obs0 = RasterStateOberver() obs1 = RasterStateOberver() obs2 = RasterStateOberver() def compute_data(fp, *args): return np.zeros(fp.shape) simple_raster = Raster(footprint=footprint, computation_function=compute_data, computation_pool=computation_pool, io_pool=io_pool, cached=True, cache_dir='./test_cache/0/', cache_fps=footprint.tile_count( 3, 3, boundary_effect='shrink'), overwrite=True, debug_callbacks=[obs0.callback]) dependent_raster_1 = Raster( footprint=footprint, computation_function=compute_data, computation_pool=computation_pool, io_pool=io_pool, primitives={"prim": simple_raster.get_multi_data_queue}, to_collect_of_to_compute=lambda fp: {"prim": fp}, cached=True, cache_dir='./test_cache/1/', cache_fps=footprint.tile_count(3, 3, boundary_effect='shrink'), overwrite=True, debug_callbacks=[obs1.callback]) dependent_raster_2 = Raster( footprint=footprint, computation_function=compute_data, computation_pool=computation_pool, io_pool=io_pool, primitives={"prim": simple_raster.get_multi_data_queue}, to_collect_of_to_compute=lambda fp: {"prim": fp}, cached=True, cache_dir='./test_cache/2/', cache_fps=footprint.tile_count(3, 3, boundary_effect='shrink'), overwrite=True, debug_callbacks=[obs2.callback]) arrays1 = list( dependent_raster_1.get_multi_data(footprint.tile_count(5, 5).flat)) arrays2 = list( dependent_raster_2.get_multi_data( reversed(list(footprint.tile_count(5, 5).flat)))) assert np.all(arrays1[0] == 0) assert np.all(arrays2[0] == 0) print("base", obs0.df) print("1", obs1.df) print("2", obs2.df)
def test_simple_cached(): computation_pool = mp.pool.ThreadPool(1) io_pool = mp.pool.ThreadPool(1) footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10)) obs = RasterStateOberver() def compute_data(fp, *args): return np.zeros(fp.shape) simple_raster = Raster(footprint=footprint, computation_function=compute_data, computation_pool=computation_pool, io_pool=io_pool, cached=True, cache_dir='./test_cache/', cache_fps=footprint.tile_count( 3, 3, boundary_effect='shrink'), overwrite=True, debug_callbacks=[ obs.callback, ]) array = simple_raster.get_data(footprint) assert np.all(array == 0)
def make_tif(path, tloffset=(0, 0), reso=(0.25, -0.25), rsize=(20, 10), proj=SRS[0]['wkt'], channel_count=1, dtype=gdal.GDT_Float32): """Create a tiff files and return info about it""" tl = ROOT_TL + tloffset reso = np.asarray(reso) fp = buzz.Footprint(tl=tl, rsize=rsize, size=np.abs(reso * rsize)) x, y = fp.meshgrid_spatial x = np.abs(x) - abs(ROOT_TL[0]) y = abs(ROOT_TL[1]) - np.abs(y) x *= 15 y *= 15 a = x / 2 + y / 2 a = np.around(a).astype('float32') driver = gdal.GetDriverByName('GTiff') dataset = driver.Create(path, rsize[0], rsize[1], channel_count, dtype) dataset.SetGeoTransform(fp.gt) dataset.SetProjection(proj) for i in range(channel_count): dataset.GetRasterBand(i + 1).WriteArray(a) dataset.GetRasterBand(i + 1).SetNoDataValue(-32000.) dataset.FlushCache() return path, fp, a
def test_move(fps1px): fps = fps1px with buzz.Env(warnings=False, allow_complex_footprint=True): # Test the `warnings` deprecation assert fpeq( fps.B, fps.A.move(fps.B.tl), fps.B.move(fps.B.tl), fps.C.move(fps.B.tl), fps.A.move(fps.B.tl, fps.B.tr), fps.B.move(fps.B.tl, fps.B.tr), fps.C.move(fps.B.tl, fps.B.tr), fps.A.move(fps.B.tl, fps.B.tr, fps.B.br), fps.B.move(fps.B.tl, fps.B.tr, fps.B.br), fps.C.move(fps.B.tl, fps.B.tr, fps.B.br), ) aff = ( Affine.translation(*fps.A.bl) * Affine.rotation(45) * Affine.scale(2**0.5, 2**0.5 * -2) ) assert fpeq( buzz.Footprint(gt=aff.to_gdal(), rsize=(1, 1)), fps.A.move(fps.A.bl, fps.A.tr, fps.I.tr), fps.B.move(fps.A.bl, fps.A.tr, fps.I.tr), fps.C.move(fps.A.bl, fps.A.tr, fps.I.tr), ) with pytest.raises(ValueError, match='angle'): fps.C.move(fps.A.bl, fps.A.tr, fps.I.c)
def predict_from_file(rgb_path, model, downsampling_factor=1, tile_size=256, no_of_gpu=1, batch_size=2): ds_rgb = buzz.Dataset(allow_interpolation=True) ds_rgb.open_raster('rgb', rgb_path) fp = buzz.Footprint( tl=ds_rgb.rgb.fp.tl, size=ds_rgb.rgb.fp.size, rsize=ds_rgb.rgb.fp.rsize / downsampling_factor, ) #unsampling overlapx = int(tile_size / 2) overlapy = int(tile_size / 2) try: predicted_probamap = predict_map(model, ds_rgb, fp, tile_size, overlapx, overlapy, batch_size) except Exception as e: print(e) print("Retrying prediction with reduced batch size") predicted_probamap = predict_map(model, ds_rgb, fp, tile_size, overlapx, overlapy, batch_size // 2) return predicted_probamap, fp
def display_results(file, gt_train="AerialImageDataset/train/gt/", images_train="AerialImageDataset/train/images/", polygons_path="geoJSON/", downsampling_factor=1): geojson_file = polygons_path + file.split('.')[0] + '.geojson' ds = buzz.DataSource(allow_interpolation=True) ds.open_raster('rgb', images_train + file) ds.open_vector('roofs', geojson_file, driver='geoJSON') # Build a low resolution Footprint to perform quicker calculations fp = buzz.Footprint( tl=ds.rgb.fp.tl, size=ds.rgb.fp.size, rsize=ds.rgb.fp.rsize // downsampling_factor, ) polygons = ds.roofs.iter_data(None) rgb = ds.rgb.get_data(band=(1, 2, 3), fp=fp).astype('uint8') fig = plt.figure(figsize=(5. / fp.height * fp.width, 5)) plt.title('Roof boundary') ax = fig.add_subplot(111) ax.imshow(rgb, extent=[fp.lx, fp.rx, fp.by, fp.ty]) for polygon_roof in polygons: ax.add_patch( descartes.PolygonPatch(polygon_roof, fill=False, ec='#ff0000', lw=3, ls='--')) plt.show()
def predict_file(file, model_nn, images_train="AerialImageDataset/train/images/", downsampling_factor=3, tile_size=128): ''' Predict binaryzed array and adapted footprint from a file_name Parameters ---------- file : string file name (with extension) model_nn : Model object trained model for the prediction of image (tile_size * tile_size) images_train : string folder name for whole input images downsampling_factor : int downsampling factor (to lower resolution) tile_size : int size of a tile (in pixel) i.e. size of the input images for the neural network ''' ds_rgb = buzz.DataSource(allow_interpolation=True) rgb_path = images_train + file ds_rgb.open_raster('rgb', rgb_path) fp = buzz.Footprint( tl=ds_rgb.rgb.fp.tl, size=ds_rgb.rgb.fp.size, rsize=ds_rgb.rgb.fp.rsize / downsampling_factor, ) #unsampling predicted_binary = predict_map(model_nn, tile_size, ds_rgb, fp) return predicted_binary, fp
def test_gc(): ws = weakref.WeakSet() fp = buzz.Footprint( tl=(1, 1), size=(10, 10), rsize=(10, 10), ) def pxfn(fp): return np.ones(fp.shape) def _test(): ds = buzz.DataSource() prox = ds.create_araster('', fp, float, 1, driver='MEM') ds.create_vector('vec', '', 'point', driver='Memory') ws.add(ds) ws.add(prox) _test() gc.collect() assert len(ws) == 0 def _test(): ds = buzz.DataSource() prox = ds.create_recipe_araster(pxfn, fp, float) ws.add(ds) ws.add(prox) _test() gc.collect() assert len(ws) == 0
def test_raster(path, driver, band_details, options, save_proj): dtype, band_count, band_schema = band_details ds = buzz.DataSource() fp = buzz.Footprint(tl=(0, 10), size=(10, 10), rsize=(30, 30)) write = ds.create_araster( path, fp, dtype, band_count, band_schema, driver, options, SRS[0]['wkt'], ) array = np.repeat( np.sum(fp.meshgrid_raster, 0)[:, :, np.newaxis], band_count, -1) write.set_data(array, band=np.arange(band_count) + 1) write.close() # open again and check read = ds.open_raster('read', path, driver=driver) array2 = read.get_data(band=np.arange(band_count) + 1) assert np.all(array == array2) assert len(read) == band_count assert read.fp == fp assert read.band_schema == band_schema if save_proj: assert buzz.srs.wkt_same(SRS[0]['wkt'], read.wkt_origin)
def test_coordinates_accessors_spatial_corners(fps): buzz.Footprint(gt=fps.A.gt, rsize=fps.A.rsize) assert eq( fps.E.tl, fps.B.bl, fps.A.br, fps.D.tr, ) assert eq( fps.E.bl, fps.D.br, fps.G.tr, fps.H.tl, ) assert eq( fps.E.br, fps.H.tr, fps.I.tl, fps.F.bl, ) assert eq( fps.E.tr, fps.F.tl, fps.C.bl, fps.B.br, ) for letter in LETTERS: those_tl = [v.tl for k, v in fps.items() if k.startswith(letter)] assert eq(*those_tl) those_br = [v.br for k, v in fps.items() if k.endswith(letter)] assert eq(*those_br)
def predict_from_file(rgb_path, model, pre_process, downsampling_factor=3, tile_size=128): ''' Predict binaryzed array and adapted footprint from a file_name Parameters ---------- rgb_path : string file name (with extension) model : Model object trained model for the prediction of image (tile_size * tile_size) downsampling_factor : int downsampling factor (to lower resolution) tile_size : int size of a tile (in pixel) i.e. size of the input images for the neural network ''' ds_rgb = buzz.DataSource(allow_interpolation=True) ds_rgb.open_raster('rgb', rgb_path) fp = buzz.Footprint( tl=ds_rgb.rgb.fp.tl, size=ds_rgb.rgb.fp.size, rsize=ds_rgb.rgb.fp.rsize / downsampling_factor, ) #unsampling predicted_binary = predict_map(model, tile_size, ds_rgb, fp, pre_process) return predicted_binary, fp
def test_raster(): ds = buzz.DataSource(max_activated=2) fp = buzz.Footprint( tl=(1, 1), size=(10, 10), rsize=(10, 10), ) with ds.create_araster('/tmp/t1.tif', fp, float, 1).delete as r1: assert (ds._queued_count, ds._locked_count, r1.activated) == (1, 0, True) with ds.create_araster('/tmp/t2.tif', fp, float, 1).delete as r2: assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True) with ds.create_araster('/tmp/t3.tif', fp, float, 1).delete as r3: assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True) # Test lru policy r1.fill(1) assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, False, True) r2.fill(2) assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, True, True, False) r3.fill(3) assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True) # Test raster proxy def pxfn(fp): return np.ones(fp.shape) * 42 with ds.create_recipe_araster(pxfn, fp, 'float32').close as r4: assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) r4.deactivate() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) r4.activate() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) assert (r4.get_data() == 42).all() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True) # Test MEM raster (should behave like raster proxy in this case) with ds.create_araster('', fp, float, 1, driver='MEM').close as r4: assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) r4.deactivate() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) r4.activate() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated, r4.activated) == (2, 0, False, True, True, True) r4.fill(42) assert (r4.get_data() == 42).all() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (2, 0, False, True, True) # Test full activations / deactivations with pytest.raises(RuntimeError, match='max_activated'): ds.activate_all() ds.deactivate_all() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated, r3.activated) == (0, 0, False, False, False) assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (0, 0, False, False) ds.activate_all() assert (ds._queued_count, ds._locked_count, r1.activated, r2.activated) == (2, 0, True, True)
def _foorprint_of_char(c): ys, xs = np.where(chars_grid == c) minx = xs.min() maxx = xs.max() miny = ys.min() maxy = ys.max() rsize = maxx - minx + 1, maxy - miny + 1 fp = buzz.Footprint(tl=(minx, -miny), rsize=rsize, size=rsize) return fp
def _footprint_per_zoom(): for zoom in range(zoom_count): pxsizex = 2**(zoom_count - zoom - 1) scale = np.asarray([pxsizex, -pxsizex]) pxvec = scale rsize = np.floor(full_rsize / pxsizex).astype(int) size = rsize * pxsizex tl = images_center - (rsize / 2 * pxvec) yield buzz.Footprint(tl=tl, size=size, rsize=rsize)
def test_pickling(): def pxfn(fp): """Pixel function for recipe. `ds` lives in the closure""" return np.ones(fp.shape) * id(ds) def slave(): print('slave', dd.get_worker()) """Slave process routine. `ds` and `oldid` live in the closure and are pickled by cloudpickle in Client.sumbit""" assert id(ds) != oldid, 'this test makes sense if `ds` was pickled' assert 'v1' in ds assert 'v2' not in ds assert 'r1' in ds assert 'r2' not in ds assert 'r3' in ds assert (ds._queued_count, ds._locked_count, ds.v1.activated, ds.r1.activated, ds.r3.activated) == (0, 0, False, False, True) assert ds.v1.get_data(0)[1] == str(oldid) assert (ds.r1.get_data() == oldid).all() assert (ds.r3.get_data() == id(ds)).all(), '`slave` and `pxfn` should share the same `ds` obj' ds.v1.insert_data((0, 1), ['42']) ds.r1.fill(42) assert ds.v1.get_data(1)[1] == '42' assert (ds.r1.get_data() == 42).all() ds.deactivate_all() ds = buzz.DataSource(max_activated=2) oldid = id(ds) fp = buzz.Footprint( tl=(1, 1), size=(10, 10), rsize=(10, 10), ) clust = dd.LocalCluster(n_workers=1, threads_per_worker=1, scheduler_port=0) print() print(clust) cl = dd.Client(clust) print(cl) with ds.create_vector('v1', '/tmp/v1.shp', **V_META).delete: with ds.create_raster('r1', '/tmp/t1.shp', fp, float, 1).delete: ds.create_raster('r2', '', fp, float, 1, driver='MEM') ds.create_recipe_raster('r3', pxfn, fp, float) ds.create_vector('v2',**MEMV_META) ds.v1.insert_data((0, 1), [str(oldid)]) ds.v2.insert_data((0, 1), [str(oldid)]) ds.r1.fill(oldid) ds.r2.fill(oldid) ds.deactivate_all() cl.submit(slave).result() assert ds.v1.get_data(1)[1] == '42' assert (ds.r1.get_data() == 42).all()
def test_reproj(): sr0 = SRS[0] sr1 = SRS[3] with buzz.Env(significant=8, allow_complex_footprint=1, warnings=0): # Create `twos`, a recipe from `sr1` to `sr0` # `fp` in sr0 # `ds.wkt` in sr0 # `twos.fp` in sr0 # `twos.fp_origin` in sr1 # `pxfun@fp` in sr1 fp = buzz.Footprint( tl=(sr0['cx'], sr0['cy']), size=(2, 2), rsize=(20, 20), ) ds = buzz.DataSource(sr0['wkt']) def pxfun(fp): assert (twos.fp_origin & fp) == fp, 'fp should be aligned and within twos.fp' return ones.get_data(fp=fp) * 2 twos = ds.create_recipe_araster(pxfun, fp, 'float32', sr=sr1['wkt'], band_schema={'nodata': 42}) # Create `ones`, a raster from `sr1` to `sr1` # `ds2.wkt` in sr1 # `ones.fp` in sr1 # `ones.fp_origin` in sr1 ds2 = buzz.DataSource(sr1['wkt']) ones = ds2.create_araster('', twos.fp_origin, 'float32', 1, driver='MEM', sr=sr1['wkt'], band_schema={'nodata': 42}) ones.fill(1) # Test that `sr1` performs reprojection of `fp` before sending it to `pxfun` assert ones.fp == ones.fp_origin, 'ones has no reproj' assert twos.fp_origin == ones.fp_origin assert ones.dtype == twos.dtype tiles = fp.tile_count( 3, 3, boundary_effect='shrink').flatten().tolist() + [fp] for tile in tiles: assert (twos.get_data(fp=tile) == 2).all() twos.close() ones.close()
def random_crop(ds, crop_rsize=700, factor=1, rotation=0): ''' Returns a random crop of a geotiff buzzard datasource Parameters ---------- ds : DataSource Input buzzard DataSource crop_size : int Size in pixel of the cropped final image factor : int Downsampling factor rotation : float Rotation angle (rad) Return ------ fp : Footprint Cropped footprint ''' # Cropping parameters crop_factor = ds.rgb.fp.rsize / crop_rsize crop_size = ds.rgb.fp.size / crop_factor # Original footprint fp = buzz.Footprint( tl=ds.rgb.fp.tl, size=ds.rgb.fp.size, rsize=ds.rgb.fp.rsize, ) # New random footprint tl = np.array([ np.random.randint(fp.tl[0], fp.tl[0] + fp.size[0] - factor * crop_size[0]), #x np.random.randint(fp.tl[1] - fp.size[1] + factor * crop_size[1], fp.tl[1]) #y ]) fp = buzz.Footprint( tl=tl, size=crop_size * factor, rsize=[crop_rsize, crop_rsize], ) return fp
def output_fp_to_input_fp(fp, scale, rsize): """ Creates a footprint of rsize from fp using dilation and scale Used to compute a keras model input footprint from output (e.g.: deduce the big RGB and slopes extents from the smaller output heatmap) """ out = buzz.Footprint(tl=fp.tl, size=fp.size, rsize=np.around(fp.size / scale)) padding = (rsize - out.rsizex) / 2 assert padding == int(padding) out = out.dilate(padding) return out
def test_save_polynoms(file): ''' test polynom identification of saving for one file ''' # Loading reference binary_path = gt_train + file ds_binary = buzz.DataSource(allow_interpolation=True) ds_binary.open_raster('rgb', binary_path) fp = buzz.Footprint( tl=ds_binary.rgb.fp.tl, size=ds_binary.rgb.fp.size, rsize=ds_binary.rgb.fp.rsize / downsampling_factor, ) #unsampling binary = ds_binary.rgb.get_data(band=(1), fp=fp) save_polynoms(file, binary, fp)
def _simulate_sparsity(self, gt, old_annot, diff_gt_pred, sparsity): """ 3 cases: - Training (simulate at random) - Test initialization (no annotations) - Test after at least one inference (simulate using the wrong prediction map) """ if sparsity == 0 or sparsity is None: sparse_gt = np.zeros_like(gt) else: if self.train: flat_gt = gt.reshape((-1)) tot_pixs = flat_gt.shape[0] probs = np.ones((tot_pixs), dtype=np.float32) if self.weights is not None: for i in range(self.n_classes): probs[flat_gt == i] = self.weights[i] probs /= np.sum(probs) # normalize sparse_points = random.choice(np.prod(gt.shape), sparsity, replace=False, p=probs) sparse_gt = np.zeros_like(flat_gt) sparse_gt[sparse_points] = 1 sparse_gt = sparse_gt.reshape(*gt.shape) else: sparse_gt = old_annot.copy() fp = buzz.Footprint(gt=(0, 0.1, 0, 10, 0, -0.1), rsize=gt.swapaxes(1, 0).shape) polygons = fp.find_polygons(diff_gt_pred) if not len(polygons): return sparse_gt order = np.argsort([p.area for p in polygons ])[max(np.random.randint(-8, 0), -len(polygons))] polygon = polygons[order] center = fp.spatial_to_raster( np.asarray(polygon.representative_point().xy).transpose( (1, 0)))[0] loc = [ min(sparse_gt.shape[0] - 1, max(0, center[1])), min(sparse_gt.shape[1] - 1, max(0, center[0])), ] sparse_gt[loc[0], loc[1]] = 1 return sparse_gt
def test_simple_raster(): computation_pool = mp.pool.ThreadPool(1) io_pool = mp.pool.ThreadPool(1) footprint = buzz.Footprint(tl=(0, 0), size=(10, 10), rsize=(10, 10)) obs = RasterStateOberver() def compute_data(fp, *args): return np.zeros(fp.shape) simple_raster = Raster(footprint=footprint, computation_function=compute_data, computation_pool=computation_pool, io_pool=io_pool, debug_callbacks=[obs.callback]) array = simple_raster.get_data(footprint) assert np.all(array == 0) print(obs.df)
def poly_to_binary(gt_path, geojson_path, downsampling_factor): ds = buzz.DataSource(allow_interpolation=True) ds.open_raster('binary', gt_path) ds.open_vector('polygons', geojson_path, driver='geoJSON') fp = buzz.Footprint( tl=ds.binary.fp.tl, size=ds.binary.fp.size, rsize=ds.binary.fp.rsize / downsampling_factor, ) binary = ds.binary.get_data(band=(1), fp=fp).astype('uint8') binary_predict = np.zeros_like(binary) for poly in ds.polygons.iter_data(None): mark_poly = fp.burn_polygons(poly) binary_predict[mark_poly] = 1 return binary, binary_predict
def poly_to_binary(file, gt_train="AerialImageDataset/train/gt/", polygons_path="geoJSON/", downsampling_factor=4): ds = buzz.DataSource(allow_interpolation=True) ds.open_raster('binary', gt_train + file) geojson_file = polygons_path + file.split('.')[0] + '.geojson' ds.open_vector('polygons', geojson_file, driver='geoJSON') fp = buzz.Footprint( tl=ds.binary.fp.tl, size=ds.binary.fp.size, rsize=ds.binary.fp.rsize / downsampling_factor, ) binary = ds.binary.get_data(band=(1), fp=fp).astype('uint8') binary_predict = np.zeros_like(binary) for poly in ds.polygons.iter_data(None): mark_poly = fp.burn_polygons(poly) binary_predict[mark_poly] = 1 return binary, binary_predict
def test_raster(): ds = buzz.DataSource(max_activated=2) meta = dict( fp=buzz.Footprint( tl=(1, 1), size=(10, 10), rsize=(10, 10), ), dtype=float, band_count=1, ) def statuses(*args): l = tuple([(ds._back.idle_count(prox._back.uid), ds._back.used_count(prox._back.uid), prox.active_count, prox.active) for prox in args]) return l assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (0, 0, 0) with ds.acreate_raster('/tmp/t1.tif', **meta).delete as r1: assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (1, 0, 1) assert (ds._back.idle_count(r1._back.uid), ds._back.used_count(r1._back.uid), r1.active_count, r1.active) == (1, 0, 1, True) with ds.acreate_raster('/tmp/t2.tif', **meta).delete as r2: assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2) == ((1, 0, 1, True), (1, 0, 1, True)) with ds.acreate_raster('/tmp/t3.tif', **meta).delete as r3: assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2, r3) == ((0, 0, 0, False), (1, 0, 1, True), (1, 0, 1, True)) r1.activate() assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2, r3) == ((1, 0, 1, True), (0, 0, 0, False), (1, 0, 1, True)) r1.deactivate() r2.activate() assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2, r3) == ((0, 0, 0, False), (1, 0, 1, True), (1, 0, 1, True)) with pytest.raises(RuntimeError, match='max_activated'): ds.activate_all() assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2, r3) == ((0, 0, 0, False), (1, 0, 1, True), (1, 0, 1, True)) ds.deactivate_all() assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (0, 0, 0) assert statuses(r1, r2, r3) == ((0, 0, 0, False), (0, 0, 0, False), (0, 0, 0, False)) r1.fill(42) r2.fill(42) r3.fill(42) assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2, r3) == ((0, 0, 0, False), (1, 0, 1, True), (1, 0, 1, True)) assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (1, 0, 1) assert statuses(r1, r2) == ((0, 0, 0, False), (1, 0, 1, True)) ds.deactivate_all() ds.activate_all() assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (2, 0, 2) assert statuses(r1, r2) == ((1, 0, 1, True), (1, 0, 1, True)) return assert (ds._back.idle_count(), ds._back.used_count(), ds.active_count) == (0, 0, 0)