def __init__(self, imagedir, maskdir, joint_transform=None, indices=None): """ imagedir: flat directory of N-band TIFF images as above maskdir: flat directory of 1-band TIFF masks as above joint_transform = a transforms.JointTransform object to apply to data indices = optional list of 0-indexed image indices for subsetting. """ super().__init__() self.joint_transform = joint_transform self.imagedir = imagedir self.maskdir = maskdir self.images = os.listdir(imagedir) self.masks = os.listdir(maskdir) print(self.imagedir) if indices is not None: self.images = [self.images[i] for i in indices] self.imagetiles = [Tile(*f.split("_")[:3]) for f in self.images] self.masktiles = [Tile(*f.split("_")[:3]) for f in self.masks] assert (set(self.imagetiles).issubset(set( self.masktiles))), "Image and Mask tilesets must be equal!"
def prefetch(self, tile: Tile, host_dir: str = None, verbose: bool = False): x, y, z = tile if z != int(z): raise ValueError("Fractional zooms not allowed!") if x == int(x) and y == int(y): x, y = int(x), int(y) tile_parents = [parent(tile, zoom=i) for i in range(z + 1)] for tile_parent in tile_parents: if tile_parent in self.cached_tiles: return if z < 14: blob = children(tile, zoom=14) if verbose: from tqdm import tqdm blob = tqdm(blob, desc="OpenStreetMap Prefetch") for child in blob: self.prefetch(child) else: self.__get_from_osmnx(tile) if host_dir is not None: self.save(host_dir) return source_tiles = [ Tile(np.trunc(x), np.trunc(y), z), Tile(np.trunc(x), np.ceil(y), z), Tile(np.ceil(x), np.trunc(y), z), Tile(np.ceil(x), np.ceil(y), z), ] for tile in source_tiles: self.prefetch(tile) return
def gdf_from_tile(self, tile: Tile) -> gpd.GeoDataFrame: x, y, z = tile if z != int(z): raise ValueError("Fractional zooms not allowed!") if x == int(x) and y == int(y): x, y, z = int(x), int(y), int(z) tile_parents = [parent((x, y, z), zoom=i) for i in range(z + 1)] for tile_parent in tile_parents: if tile_parent in self.cached_tiles: return self.__get_from_cache(tile) if self.use_overpass: print( f"[WARNING] Fetching {tile} from OpenStreetMap! This is slower than using regional shapefiles." ) return self.__get_from_osmnx(tile) else: return self.__get_from_cache(tile) source_tiles = [ Tile(np.trunc(x), np.trunc(y), z), Tile(np.trunc(x), np.ceil(y), z), Tile(np.ceil(x), np.trunc(y), z), Tile(np.ceil(x), np.ceil(y), z), ] returned_gdfs = [self.gdf_from_tile(tile) for tile in source_tiles] return pd.concat(returned_gdfs, ignore_index=True)
def test_kml_tiles(): tiles = { Tile(x=8783, y=5410, z=14), Tile(x=8783, y=5411, z=14), Tile(x=8783, y=5412, z=14), } assert statshunters.kml_tiles(statshunters.tiles_geometry(tiles)) != ""
def pred_2geojson(csv_file, keyword, threshold): features = list() with open(csv_file, 'r') as csvfile: reader = csv.reader(csvfile) next(reader) for row in tqdm(reader): pred = json.loads(row[1]) pred_red = list(map(lambda x: round(x, 2), pred)) pred_obj = dict( zip(map(lambda x: 'p%s' % x, range(len(pred_red))), pred_red)) if keyword == "point": pred_j = ul(*[int(t) for t in row[0].split('-')]) feature_collection = Feature(geometry=dict( type='Point', coordinates=[pred_j.lng, pred_j.lat]), properties=pred_obj) else: pred_j = feature(Tile(*[int(t) for t in row[0].split('-')])) feature_ = Feature(geometry=pred_j['geometry'], properties=pred_obj) if pred_obj['p1'] >= float(threshold): features.append(feature_) feature_collection = FeatureCollection(features) with open('results_{}_{}.geojson'.format(keyword, threshold), 'w') as results: json.dump(feature_collection, results)
def parse_tile_str(tile_str): """ Parse a string like tiletype/zoom/x/y.suffix into a (tiletype, tile) tuple. """ m = TILE_RE.match(tile_str) return (m.group(1), Tile(int(m.group(3)), int(m.group(4)), int(m.group(2))))
def render_tif(z, x, y): tile = Tile(x, y, z) headers, data = tiling.render_tile(tile, CATALOG, format=GEOTIFF_FORMAT) headers.update(CATALOG.headers) return data, 200, headers
def resolve_seams(self): tile_indices = arange(TILE_SIZE**2) left_indices = tile_indices[::TILE_SIZE] bottom_indices = tile_indices[TILE_SIZE**2 - TILE_SIZE:] right_indices = tile_indices[TILE_SIZE - 1::TILE_SIZE] top_indices = tile_indices[:TILE_SIZE] for mesh in self.items: geometry = mesh.geometry t = geometry.tile left = Tile(t.x - 1, t.y, t.z) bottom = Tile(t.x, t.y + 1, t.x) right = Tile(t.x + 1, t.y, t.z) top = Tile(t.x, t.y - 1, t.z) neighborhood = [ x.geometry for x in self.items if x.geometry.tile in (left, bottom, right, top) ] if neighborhood: heights = geometry.heights.ravel() for neighbor in neighborhood: ntile = neighbor.tile if ntile == left: indices = left_indices neighbor_indices = right_indices elif ntile == bottom: indices = bottom_indices neighbor_indices = top_indices elif ntile == right: indices = right_indices neighbor_indices = left_indices elif ntile == top: indices = top_indices neighbor_indices = bottom_indices else: raise ValueError("Neighbor is not a neighbor!") neighbor_heights = neighbor.heights.ravel() heights[indices] = neighbor_heights[neighbor_indices] geometry.heights = heights.reshape((TILE_SIZE, TILE_SIZE))
def render_geotiff(z, x, y, **kwargs): tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, ELEVATION_CATALOG, format=GEOTIFF_FORMAT, scale=2, data_band_count=1) return data, 200, headers
def render_hillshade_tiff(z, x, y, **kwargs): tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, CATALOGS['hillshade'], format=HILLSHADE_GEOTIFF_FORMAT, transformation=HILLSHADE_TRANSFORMATION, scale=2, data_band_count=1) return data, 200, headers
def get_tiles(event: SQSEvent) -> List[Tile]: """ Return the body of our incoming SQS messages as an array of mercantile Tiles Expects events of the following format: { 'Records': [ { "body": '{ "x": 4, "y": 5, "z":3 }' }] } """ return [ Tile(*json.loads(record['body']).values()) for record in event['Records'] ]
def get_strided_tiles(root: Tile, zoom: int = 15): child_bounds = children(root, zoom=zoom) min_x = min((child.x for child in child_bounds)) min_y = min((child.y for child in child_bounds)) max_x = max((child.x for child in child_bounds)) max_y = max((child.y for child in child_bounds)) n = max_x - min_x + 1 strided_tiles = [] for x, y in product(np.arange(min_x, max_x + 0.1, 0.5), np.arange(min_y, max_y + 0.1, 0.5)): strided_tiles += [Tile(float(x), float(y), zoom)] return strided_tiles
def render_png(renderer, z, x, y, scale=1, **kwargs): tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, CATALOGS[renderer], format=FORMATS[renderer], transformation=TRANSFORMATIONS.get(renderer), scale=scale, data_band_count=DATA_BAND_COUNTS.get(renderer, 1)) return data, 200, headers
def convert_csv(fname_csv, fname_geojson, tile_format, thresh_ind, thresh): """Convert tile indices in CSV file to geojson""" if not op.exists(fname_csv): raise ValueError(f'Cannot find file {fname_csv}') # Error check tile format if tile_format == 'tms': tile_func = Pygeo_tile.from_tms elif tile_format == 'google': tile_func = Pygeo_tile.from_google else: raise ValueError(f'Tile format not understood. Got: {tile_format}') if not 0 <= thresh <= 1.: raise ValueError(f"'thresh' must be on interval [0, 1]. Got: {thresh}") with open(fname_csv, 'r') as csvfile: with open(fname_geojson, 'w') as results: reader = csv.reader(csvfile) first_line = True # Create a FeatureCollection results.write('{"type":"FeatureCollection","features":[') next(reader) # Skip header for row in reader: # Load as pygeotile using TMS coords geot = tile_func(*[int(t) for t in row[0].split('-')]) # Create feature with mercantile feat = feature(Tile(geot.google[0], geot.google[1], geot.zoom)) # Get class prediction confidences pred = json.loads(','.join(row[1:])) pred_red = list(map(lambda x: round(x, 2), pred)) if pred_red[thresh_ind] >= thresh: # Add commas prior to any feature that isn't the first one if first_line: first_line = False else: results.write(',') pred_obj = dict(zip(map(lambda x: 'p%s' % x, range(len(pred_red))), pred_red)) results.write(json.dumps(Feature(geometry=feat['geometry'], properties=pred_obj))) # Finalize the feature FeatureCollection results.write(']}')
def tile(id, z, x, y, **kwargs): meta = get_metadata(id, **kwargs) tile = read_tile(id, Tile(x, y, z), **kwargs) headers = { 'Content-Type': 'image/png', } if meta['meta'].get('oinMetadataUrl'): headers['X-OIN-Metadata-URL'] = meta['meta'].get('oinMetadataUrl') if meta['meta'].get('acquisitionStart') or meta['meta'].get( 'acquisitionEnd'): start = meta['meta'].get('acquisitionStart') end = meta['meta'].get('acquisitionEnd') if start and end: start = arrow.get(start) end = arrow.get(end) capture_range = '{}-{}'.format(start.format('M/D/YYYY'), end.format('M/D/YYYY')) headers['X-OIN-Acquisition-Start'] = start.format( 'YYYY-MM-DDTHH:mm:ssZZ') headers['X-OIN-Acquisition-End'] = end.format( 'YYYY-MM-DDTHH:mm:ssZZ') elif start: start = arrow.get(start) capture_range = start.format('M/D/YYYY') headers['X-OIN-Acquisition-Start'] = start.format( 'YYYY-MM-DDTHH:mm:ssZZ') elif end: end = arrow.get(end) capture_range = end.format('M/D/YYYY') headers['X-OIN-Acquisition-End'] = end.format( 'YYYY-MM-DDTHH:mm:ssZZ') # Bing Maps-compatibility (JOSM uses this) headers['X-VE-TILEMETA-CaptureDatesRange'] = capture_range if meta['meta'].get('provider'): headers['X-OIN-Provider'] = unicodedata.normalize( 'NFKD', meta['meta'].get('provider')).encode('ascii', 'ignore') if meta['meta'].get('platform'): headers['X-OIN-Platform'] = unicodedata.normalize( 'NFKD', meta['meta'].get('platform')).encode('ascii', 'ignore') return tile, 200, headers
def render_png(z, x, y, scale=1): tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, CATALOG, format=IMAGE_FORMAT, transformation=COLORMAP_TRANSFORMATION, scale=scale, ) headers.update(CATALOG.headers) return data, 200, headers
def test_get_tiles(): # create a class with fake environment variables dap = DownloadAndPredict( imagery='https://example.com/{z}/{x}/{y}.png', db='postgres://*****:*****@host:port/database', prediction_endpoint='https://myloadbalancer.com/v1/models/ml:predict' ) # create an example SQS event which invokes a lambda event = { 'Records': [ { "body": '{ "x": 4, "y": 5, "z":3 }' }] } tiles = dap.get_tiles(event) fixture_tiles = [Tile(x=4, y=5, z=3)] assert(tiles == fixture_tiles)
def user_render(id, z, x, y, scale=1, prefix=None): catalog = make_remote_catalog("user", id) tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, catalog, format=OPTIMAL_FORMAT, transformation=IMAGE_TRANSFORMATION, scale=scale, ) headers.update(catalog.headers) return data, 200, headers
def render_png(z, x, y, scale=1): catalog = make_catalog(request.args) tile = Tile(x, y, z) headers, data = tiling.render_tile( tile, catalog, format=IMAGE_FORMAT, transformation=IMAGE_TRANSFORMATION, scale=scale, ) headers.update(catalog.headers) return data, 200, headers
def render_json(z, x, y, scale=1): tile = Tile(x, y, z) sieve = int(request.args.get("sieve", 4)) headers, data = tiling.render_tile( tile, CATALOG, format=GeoJSON(sieve_size=sieve), transformation=Transformation(collar=8 * scale), scale=scale, ) headers.update(CATALOG.headers) return data, 200, headers
def generate_tiles(tile, max_zoom, metatile=1, materialize_zooms=None): tiles = [] metatile = min(metatile, 2**tile.z) for dx in range(0, metatile): for dy in range(0, metatile): tiles.append(Tile(tile.x + dx, tile.y + dy, tile.z)) for z in range(tile.z, max_zoom + 1): if materialize_zooms is None or z in materialize_zooms: a, tiles = itertools.tee(tiles, 2) yield from a tiles = itertools.chain.from_iterable( mercantile.children(t) for t in tiles)
def render_json(renderer, z, x, y, scale=1, max_zoom=None, **kwargs): tile = Tile(x, y, z) headers = {"Content-Type": "application/json"} min_zoom = None if max_zoom is not None: min_zoom = z data = list( footprints.sources_for_tile( tile, CATALOGS[renderer], scale=scale, min_zoom=min_zoom, max_zoom=max_zoom)) return jsonify(data), 200, headers
def render_png_from_remote(id, z, x, y, scale=1, prefix=None): # prefix is for URL generation only (API Gateway stages); if it matched the # URL, it's part of the id if prefix is not None: id = "/".join([prefix, id]) catalog = make_remote_catalog(id) tile = Tile(x, y, z) headers, data = tiling.render_tile(tile, catalog, format=PNG_FORMAT, transformation=IMAGE_TRANSFORMATION, scale=scale) headers.update(catalog.headers) return data, 200, headers
def get_image(self, *tile: Tile, source_zoom: int = 18, black_fail: bool = False): if len(tile) == 1: tile = tile[0] x, y, z = tile if not self.max_res: source_zoom = z parents = [ parent(Tile(int(fn(x)), int(fn(y)), z), zoom=i) for i, fn in product(range(z + 1), [np.ceil, np.floor]) ] for supertile in parents: if supertile in self.stacks: return self.__make_image(self.stacks[supertile], tile, source_zoom) raise FileNotFoundError( f"Could not find a parent tilestack for requested subtile {tile}!")
def render_tile( format, output, scale, transformation, tile, ): z, x, y = map(int, tile.split("/")) tile = Tile(x, y, z) (headers, data) = tiling.render_tile( tile, PostGISCatalog(), format=FORMATS[format], transformation=TRANSFORMATIONS.get(transformation), scale=scale) [click.echo("{}: {}".format(k, v), err=True) for k, v in headers.items()] output.write(data)
def render_geojson(renderer, z, x, y, scale=1, max_zoom=None, **kwargs): tile = Tile(x, y, z) data = { "type": "FeatureCollection", "features": [], } headers = {"Content-Type": "application/json"} min_zoom = None if max_zoom is not None: min_zoom = z [ data["features"].append(footprint) for footprint in footprints.features_for_tile( tile, CATALOGS[renderer], scale=scale, min_zoom=min_zoom, max_zoom=max_zoom) ] return jsonify(data), 200, headers
"""Module tests""" from mercantile import Tile import pytest import mbtiles.worker @pytest.mark.parametrize( "tile", [Tile(36, 73, 7), Tile(0, 0, 0), Tile(1, 1, 1)]) @pytest.mark.parametrize("filename", ["RGB.byte.tif", "RGBA.byte.tif"]) def test_process_tile(data, filename, tile): sourcepath = str(data.join(filename)) mbtiles.worker.init_worker( sourcepath, { "driver": "PNG", "dtype": "uint8", "nodata": 0, "height": 256, "width": 256, "count": 3, "crs": "EPSG:3857", }, "nearest", {}, {}, ) t, contents = mbtiles.worker.process_tile(tile) assert t.x == tile.x
def make_labels(dest_folder, zoom, country, classes, ml_type, bounding_box, sparse, **kwargs): """Create label data from OSM QA tiles for specified classes Perform the following operations: - If necessary, re-tile OSM QA Tiles to the specified zoom level - Iterate over all tiles within the bounding box and produce a label for each - Save the label file as labels.npz - Create an output for previewing the labels (GeoJSON or PNG depending upon ml_type) Parameters ------------ dest_folder: str Folder to save labels and example tiles into zoom: int The zoom level to create tiles at country: str The OSM QA Tile extract to download. The value should be a country string matching a value found in `label_maker/countries.txt` classes: list A list of classes for machine learning training. Each class is defined as a dict with two required properties: - name: class name - filter: A Mapbox GL Filter. See the README for more details ml_type: str Defines the type of machine learning. One of "classification", "object-detection", or "segmentation" bounding_box: list The bounding box to create images from. This should be given in the form: `[xmin, ymin, xmax, ymax]` as longitude and latitude values between `[-180, 180]` and `[-90, 90]` respectively sparse: boolean Limit the total background tiles to write based on `background_ratio` kwarg. geojson: str Filepath to optional geojson label input **kwargs: dict Other properties from CLI config passed as keywords to other utility functions """ mbtiles_file = op.join(dest_folder, '{}.mbtiles'.format(country)) mbtiles_file_zoomed = op.join(dest_folder, '{}-z{!s}.mbtiles'.format(country, zoom)) if not op.exists(mbtiles_file_zoomed): filtered_geo = kwargs.get('geojson') or op.join( dest_folder, '{}.geojson'.format(country)) fast_parse = [] if not op.exists(filtered_geo): fast_parse = ['-P'] print('Retiling QA Tiles to zoom level {} (takes a bit)'.format( zoom)) ps = Popen(['tippecanoe-decode', '-c', '-f', mbtiles_file], stdout=PIPE) stream_filter_fpath = op.join(op.dirname(label_maker.__file__), 'stream_filter.py') run([ sys.executable, stream_filter_fpath, json.dumps(bounding_box) ], stdin=ps.stdout, stdout=open(filtered_geo, 'w')) ps.wait() run(['tippecanoe', '--no-feature-limit', '--no-tile-size-limit'] + fast_parse + [ '-l', 'osm', '-f', '-z', str(zoom), '-Z', str(zoom), '-o', mbtiles_file_zoomed, filtered_geo ]) # Call tilereduce print('Determining labels for each tile') mbtiles_to_reduce = mbtiles_file_zoomed tilereduce( dict(zoom=zoom, source=mbtiles_to_reduce, bbox=bounding_box, args=dict(ml_type=ml_type, classes=classes)), _mapper, _callback, _done) # Add empty labels to any tiles which didn't have data empty_label = _create_empty_label(ml_type, classes) for tile in tiles(*bounding_box, [zoom]): index = '-'.join([str(i) for i in tile]) global tile_results if tile_results.get(index) is None: tile_results[index] = empty_label # Print a summary of the labels _tile_results_summary(ml_type, classes) # If the --sparse flag is provided, limit the total background tiles to write if sparse: pos_examples, neg_examples = [], [] for k in tile_results.keys(): # if we don't match any class, this is a negative example if not sum([ class_match(ml_type, tile_results[k], i + 1) for i, c in enumerate(classes) ]): neg_examples.append(k) else: pos_examples.append(k) # Choose random subset of negative examples n_neg_ex = int(kwargs['background_ratio'] * len(pos_examples)) neg_examples = np.random.choice(neg_examples, n_neg_ex, replace=False).tolist() tile_results = { k: tile_results.get(k) for k in pos_examples + neg_examples } print('Using sparse mode; subselected {} background tiles'.format( n_neg_ex)) # write out labels as numpy arrays labels_file = op.join(dest_folder, 'labels.npz') print('Writing out labels to {}'.format(labels_file)) np.savez(labels_file, **tile_results) # write out labels as GeoJSON or PNG if ml_type == 'classification': features = [] for tile, label in tile_results.items(): feat = feature(Tile(*[int(t) for t in tile.split('-')])) features.append( Feature(geometry=feat['geometry'], properties=dict(label=label.tolist()))) json.dump(fc(features), open(op.join(dest_folder, 'classification.geojson'), 'w')) elif ml_type == 'object-detection': label_folder = op.join(dest_folder, 'labels') if not op.isdir(label_folder): makedirs(label_folder) for tile, label in tile_results.items(): # if we have at least one bounding box label if bool(label.shape[0]): label_file = '{}.png'.format(tile) img = Image.new('RGB', (256, 256)) draw = ImageDraw.Draw(img) for box in label: draw.rectangle(((box[0], box[1]), (box[2], box[3])), outline=class_color(box[4])) print('Writing {}'.format(label_file)) img.save(op.join(label_folder, label_file)) elif ml_type == 'segmentation': label_folder = op.join(dest_folder, 'labels') if not op.isdir(label_folder): makedirs(label_folder) for tile, label in tile_results.items(): # if we have any class pixels if np.sum(label): label_file = '{}.png'.format(tile) visible_label = np.array([ class_color(l) for l in np.nditer(label) ]).reshape(256, 256, 3) img = Image.fromarray(visible_label.astype(np.uint8)) print('Writing {}'.format(label_file)) img.save(op.join(label_folder, label_file))
default=["file://./"], nargs="+", help="Target path/URI(s) for archives") args = parser.parse_args() if args.verbose: logger.setLevel(logging.DEBUG) if len(args.targets) > 1 and not args.skip_meta: logger.error( "Outputting meta.json with more than one target is not supported yet." ) sys.exit(1) root = Tile(args.x, args.y, args.zoom) concurrency = args.concurrency min_zoom = args.zoom max_zoom = args.max_zoom metatile = args.metatile materialize_zooms = args.materialize or [] scale = args.scale materialize_zooms = list(filter(lambda z: z <= max_zoom, materialize_zooms)) if args.cache_sources: logger.info( "Caching sources for root tile %s from zoom %d to %d", root, min_zoom,
def get_scaled_tile(id, z, x, y, scale): tile = read_tile(id, Tile(x, y, z), scale=scale) return tile, 200, { 'Content-Type': 'image/png' }