def tiles_to_geojson(tiles, union=True):
"""Convert tiles to their footprint GeoJSON."""
first = True
geojson = '{"type":"FeatureCollection","features":['
if union: # smaller tiles union geometries (but losing properties)
tiles = [
str(tile.z) + "-" + str(tile.x) + "-" + str(tile.y) + "\n"
for tile in tiles
]
for feature in supermercado.uniontiles.union(tiles, True):
geojson += json.dumps(
feature) if first else "," + json.dumps(feature)
first = False
else: # keep each tile geometry and properties (but fat)
for tile in tiles:
prop = '"properties":{{"x":{},"y":{},"z":{}}}'.format(
tile.x, tile.y, tile.z)
geom = '"geometry":{}'.format(
json.dumps(mercantile.feature(tile, precision=6)["geometry"]))
geojson += '{}{{"type":"Feature",{},{}}}'.format(
"," if not first else "", geom, prop)
first = False
geojson += "]}"
return geojson
def web_ui(out, base_url, coverage_tiles, selected_tiles, ext, template):
try:
if os.path.isfile(template):
web_ui = open(template, "r").read()
else:
web_ui = open(os.path.join(Path(__file__).parent, "tools", "templates", template), "r").read()
except:
sys.exit("Unable to open Web UI template {}".format(template))
web_ui = re.sub("{{base_url}}", base_url, web_ui)
web_ui = re.sub("{{ext}}", ext, web_ui)
web_ui = re.sub("{{tiles}}", "tiles.json" if selected_tiles else "''", web_ui)
if coverage_tiles:
# Could surely be improve, but for now, took the first tile to center on
tile = list(coverage_tiles)[0]
x, y, z = map(int, [tile.x, tile.y, tile.z])
web_ui = re.sub("{{zoom}}", str(z), web_ui)
web_ui = re.sub("{{center}}", str(list(tile_pixel_to_location(tile, 0.5, 0.5))[::-1]), web_ui)
with open(os.path.join(out, "index.html"), "w", encoding="utf-8") as fp:
fp.write(web_ui)
if selected_tiles:
with open(os.path.join(out, "tiles.json"), "w", encoding="utf-8") as fp:
fp.write('{"type":"FeatureCollection","features":[')
first = True
for tile in selected_tiles:
prop = '"properties":{{"x":{},"y":{},"z":{}}}'.format(int(tile.x), int(tile.y), int(tile.z))
geom = '"geometry":{}'.format(json.dumps(feature(tile, precision=6)["geometry"]))
fp.write('{}{{"type":"Feature",{},{}}}'.format("," if not first else "", geom, prop))
first = False
fp.write("]}")
def _get_geojson(mosaicid: str = None,
url: str = None) -> Tuple[str, str, str]:
"""
Handle /geojson requests.
Attributes
----------
url : str, required
Mosaic definition url.
Returns
-------
status : str
Status of the request (e.g. OK, NOK).
MIME type : str
response body MIME type (e.g. application/json).
body : str
String encoded JSON metata
"""
bucket = os.environ["MOSAIC_DEF_BUCKET"]
url = f"s3://{bucket}/mosaics/{mosaicid}.json.gz"
mosaic_definition = get_mosaic_content(url)
geojson = {
"type":
"FeatureCollection",
"features": [
mercantile.feature(mercantile.quadkey_to_tile(qk),
props=dict(quadkey=qk, files=files))
for qk, files in mosaic_definition["tiles"].items()
],
}
return ("OK", "application/json", json.dumps(geojson))
def get_tile_list(geom,
zoom=17):
"""Generate the Tile List for The Tasking List
Parameters
----------
geom: shapely geometry of area.
zoom : int Zoom Level for Tiles
One or more zoom levels.
Yields
------
list of tiles that intersect with
"""
west, south, east, north = geom.bounds
tiles = mercantile.tiles(west, south, east, north, zooms=zoom)
tile_list = []
for tile in tiles:
tile_geom = geometry.shape(mercantile.feature(tile)['geometry'])
if tile_geom.intersects(geom):
tile_list.append(tile)
return tile_list
def default_filter(
tile: mercantile.Tile,
dataset: Sequence[Dict],
geoms: Sequence[polygons],
minimum_tile_cover=None,
tile_cover_sort=False,
maximum_items_per_tile: Optional[int] = None,
) -> List:
"""Filter and/or sort dataset per intersection coverage."""
indices = list(range(len(dataset)))
if minimum_tile_cover or tile_cover_sort:
tile_geom = polygons(
mercantile.feature(tile)["geometry"]["coordinates"][0])
int_pcts = _intersect_percent(tile_geom, geoms)
if minimum_tile_cover:
indices = [
ind for ind in indices if int_pcts[ind] > minimum_tile_cover
]
if tile_cover_sort:
# https://stackoverflow.com/a/9764364
indices, _ = zip(*sorted(zip(indices, int_pcts), reverse=True))
if maximum_items_per_tile:
indices = indices[:maximum_items_per_tile]
return [dataset[ind] for ind in indices]
def map_interacted(event):
if event["type"] == "change" and event["name"] == "bounds":
self.ht.value = f"{self.tile_id}, Map zoom: {int(a_map.zoom)}"
self.slider.max = int(a_map.zoom) + self._max_zoom_delta
m = event["owner"]
((south, west), (north, east)) = m.bounds
b_poly = list(m.bounds_polygon)
b_poly += [tuple(b_poly[0])]
# m += Polyline(locations=b_poly)
# Attention in the order of west, south, east, north!
tiles = mercantile.tiles(west,
south,
east,
north,
zooms=self.level)
features = [mercantile.feature(t) for t in tiles]
self.gj.data = geojson.FeatureCollection(features=features)
# Ipyleaflet buglet(?): This name is updated in the GeoJSON layer,
# but not in the LayersControl!
self.gj.name = f"Mercator" # level {self.level}"
self.gj.on_hover(hover)
def pred_2geojson(csv_file, keyword, threshold):
features = list()
with open(csv_file, 'r') as csvfile:
reader = csv.reader(csvfile)
next(reader)
for row in tqdm(reader):
pred = json.loads(row[1])
pred_red = list(map(lambda x: round(x, 2), pred))
pred_obj = dict(
zip(map(lambda x: 'p%s' % x, range(len(pred_red))), pred_red))
if keyword == "point":
pred_j = ul(*[int(t) for t in row[0].split('-')])
feature_collection = Feature(geometry=dict(
type='Point', coordinates=[pred_j.lng, pred_j.lat]),
properties=pred_obj)
else:
pred_j = feature(Tile(*[int(t) for t in row[0].split('-')]))
feature_ = Feature(geometry=pred_j['geometry'],
properties=pred_obj)
if pred_obj['p1'] >= float(threshold):
features.append(feature_)
feature_collection = FeatureCollection(features)
with open('results_{}_{}.geojson'.format(keyword, threshold),
'w') as results:
json.dump(feature_collection, results)
def test_tile_geom(self):
# load src geojson
with open('gladAnalysis/tests/fixtures/BRA_src.geojson') as src:
geojson = json.load(src)
# load tiled output
with open('gladAnalysis/tests/fixtures/BRA_tiled.geojson') as src:
tiled_geojson = json.load(src)
# grab actual geom
geom = shape(geojson['features'][0]['geometry'])
# start empty feature collection
tiled_features = {'type': 'FeatureCollection', 'features': []}
# cut our geom into tiles
tile_dict = build_tile_dict(geom)
# grab the shape of these tiles so we can visualize it
for t, pct_intersect in tile_dict.iteritems():
tile_geom = mercantile.feature(t)
tile_geom['properties']['tile_coverage_fraction'] = pct_intersect
tiled_features['features'].append(tile_geom)
# compare to our saved version of the tiled geometry
# this includes tile_fraction calcs as well
self.assertEqual(json.dumps(sorted(tiled_geojson)),
json.dumps(sorted(tiled_features)))
def find_child_land_tiles(tile: mercantile.Tile, gdf: gpd.GeoDataFrame,
maxzoom: int) -> List[mercantile.Tile]:
"""Recursively find tiles at desired zoom that intersect land areas
Args:
- tile: tile to recursively search within
- gdf: GeoDataFrame with geometries of land masses
- maxzoom: zoom at which to stop recursing
Returns:
List of tiles intersecting land
"""
land_tiles = []
for child in mercantile.children(tile):
tile_geom = shape(mercantile.feature(tile)['geometry'])
intersects_land = gdf.intersects(tile_geom).any()
if not intersects_land:
continue
if child.z == maxzoom:
land_tiles.append(child)
continue
land_tiles.extend(find_child_land_tiles(child, gdf, maxzoom))
return land_tiles
def get_intersect_area(aoi, t):
# grab the geometry of our tile of interest, then intersect it with our AOI
tile_geom = shape(mercantile.feature(t)['geometry'])
intersect_geom = aoi.intersection(tile_geom)
# calc area in web mercator; important given that all tiles of
# a certain z level will have the same area in web mercator
return calc_area(intersect_geom, init="EPSG:3857")
def create_tiles_gdf(bounds: List[float],
quadkey_zoom: int) -> gpd.GeoDataFrame:
"""Create GeoDataFrame of all tiles within bounds at quadkey_zoom
"""
features = [
mercantile.feature(tile, props={'quadkey': mercantile.quadkey(tile)})
for tile in mercantile.tiles(*bounds, quadkey_zoom)
]
return gpd.GeoDataFrame.from_features(features, crs='EPSG:4326')
def tiles_geometry(tiles: Set[Tile], individual: bool = False):
geometry = shapely.geometry.shape({
'type':
"GeometryCollection",
'geometries': [mercantile.feature(tile)['geometry'] for tile in tiles]
})
if not individual:
geometry = polygon_split_holes(shapely.ops.unary_union(geometry))
return geometry
def geojson(self) -> dict:
"""Get Raster metadata."""
return {
"type": "FeatureCollection",
"features": [
mercantile.feature(
mercantile.quadkey_to_tile(qk), props=dict(files=files)
)
for qk, files in self.mosaic["tiles"].items()
],
}
def clip_polygon(tile, polygon):
"""
Clip a polygon to the given tile
"""
tilePolygon = shape(mercantile.feature(tile)['geometry'])
polygonShape = shape(polygon)
if (polygonShape.crosses(tilePolygon)):
intersection = tilePolygon.intersection(polygonShape)
return json.loads(json.dumps(mapping(intersection)))
else:
return json.loads(json.dumps(polygon))
def convert_csv(fname_csv, fname_geojson, tile_format, thresh_ind, thresh):
"""Convert tile indices in CSV file to geojson"""
if not op.exists(fname_csv):
raise ValueError(f'Cannot find file {fname_csv}')
# Error check tile format
if tile_format == 'tms':
tile_func = Pygeo_tile.from_tms
elif tile_format == 'google':
tile_func = Pygeo_tile.from_google
else:
raise ValueError(f'Tile format not understood. Got: {tile_format}')
if not 0 <= thresh <= 1.:
raise ValueError(f"'thresh' must be on interval [0, 1]. Got: {thresh}")
with open(fname_csv, 'r') as csvfile:
with open(fname_geojson, 'w') as results:
reader = csv.reader(csvfile)
first_line = True
# Create a FeatureCollection
results.write('{"type":"FeatureCollection","features":[')
next(reader) # Skip header
for row in reader:
# Load as pygeotile using TMS coords
geot = tile_func(*[int(t) for t in row[0].split('-')])
# Create feature with mercantile
feat = feature(Tile(geot.google[0], geot.google[1], geot.zoom))
# Get class prediction confidences
pred = json.loads(','.join(row[1:]))
pred_red = list(map(lambda x: round(x, 2), pred))
if pred_red[thresh_ind] >= thresh:
# Add commas prior to any feature that isn't the first one
if first_line:
first_line = False
else:
results.write(',')
pred_obj = dict(zip(map(lambda x: 'p%s' % x,
range(len(pred_red))), pred_red))
results.write(json.dumps(Feature(geometry=feat['geometry'],
properties=pred_obj)))
# Finalize the feature FeatureCollection
results.write(']}')
def tiles_to_geojson(tiles):
"""Convert tiles to their footprint GeoJSON."""
geojson = '{"type":"FeatureCollection","features":['
first = True
for tile in tiles:
prop = '"properties":{{"x":{},"y":{},"z":{}}}'.format(
tile.x, tile.y, tile.z)
geom = '"geometry":{}'.format(
json.dumps(mercantile.feature(tile, precision=6)["geometry"]))
geojson += '{}{{"type":"Feature",{},{}}}'.format(
"," if not first else "", geom, prop)
first = False
geojson += "]}"
return geojson
def _geojson(mosaicid: str = None, url: str = None) -> Tuple:
"""Handle /geojson requests."""
if not mosaicid and not url:
return ("NOK", "text/plain", "Missing 'MosaicID or URL' parameter")
mosaic_path = _create_mosaic_path(mosaicid) if mosaicid else url
with MosaicBackend(mosaic_path) as mosaic:
geojson = {
"type": "FeatureCollection",
"features": [
mercantile.feature(
mercantile.quadkey_to_tile(qk), props=dict(files=files)
)
for qk, files in mosaic.mosaic_def.tiles.items()
],
}
return ("OK", "application/json", json.dumps(geojson, separators=(",", ":")))
async def _update_debug_layer(self,
interval: float = DEBUG_INTERVAL) -> None:
"Update a GeoJSON layer showing all the tiles we know about and their status"
while True:
self.debug_layer.data = dict(
type="FeatureCollection",
features=[
mercantile.feature(
xyz,
props=dict(
speculative=ref.speculative,
done=ref.task.done(),
cancelled=ref.task.cancelled(),
),
) for xyz, ref in self.tiles.items()
],
)
await asyncio.sleep(interval)
def _geojson(mosaicid: str = None, url: str = None) -> Tuple[str, str, str]:
"""Handle /geojson requests."""
if mosaicid:
url = _create_path(mosaicid)
elif url is None:
return ("NOK", "text/plain", "Missing 'URL' parameter")
mosaic_def = fetch_mosaic_definition(url)
geojson = {
"type":
"FeatureCollection",
"features": [
mercantile.feature(mercantile.quadkey_to_tile(qk),
props=dict(files=files))
for qk, files in mosaic_def["tiles"].items()
],
}
return ("OK", "application/json", json.dumps(geojson))
def process_tile(tile_list, aoi):
# main function to compare a list of tiles to an input geometry
# will eventually return a list of tiles completely within the aoi (all zoom levels possible)
# and tiles that intersect the aoi (must be of max_z because that's as low as we go)
within_list = []
intersect_list = []
max_z = 12
for t in tile_list:
# a tile either is completely within, completely outside, or intersects
tile_geom = shape(mercantile.feature(t)['geometry'])
# if it's within, great- our work is done
if aoi.contains(tile_geom):
within_list.append(t)
elif tile_geom.intersects(aoi):
# if it intersects and is < max_z, subdivide it and start the process again
if t.z < max_z:
# find the four children of this tile and check them for within/intersect/outside-ness
tile_children = mercantile.children(t)
new_within_list, new_intersect_list = process_tile(
tile_children, aoi)
# add the results to our initial lists
within_list.extend(new_within_list)
intersect_list.extend(new_intersect_list)
# if it intersects and is at max_z, add it to our intersect list
else:
intersect_list.append(t)
# and if it's outside our geometry, drop it
else:
pass
return within_list, intersect_list
def map_interacted(event):
if event["type"] == "change" and event["name"] == "bounds":
self.ht.value = f"{self.tile_xy}, Map zoom: {int(a_map.zoom)}"
self.slider.max = int(a_map.zoom) + self._max_zoom_delta
m = event["owner"]
((south, west), (north, east)) = m.bounds
# Attention in the order of west, south, east, north!
tiles = mercantile.tiles(west,
south,
east,
north,
zooms=self.level)
features = [mercantile.feature(t) for t in tiles]
self.gj.data = geojson.FeatureCollection(features=features)
# Ipyleaflet buglet: This name is updated in the GeoJSON layer,
# but not in the LayersControl!
self.gj.name = str(int(self.level))
self.gj.on_hover(hover)
def get_tile_as_virtual_raster(tile):
'''NOTE: Here "tile" refers to a mercantile "Tile" object.'''
img = get_image_by_xyz_from_url(tile)
geom = shapely.geometry.shape(mercantile.feature(tile)["geometry"])
minx, miny, maxx, maxy = geom.bounds
dst_transform = rasterio.transform.from_bounds(minx, miny, maxx, maxy, 256, 256)
dst_profile = {
"driver": "GTiff",
"width": 256,
"height": 256,
"transform": dst_transform,
"crs": "epsg:4326",
"count": 3,
"dtype": "uint8"
}
test_f = rasterio.io.MemoryFile()
with test_f.open(**dst_profile) as test_d:
test_d.write(img[:,:,0], 1)
test_d.write(img[:,:,1], 2)
test_d.write(img[:,:,2], 3)
test_f.seek(0)
return test_f
def missing_quadkeys(mosaic: Dict,
shp_path: str,
bounds: List[float] = None,
simplify: bool = True) -> Dict:
"""Find quadkeys over land missing from mosaic
Args:
- mosaic: mosaic definition
- shp_path: path to Natural Earth shapefile of land boundaries
- bounds: force given bounds
- simplify: reduce size of the tileset as much as possible by merging leaves into parents
Returns:
- GeoJSON FeatureCollection of missing tiles
"""
bounds = bounds or mosaic['bounds']
top_tile = mercantile.bounding_tile(*bounds)
gdf = gpd.read_file(shp_path)
quadkey_zoom = mosaic.get('quadkey_zoom', mosaic['minzoom'])
# Remove null island
# Keep the landmasses that are visible at given zoom
gdf = gdf[gdf['max_zoom'] <= quadkey_zoom]
land_tiles = find_child_land_tiles(top_tile, gdf, quadkey_zoom)
quadkeys = {mercantile.quadkey(tile) for tile in land_tiles}
mosaic_quadkeys = set(mosaic['tiles'].keys())
not_in_mosaic = quadkeys.difference(mosaic_quadkeys)
not_in_mosaic = [mercantile.quadkey_to_tile(qk) for qk in not_in_mosaic]
if simplify:
not_in_mosaic = mercantile.simplify(not_in_mosaic)
features = [mercantile.feature(tile) for tile in not_in_mosaic]
return {'type': 'FeatureCollection', 'features': features}
def tiles_over_shape(geodata,
zoom,
filter_by_shape=True,
fill_area_filter_factor=0.0):
'''Generate slippy tile polygons in GeoDataFrame. Only EPSG:4326!
Args:
geodata (geopandas.GeoSeries|geopandas.GeoDataFrame): Экстенты, по котором необходимо создать тайлы
zoom (int): Z-координата, от которой зависит дробление тайлов
filter_by_shape (bool, optional): если True, тайлы bbox'а, которые не пересекаются c экстентом, будут отсечены в противном случае тайлы будут распределены по всему bbox
fill_area_filter_factor (float, optional): коэффициент площади, меньше которой тайл отсекается при filter_by_shape. Например, при коэффициенте равном 0.25, тайл, который покрывает меньше 25% экстента, будет отсечен
Returns:
geopandas.GeoDataFrame: тайлы и их x,y,z
'''
geoseries = _geo_input_handler(geodata, 4326)
x1, y1 = geoseries.bounds.min()[['minx', 'miny']]
x2, y2 = geoseries.bounds.max()[['maxx', 'maxy']]
features = []
for tile in mercantile.tiles(x1, y1, x2, y2, zoom):
feature = mercantile.feature(tile)
feature['properties'] = {
'x': tile.x,
'y': tile.y,
'z': tile.z,
}
features.append(feature)
tiles = gpd.GeoDataFrame.from_features(
{
'type': 'FeatureCollection',
'features': features
}, crs=4326)
if filter_by_shape:
tiles = _tiles_filtering_by_series(tiles, geoseries,
fill_area_filter_factor)
return tiles
def run(self):
if self.first_start:
self.first_start = False
self.dlg = GoToXYZDialog()
self.dlg.show()
result = self.dlg.exec_()
if result:
tile_z = int(self.dlg.z_lineEdit.text())
tile_x = int(self.dlg.x_lineEdit.text())
tile_y = int(self.dlg.y_lineEdit.text())
name = '{}_{}_{}_extend'.format(tile_z, tile_x, tile_y)
if self.dlg.XYZ_radiobutton.isChecked():
tile_y = (2**tile_z) - tile_y - 1
tile = mercantile.Tile(tile_x, tile_y, tile_z)
feature = mercantile.feature(tile)
json_file = tempfile.mktemp(suffix='.geojson')
json.dump(feature, open(json_file, mode='w+'))
self.iface.addVectorLayer(json_file, name, "ogr")
def make_labels(dest_folder, zoom, country, classes, ml_type, bounding_box,
sparse, **kwargs):
"""Create label data from OSM QA tiles for specified classes
Perform the following operations:
- If necessary, re-tile OSM QA Tiles to the specified zoom level
- Iterate over all tiles within the bounding box and produce a label for each
- Save the label file as labels.npz
- Create an output for previewing the labels (GeoJSON or PNG depending upon ml_type)
Parameters
------------
dest_folder: str
Folder to save labels and example tiles into
zoom: int
The zoom level to create tiles at
country: str
The OSM QA Tile extract to download. The value should be a country string matching a value found in
`label_maker/countries.txt`
classes: list
A list of classes for machine learning training. Each class is defined as a dict
with two required properties:
- name: class name
- filter: A Mapbox GL Filter.
See the README for more details
ml_type: str
Defines the type of machine learning. One of "classification", "object-detection", or "segmentation"
bounding_box: list
The bounding box to create images from. This should be given in the form: `[xmin, ymin, xmax, ymax]`
as longitude and latitude values between `[-180, 180]` and `[-90, 90]` respectively
sparse: boolean
Limit the total background tiles to write based on `background_ratio` kwarg.
geojson: str
Filepath to optional geojson label input
**kwargs: dict
Other properties from CLI config passed as keywords to other utility functions
"""
mbtiles_file = op.join(dest_folder, '{}.mbtiles'.format(country))
mbtiles_file_zoomed = op.join(dest_folder,
'{}-z{!s}.mbtiles'.format(country, zoom))
if not op.exists(mbtiles_file_zoomed):
filtered_geo = kwargs.get('geojson') or op.join(
dest_folder, '{}.geojson'.format(country))
fast_parse = []
if not op.exists(filtered_geo):
fast_parse = ['-P']
print('Retiling QA Tiles to zoom level {} (takes a bit)'.format(
zoom))
ps = Popen(['tippecanoe-decode', '-c', '-f', mbtiles_file],
stdout=PIPE)
stream_filter_fpath = op.join(op.dirname(label_maker.__file__),
'stream_filter.py')
run([
sys.executable, stream_filter_fpath,
json.dumps(bounding_box)
],
stdin=ps.stdout,
stdout=open(filtered_geo, 'w'))
ps.wait()
run(['tippecanoe', '--no-feature-limit', '--no-tile-size-limit'] +
fast_parse + [
'-l', 'osm', '-f', '-z',
str(zoom), '-Z',
str(zoom), '-o', mbtiles_file_zoomed, filtered_geo
])
# Call tilereduce
print('Determining labels for each tile')
mbtiles_to_reduce = mbtiles_file_zoomed
tilereduce(
dict(zoom=zoom,
source=mbtiles_to_reduce,
bbox=bounding_box,
args=dict(ml_type=ml_type, classes=classes)), _mapper, _callback,
_done)
# Add empty labels to any tiles which didn't have data
empty_label = _create_empty_label(ml_type, classes)
for tile in tiles(*bounding_box, [zoom]):
index = '-'.join([str(i) for i in tile])
global tile_results
if tile_results.get(index) is None:
tile_results[index] = empty_label
# Print a summary of the labels
_tile_results_summary(ml_type, classes)
# If the --sparse flag is provided, limit the total background tiles to write
if sparse:
pos_examples, neg_examples = [], []
for k in tile_results.keys():
# if we don't match any class, this is a negative example
if not sum([
class_match(ml_type, tile_results[k], i + 1)
for i, c in enumerate(classes)
]):
neg_examples.append(k)
else:
pos_examples.append(k)
# Choose random subset of negative examples
n_neg_ex = int(kwargs['background_ratio'] * len(pos_examples))
neg_examples = np.random.choice(neg_examples, n_neg_ex,
replace=False).tolist()
tile_results = {
k: tile_results.get(k)
for k in pos_examples + neg_examples
}
print('Using sparse mode; subselected {} background tiles'.format(
n_neg_ex))
# write out labels as numpy arrays
labels_file = op.join(dest_folder, 'labels.npz')
print('Writing out labels to {}'.format(labels_file))
np.savez(labels_file, **tile_results)
# write out labels as GeoJSON or PNG
if ml_type == 'classification':
features = []
for tile, label in tile_results.items():
feat = feature(Tile(*[int(t) for t in tile.split('-')]))
features.append(
Feature(geometry=feat['geometry'],
properties=dict(label=label.tolist())))
json.dump(fc(features),
open(op.join(dest_folder, 'classification.geojson'), 'w'))
elif ml_type == 'object-detection':
label_folder = op.join(dest_folder, 'labels')
if not op.isdir(label_folder):
makedirs(label_folder)
for tile, label in tile_results.items():
# if we have at least one bounding box label
if bool(label.shape[0]):
label_file = '{}.png'.format(tile)
img = Image.new('RGB', (256, 256))
draw = ImageDraw.Draw(img)
for box in label:
draw.rectangle(((box[0], box[1]), (box[2], box[3])),
outline=class_color(box[4]))
print('Writing {}'.format(label_file))
img.save(op.join(label_folder, label_file))
elif ml_type == 'segmentation':
label_folder = op.join(dest_folder, 'labels')
if not op.isdir(label_folder):
makedirs(label_folder)
for tile, label in tile_results.items():
# if we have any class pixels
if np.sum(label):
label_file = '{}.png'.format(tile)
visible_label = np.array([
class_color(l) for l in np.nditer(label)
]).reshape(256, 256, 3)
img = Image.fromarray(visible_label.astype(np.uint8))
print('Writing {}'.format(label_file))
img.save(op.join(label_folder, label_file))
def main(args):
if not args.masks or not args.labels:
assert args.mode != "list", "Parameters masks and labels are mandatories in list mode."
assert not (
args.min or args.max
), "Both --masks and --labels mandatory, for metric filtering."
if args.min or args.max:
config = load_config(args.config)
args.out = os.path.expanduser(args.out)
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
args_minmax = set()
args.min = {(m[0], m[1]): m[2] for m in args.min} if args.min else dict()
args.max = {(m[0], m[1]): m[2] for m in args.max} if args.max else dict()
args_minmax.update(args.min.keys())
args_minmax.update(args.max.keys())
minmax = dict()
for mm in args_minmax:
mm_min = float(args.min[mm]) if mm in args.min else 0.0
mm_max = float(args.max[mm]) if mm in args.max else 1.0
assert mm_min < mm_max, "--min must be lower than --max, on {}".format(
mm)
minmax[mm] = {
"min":
mm_min,
"max":
mm_max,
"class_id": [
c for c, classe in enumerate(config["classes"])
if classe["title"] == mm[0]
][0],
"module":
load_module("abd_model.metrics." + mm[1]),
}
if not args.workers:
args.workers = os.cpu_count()
print("abd compare {} on CPU, with {} workers".format(
args.mode, args.workers),
file=sys.stderr,
flush=True)
if args.images:
tiles = [tile for tile in tiles_from_dir(args.images[0], cover=cover)]
assert len(tiles), "Empty images dir: {}".format(args.images[0])
for image in args.images[1:]:
assert sorted(tiles) == sorted([
tile for tile in tiles_from_dir(image, cover=cover)
]), "Unconsistent images dirs"
if args.labels and args.masks:
tiles_masks = [
tile for tile in tiles_from_dir(args.masks, cover=cover)
]
tiles_labels = [
tile for tile in tiles_from_dir(args.labels, cover=cover)
]
if args.images:
assert sorted(tiles) == sorted(tiles_masks) == sorted(
tiles_labels), "Unconsistent images/label/mask directories"
else:
assert len(tiles_masks), "Empty masks dir: {}".format(args.masks)
assert len(tiles_labels), "Empty labels dir: {}".format(
args.labels)
assert sorted(tiles_masks) == sorted(
tiles_labels), "Label and Mask directories are not consistent"
tiles = tiles_masks
tiles_list = []
tiles_compare = []
progress = tqdm(total=len(tiles), ascii=True, unit="tile")
log = False if args.mode == "list" else Logs(os.path.join(args.out, "log"))
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile):
x, y, z = list(map(str, tile))
if args.masks and args.labels:
label = np.array(
Image.open(
os.path.join(args.labels, z, x, "{}.png".format(y))))
mask = np.array(
Image.open(
os.path.join(args.masks, z, x, "{}.png".format(y))))
assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)"
metrics = dict()
for mm in minmax:
try:
metrics[mm] = getattr(minmax[mm]["module"], "get")(
torch.as_tensor(label, device="cpu"),
torch.as_tensor(mask, device="cpu"),
minmax[mm]["class_id"],
)
except:
progress.update()
return False, tile
if not (minmax[mm]["min"] <= metrics[mm] <=
minmax[mm]["max"]):
progress.update()
return True, tile
tiles_compare.append(tile)
if args.mode == "side":
for i, root in enumerate(args.images):
img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1],
force_rgb=True)
if i == 0:
side = np.zeros(
(img.shape[0], img.shape[1] * len(args.images), 3))
side = np.swapaxes(side, 0,
1) if args.vertical else side
image_shape = img.shape
else:
assert image_shape[0:2] == img.shape[
0:2], "Unconsistent image size to compare"
if args.vertical:
side[i * image_shape[0]:(i + 1) *
image_shape[0], :, :] = img
else:
side[:, i * image_shape[0]:(i + 1) *
image_shape[0], :] = img
tile_image_to_file(args.out, tile, np.uint8(side))
elif args.mode == "stack":
for i, root in enumerate(args.images):
tile_image = tile_image_from_file(tile_from_xyz(
root, x, y, z)[1],
force_rgb=True)
if i == 0:
image_shape = tile_image.shape[0:2]
stack = tile_image / len(args.images)
else:
assert image_shape == tile_image.shape[
0:2], "Unconsistent image size to compare"
stack = stack + (tile_image / len(args.images))
tile_image_to_file(args.out, tile, np.uint8(stack))
elif args.mode == "list":
tiles_list.append([tile, metrics])
progress.update()
return True, tile
for ok, tile in executor.map(worker, tiles):
if not ok and log:
log.log("Warning: skipping. {}".format(str(tile)))
if args.mode == "list":
with open(args.out, mode="w") as out:
if args.geojson:
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile_list in tiles_list:
tile, metrics = tile_list
x, y, z = list(map(str, tile))
if args.geojson:
prop = '"properties":{{"x":{},"y":{},"z":{}'.format(
x, y, z)
for metric in metrics:
prop += ',"{}":{:.3f}'.format(metric, metrics[metric])
geom = '"geometry":{}'.format(
json.dumps(feature(tile, precision=6)["geometry"]))
out.write('{}{{"type":"Feature",{},{}}}}}'.format(
"," if not first else "", geom, prop))
first = False
if not args.geojson:
out.write("{},{},{}".format(x, y, z))
for metric in metrics:
out.write("\t{:.3f}".format(metrics[metric]))
out.write(os.linesep)
if args.geojson:
out.write("]}")
out.close()
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
if args.mode == "side" and not args.no_web_ui:
template = "compare.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out,
base_url,
tiles,
tiles_compare,
args.format,
template,
union_tiles=False)
if args.mode == "stack" and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles_from_dir(args.images[0])]
web_ui(args.out, base_url, tiles, tiles_compare, args.format, template)
def shapes(
ctx,
input,
precision,
indent,
compact,
projected,
seq,
output_mode,
collect,
extents,
buffer,
):
"""Print tiles as GeoJSON feature collections or sequences.
Input may be a compact newline-delimited sequences of JSON or
a pretty-printed ASCII RS-delimited sequence of JSON (like
https://tools.ietf.org/html/rfc8142 and
https://tools.ietf.org/html/rfc7159).
Tile descriptions may be either an [x, y, z] array or a JSON
object of the form
{"tile": [x, y, z], "properties": {"name": "foo", ...}}
In the latter case, the properties object will be used to update
the properties object of the output feature.
Example:
\b
echo "[486, 332, 10]" | mercantile shapes --precision 4 --bbox
[-9.1406, 53.1204, -8.7891, 53.3309]
"""
dump_kwds = {"sort_keys": True}
if indent:
dump_kwds["indent"] = indent
if compact:
dump_kwds["separators"] = (",", ":")
src = normalize_input(input)
features = []
col_xs = []
col_ys = []
for i, line in enumerate(iter_lines(src)):
obj = json.loads(line)
if isinstance(obj, dict):
x, y, z = obj["tile"][:3]
props = obj.get("properties")
fid = obj.get("id")
elif isinstance(obj, list):
x, y, z = obj[:3]
props = {}
fid = None
else:
raise click.BadParameter("{0}".format(obj),
param=input,
param_hint="input")
feature = mercantile.feature(
(x, y, z),
fid=fid,
props=props,
projected=projected,
buffer=buffer,
precision=precision,
)
bbox = feature["bbox"]
w, s, e, n = bbox
col_xs.extend([w, e])
col_ys.extend([s, n])
if collect:
features.append(feature)
elif extents:
click.echo(" ".join(map(str, bbox)))
else:
if seq:
click.echo(RS)
if output_mode == "bbox":
click.echo(json.dumps(bbox, **dump_kwds))
elif output_mode == "feature":
click.echo(json.dumps(feature, **dump_kwds))
if collect and features:
bbox = [min(col_xs), min(col_ys), max(col_xs), max(col_ys)]
click.echo(
json.dumps(
{
"type": "FeatureCollection",
"bbox": bbox,
"features": features
}, **dump_kwds))
def main(args):
args.out = os.path.expanduser(args.out)
if not args.workers:
args.workers = os.cpu_count()
print("RoboSat.pink - compare {} on CPU, with {} workers".format(args.mode, args.workers), file=sys.stderr, flush=True)
if not args.masks or not args.labels:
assert args.mode != "list", "Parameters masks and labels are mandatories in list mode."
assert args.minimum_fg == 0.0 and args.maximum_fg == 100.0, "Both masks and labels mandatory in QoD filtering."
assert args.minimum_qod == 0.0 and args.maximum_qod == 100.0, "Both masks and labels mandatory in QoD filtering."
if args.images:
tiles = [tile for tile in tiles_from_dir(args.images[0])]
for image in args.images[1:]:
assert sorted(tiles) == sorted([tile for tile in tiles_from_dir(image)]), "Unconsistent images directories"
if args.labels and args.masks:
tiles_masks = [tile for tile in tiles_from_dir(args.masks)]
tiles_labels = [tile for tile in tiles_from_dir(args.labels)]
if args.images:
assert sorted(tiles) == sorted(tiles_masks) == sorted(tiles_labels), "Unconsistent images/label/mask directories"
else:
assert sorted(tiles_masks) == sorted(tiles_labels), "Label and Mask directories are not consistent"
tiles = tiles_masks
tiles_list = []
tiles_compare = []
progress = tqdm(total=len(tiles), ascii=True, unit="tile")
log = False if args.mode == "list" else Logs(os.path.join(args.out, "log"))
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile):
x, y, z = list(map(str, tile))
if args.masks and args.labels:
label = np.array(Image.open(os.path.join(args.labels, z, x, "{}.png".format(y))))
mask = np.array(Image.open(os.path.join(args.masks, z, x, "{}.png".format(y))))
assert label.shape == mask.shape, "Inconsistent tiles (size or dimensions)"
try:
dist, fg_ratio, qod = compare(torch.as_tensor(label, device="cpu"), torch.as_tensor(mask, device="cpu"))
except:
progress.update()
return False, tile
if not args.minimum_fg <= fg_ratio <= args.maximum_fg or not args.minimum_qod <= qod <= args.maximum_qod:
progress.update()
return True, tile
tiles_compare.append(tile)
if args.mode == "side":
for i, root in enumerate(args.images):
img = tile_image_from_file(tile_from_xyz(root, x, y, z)[1])
if i == 0:
side = np.zeros((img.shape[0], img.shape[1] * len(args.images), 3))
side = np.swapaxes(side, 0, 1) if args.vertical else side
image_shape = img.shape
else:
assert image_shape[0:2] == img.shape[0:2], "Unconsistent image size to compare"
if args.vertical:
side[i * image_shape[0] : (i + 1) * image_shape[0], :, :] = img
else:
side[:, i * image_shape[0] : (i + 1) * image_shape[0], :] = img
tile_image_to_file(args.out, tile, np.uint8(side))
elif args.mode == "stack":
for i, root in enumerate(args.images):
tile_image = tile_image_from_file(tile_from_xyz(root, x, y, z)[1])
if i == 0:
image_shape = tile_image.shape[0:2]
stack = tile_image / len(args.images)
else:
assert image_shape == tile_image.shape[0:2], "Unconsistent image size to compare"
stack = stack + (tile_image / len(args.images))
tile_image_to_file(args.out, tile, np.uint8(stack))
elif args.mode == "list":
tiles_list.append([tile, fg_ratio, qod])
progress.update()
return True, tile
for tile, ok in executor.map(worker, tiles):
if not ok and log:
log.log("Warning: skipping. {}".format(str(tile)))
if args.mode == "list":
with open(args.out, mode="w") as out:
if args.geojson:
out.write('{"type":"FeatureCollection","features":[')
first = True
for tile_list in tiles_list:
tile, fg_ratio, qod = tile_list
x, y, z = list(map(str, tile))
if args.geojson:
prop = '"properties":{{"x":{},"y":{},"z":{},"fg":{:.1f},"qod":{:.1f}}}'.format(x, y, z, fg_ratio, qod)
geom = '"geometry":{}'.format(json.dumps(feature(tile, precision=6)["geometry"]))
out.write('{}{{"type":"Feature",{},{}}}'.format("," if not first else "", geom, prop))
first = False
else:
out.write("{},{},{}\t{:.1f}\t{:.1f}{}".format(x, y, z, fg_ratio, qod, os.linesep))
if args.geojson:
out.write("]}")
out.close()
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
if args.mode == "side" and not args.no_web_ui:
template = "compare.html" if not args.web_ui_template else args.web_ui_template
web_ui(args.out, base_url, tiles, tiles_compare, args.format, template, union_tiles=False)
if args.mode == "stack" and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles_from_dir(args.images[0])]
web_ui(args.out, base_url, tiles, tiles_compare, args.format, template)
def shapes(
ctx, input, precision, indent, compact, projected,
seq, output_mode, collect, extents, buffer):
"""Reads one or more Web Mercator tile descriptions
from stdin and writes either a GeoJSON feature collection (the
default) or a JSON sequence of GeoJSON features/collections to
stdout.
Input may be a compact newline-delimited sequences of JSON or
a pretty-printed ASCII RS-delimited sequence of JSON (like
https://tools.ietf.org/html/rfc8142 and
https://tools.ietf.org/html/rfc7159).
Tile descriptions may be either an [x, y, z] array or a JSON
object of the form
{"tile": [x, y, z], "properties": {"name": "foo", ...}}
In the latter case, the properties object will be used to update
the properties object of the output feature.
"""
dump_kwds = {'sort_keys': True}
if indent:
dump_kwds['indent'] = indent
if compact:
dump_kwds['separators'] = (',', ':')
src = normalize_input(input)
features = []
col_xs = []
col_ys = []
for i, line in enumerate(iter_lines(src)):
obj = json.loads(line)
if isinstance(obj, dict):
x, y, z = obj['tile'][:3]
props = obj.get('properties')
fid = obj.get('id')
elif isinstance(obj, list):
x, y, z = obj[:3]
props = {}
fid = None
else:
raise click.BadParameter(
"{0}".format(obj), param=input, param_hint='input')
feature = mercantile.feature(
(x, y, z), fid=fid, props=props, projected=projected,
buffer=buffer, precision=precision)
bbox = feature['bbox']
w, s, e, n = bbox
col_xs.extend([w, e])
col_ys.extend([s, n])
if collect:
features.append(feature)
elif extents:
click.echo(" ".join(map(str, bbox)))
else:
if seq:
click.echo(u'\x1e')
if output_mode == 'bbox':
click.echo(json.dumps(bbox, **dump_kwds))
elif output_mode == 'feature':
click.echo(json.dumps(feature, **dump_kwds))
if collect and features:
bbox = [min(col_xs), min(col_ys), max(col_xs), max(col_ys)]
click.echo(json.dumps({
'type': 'FeatureCollection',
'bbox': bbox, 'features': features},
**dump_kwds))
