def __init__(self, imagedir, maskdir, joint_transform=None, indices=None):
"""
imagedir: flat directory of N-band TIFF images as above
maskdir: flat directory of 1-band TIFF masks as above
joint_transform = a transforms.JointTransform object to apply to data
indices = optional list of 0-indexed image indices for subsetting.
"""
super().__init__()
self.joint_transform = joint_transform
self.imagedir = imagedir
self.maskdir = maskdir
self.images = os.listdir(imagedir)
self.masks = os.listdir(maskdir)
print(self.imagedir)
if indices is not None:
self.images = [self.images[i] for i in indices]
self.imagetiles = [Tile(*f.split("_")[:3]) for f in self.images]
self.masktiles = [Tile(*f.split("_")[:3]) for f in self.masks]
assert (set(self.imagetiles).issubset(set(
self.masktiles))), "Image and Mask tilesets must be equal!"
def prefetch(self,
tile: Tile,
host_dir: str = None,
verbose: bool = False):
x, y, z = tile
if z != int(z):
raise ValueError("Fractional zooms not allowed!")
if x == int(x) and y == int(y):
x, y = int(x), int(y)
tile_parents = [parent(tile, zoom=i) for i in range(z + 1)]
for tile_parent in tile_parents:
if tile_parent in self.cached_tiles:
return
if z < 14:
blob = children(tile, zoom=14)
if verbose:
from tqdm import tqdm
blob = tqdm(blob, desc="OpenStreetMap Prefetch")
for child in blob:
self.prefetch(child)
else:
self.__get_from_osmnx(tile)
if host_dir is not None:
self.save(host_dir)
return
source_tiles = [
Tile(np.trunc(x), np.trunc(y), z),
Tile(np.trunc(x), np.ceil(y), z),
Tile(np.ceil(x), np.trunc(y), z),
Tile(np.ceil(x), np.ceil(y), z),
]
for tile in source_tiles:
self.prefetch(tile)
return
def gdf_from_tile(self, tile: Tile) -> gpd.GeoDataFrame:
x, y, z = tile
if z != int(z):
raise ValueError("Fractional zooms not allowed!")
if x == int(x) and y == int(y):
x, y, z = int(x), int(y), int(z)
tile_parents = [parent((x, y, z), zoom=i) for i in range(z + 1)]
for tile_parent in tile_parents:
if tile_parent in self.cached_tiles:
return self.__get_from_cache(tile)
if self.use_overpass:
print(
f"[WARNING] Fetching {tile} from OpenStreetMap! This is slower than using regional shapefiles."
)
return self.__get_from_osmnx(tile)
else:
return self.__get_from_cache(tile)
source_tiles = [
Tile(np.trunc(x), np.trunc(y), z),
Tile(np.trunc(x), np.ceil(y), z),
Tile(np.ceil(x), np.trunc(y), z),
Tile(np.ceil(x), np.ceil(y), z),
]
returned_gdfs = [self.gdf_from_tile(tile) for tile in source_tiles]
return pd.concat(returned_gdfs, ignore_index=True)
def test_kml_tiles():
tiles = {
Tile(x=8783, y=5410, z=14),
Tile(x=8783, y=5411, z=14),
Tile(x=8783, y=5412, z=14),
}
assert statshunters.kml_tiles(statshunters.tiles_geometry(tiles)) != ""
def pred_2geojson(csv_file, keyword, threshold):
features = list()
with open(csv_file, 'r') as csvfile:
reader = csv.reader(csvfile)
next(reader)
for row in tqdm(reader):
pred = json.loads(row[1])
pred_red = list(map(lambda x: round(x, 2), pred))
pred_obj = dict(
zip(map(lambda x: 'p%s' % x, range(len(pred_red))), pred_red))
if keyword == "point":
pred_j = ul(*[int(t) for t in row[0].split('-')])
feature_collection = Feature(geometry=dict(
type='Point', coordinates=[pred_j.lng, pred_j.lat]),
properties=pred_obj)
else:
pred_j = feature(Tile(*[int(t) for t in row[0].split('-')]))
feature_ = Feature(geometry=pred_j['geometry'],
properties=pred_obj)
if pred_obj['p1'] >= float(threshold):
features.append(feature_)
feature_collection = FeatureCollection(features)
with open('results_{}_{}.geojson'.format(keyword, threshold),
'w') as results:
json.dump(feature_collection, results)
def parse_tile_str(tile_str):
""" Parse a string like tiletype/zoom/x/y.suffix
into a (tiletype, tile) tuple. """
m = TILE_RE.match(tile_str)
return (m.group(1), Tile(int(m.group(3)), int(m.group(4)),
int(m.group(2))))
def render_tif(z, x, y):
tile = Tile(x, y, z)
headers, data = tiling.render_tile(tile, CATALOG, format=GEOTIFF_FORMAT)
headers.update(CATALOG.headers)
return data, 200, headers
def resolve_seams(self):
tile_indices = arange(TILE_SIZE**2)
left_indices = tile_indices[::TILE_SIZE]
bottom_indices = tile_indices[TILE_SIZE**2 - TILE_SIZE:]
right_indices = tile_indices[TILE_SIZE - 1::TILE_SIZE]
top_indices = tile_indices[:TILE_SIZE]
for mesh in self.items:
geometry = mesh.geometry
t = geometry.tile
left = Tile(t.x - 1, t.y, t.z)
bottom = Tile(t.x, t.y + 1, t.x)
right = Tile(t.x + 1, t.y, t.z)
top = Tile(t.x, t.y - 1, t.z)
neighborhood = [
x.geometry for x in self.items
if x.geometry.tile in (left, bottom, right, top)
]
if neighborhood:
heights = geometry.heights.ravel()
for neighbor in neighborhood:
ntile = neighbor.tile
if ntile == left:
indices = left_indices
neighbor_indices = right_indices
elif ntile == bottom:
indices = bottom_indices
neighbor_indices = top_indices
elif ntile == right:
indices = right_indices
neighbor_indices = left_indices
elif ntile == top:
indices = top_indices
neighbor_indices = bottom_indices
else:
raise ValueError("Neighbor is not a neighbor!")
neighbor_heights = neighbor.heights.ravel()
heights[indices] = neighbor_heights[neighbor_indices]
geometry.heights = heights.reshape((TILE_SIZE, TILE_SIZE))
def render_geotiff(z, x, y, **kwargs):
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
ELEVATION_CATALOG,
format=GEOTIFF_FORMAT,
scale=2,
data_band_count=1)
return data, 200, headers
def render_hillshade_tiff(z, x, y, **kwargs):
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
CATALOGS['hillshade'],
format=HILLSHADE_GEOTIFF_FORMAT,
transformation=HILLSHADE_TRANSFORMATION,
scale=2,
data_band_count=1)
return data, 200, headers
def get_tiles(event: SQSEvent) -> List[Tile]:
"""
Return the body of our incoming SQS messages as an array of mercantile Tiles
Expects events of the following format:
{ 'Records': [ { "body": '{ "x": 4, "y": 5, "z":3 }' }] }
"""
return [
Tile(*json.loads(record['body']).values())
for record in event['Records']
]
def get_strided_tiles(root: Tile, zoom: int = 15):
child_bounds = children(root, zoom=zoom)
min_x = min((child.x for child in child_bounds))
min_y = min((child.y for child in child_bounds))
max_x = max((child.x for child in child_bounds))
max_y = max((child.y for child in child_bounds))
n = max_x - min_x + 1
strided_tiles = []
for x, y in product(np.arange(min_x, max_x + 0.1, 0.5),
np.arange(min_y, max_y + 0.1, 0.5)):
strided_tiles += [Tile(float(x), float(y), zoom)]
return strided_tiles
def render_png(renderer, z, x, y, scale=1, **kwargs):
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
CATALOGS[renderer],
format=FORMATS[renderer],
transformation=TRANSFORMATIONS.get(renderer),
scale=scale,
data_band_count=DATA_BAND_COUNTS.get(renderer, 1))
return data, 200, headers
def convert_csv(fname_csv, fname_geojson, tile_format, thresh_ind, thresh):
"""Convert tile indices in CSV file to geojson"""
if not op.exists(fname_csv):
raise ValueError(f'Cannot find file {fname_csv}')
# Error check tile format
if tile_format == 'tms':
tile_func = Pygeo_tile.from_tms
elif tile_format == 'google':
tile_func = Pygeo_tile.from_google
else:
raise ValueError(f'Tile format not understood. Got: {tile_format}')
if not 0 <= thresh <= 1.:
raise ValueError(f"'thresh' must be on interval [0, 1]. Got: {thresh}")
with open(fname_csv, 'r') as csvfile:
with open(fname_geojson, 'w') as results:
reader = csv.reader(csvfile)
first_line = True
# Create a FeatureCollection
results.write('{"type":"FeatureCollection","features":[')
next(reader) # Skip header
for row in reader:
# Load as pygeotile using TMS coords
geot = tile_func(*[int(t) for t in row[0].split('-')])
# Create feature with mercantile
feat = feature(Tile(geot.google[0], geot.google[1], geot.zoom))
# Get class prediction confidences
pred = json.loads(','.join(row[1:]))
pred_red = list(map(lambda x: round(x, 2), pred))
if pred_red[thresh_ind] >= thresh:
# Add commas prior to any feature that isn't the first one
if first_line:
first_line = False
else:
results.write(',')
pred_obj = dict(zip(map(lambda x: 'p%s' % x,
range(len(pred_red))), pred_red))
results.write(json.dumps(Feature(geometry=feat['geometry'],
properties=pred_obj)))
# Finalize the feature FeatureCollection
results.write(']}')
def tile(id, z, x, y, **kwargs):
meta = get_metadata(id, **kwargs)
tile = read_tile(id, Tile(x, y, z), **kwargs)
headers = {
'Content-Type': 'image/png',
}
if meta['meta'].get('oinMetadataUrl'):
headers['X-OIN-Metadata-URL'] = meta['meta'].get('oinMetadataUrl')
if meta['meta'].get('acquisitionStart') or meta['meta'].get(
'acquisitionEnd'):
start = meta['meta'].get('acquisitionStart')
end = meta['meta'].get('acquisitionEnd')
if start and end:
start = arrow.get(start)
end = arrow.get(end)
capture_range = '{}-{}'.format(start.format('M/D/YYYY'),
end.format('M/D/YYYY'))
headers['X-OIN-Acquisition-Start'] = start.format(
'YYYY-MM-DDTHH:mm:ssZZ')
headers['X-OIN-Acquisition-End'] = end.format(
'YYYY-MM-DDTHH:mm:ssZZ')
elif start:
start = arrow.get(start)
capture_range = start.format('M/D/YYYY')
headers['X-OIN-Acquisition-Start'] = start.format(
'YYYY-MM-DDTHH:mm:ssZZ')
elif end:
end = arrow.get(end)
capture_range = end.format('M/D/YYYY')
headers['X-OIN-Acquisition-End'] = end.format(
'YYYY-MM-DDTHH:mm:ssZZ')
# Bing Maps-compatibility (JOSM uses this)
headers['X-VE-TILEMETA-CaptureDatesRange'] = capture_range
if meta['meta'].get('provider'):
headers['X-OIN-Provider'] = unicodedata.normalize(
'NFKD', meta['meta'].get('provider')).encode('ascii', 'ignore')
if meta['meta'].get('platform'):
headers['X-OIN-Platform'] = unicodedata.normalize(
'NFKD', meta['meta'].get('platform')).encode('ascii', 'ignore')
return tile, 200, headers
def render_png(z, x, y, scale=1):
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
CATALOG,
format=IMAGE_FORMAT,
transformation=COLORMAP_TRANSFORMATION,
scale=scale,
)
headers.update(CATALOG.headers)
return data, 200, headers
def test_get_tiles():
# create a class with fake environment variables
dap = DownloadAndPredict(
imagery='https://example.com/{z}/{x}/{y}.png',
db='postgres://*****:*****@host:port/database',
prediction_endpoint='https://myloadbalancer.com/v1/models/ml:predict'
)
# create an example SQS event which invokes a lambda
event = { 'Records': [ { "body": '{ "x": 4, "y": 5, "z":3 }' }] }
tiles = dap.get_tiles(event)
fixture_tiles = [Tile(x=4, y=5, z=3)]
assert(tiles == fixture_tiles)
def user_render(id, z, x, y, scale=1, prefix=None):
catalog = make_remote_catalog("user", id)
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
catalog,
format=OPTIMAL_FORMAT,
transformation=IMAGE_TRANSFORMATION,
scale=scale,
)
headers.update(catalog.headers)
return data, 200, headers
def render_png(z, x, y, scale=1):
catalog = make_catalog(request.args)
tile = Tile(x, y, z)
headers, data = tiling.render_tile(
tile,
catalog,
format=IMAGE_FORMAT,
transformation=IMAGE_TRANSFORMATION,
scale=scale,
)
headers.update(catalog.headers)
return data, 200, headers
def render_json(z, x, y, scale=1):
tile = Tile(x, y, z)
sieve = int(request.args.get("sieve", 4))
headers, data = tiling.render_tile(
tile,
CATALOG,
format=GeoJSON(sieve_size=sieve),
transformation=Transformation(collar=8 * scale),
scale=scale,
)
headers.update(CATALOG.headers)
return data, 200, headers
def generate_tiles(tile, max_zoom, metatile=1, materialize_zooms=None):
tiles = []
metatile = min(metatile, 2**tile.z)
for dx in range(0, metatile):
for dy in range(0, metatile):
tiles.append(Tile(tile.x + dx, tile.y + dy, tile.z))
for z in range(tile.z, max_zoom + 1):
if materialize_zooms is None or z in materialize_zooms:
a, tiles = itertools.tee(tiles, 2)
yield from a
tiles = itertools.chain.from_iterable(
mercantile.children(t) for t in tiles)
def render_json(renderer, z, x, y, scale=1, max_zoom=None, **kwargs):
tile = Tile(x, y, z)
headers = {"Content-Type": "application/json"}
min_zoom = None
if max_zoom is not None:
min_zoom = z
data = list(
footprints.sources_for_tile(
tile,
CATALOGS[renderer],
scale=scale,
min_zoom=min_zoom,
max_zoom=max_zoom))
return jsonify(data), 200, headers
def render_png_from_remote(id, z, x, y, scale=1, prefix=None):
# prefix is for URL generation only (API Gateway stages); if it matched the
# URL, it's part of the id
if prefix is not None:
id = "/".join([prefix, id])
catalog = make_remote_catalog(id)
tile = Tile(x, y, z)
headers, data = tiling.render_tile(tile,
catalog,
format=PNG_FORMAT,
transformation=IMAGE_TRANSFORMATION,
scale=scale)
headers.update(catalog.headers)
return data, 200, headers
def get_image(self,
*tile: Tile,
source_zoom: int = 18,
black_fail: bool = False):
if len(tile) == 1:
tile = tile[0]
x, y, z = tile
if not self.max_res:
source_zoom = z
parents = [
parent(Tile(int(fn(x)), int(fn(y)), z), zoom=i)
for i, fn in product(range(z + 1), [np.ceil, np.floor])
]
for supertile in parents:
if supertile in self.stacks:
return self.__make_image(self.stacks[supertile], tile,
source_zoom)
raise FileNotFoundError(
f"Could not find a parent tilestack for requested subtile {tile}!")
def render_tile(
format,
output,
scale,
transformation,
tile,
):
z, x, y = map(int, tile.split("/"))
tile = Tile(x, y, z)
(headers, data) = tiling.render_tile(
tile,
PostGISCatalog(),
format=FORMATS[format],
transformation=TRANSFORMATIONS.get(transformation),
scale=scale)
[click.echo("{}: {}".format(k, v), err=True) for k, v in headers.items()]
output.write(data)
def render_geojson(renderer, z, x, y, scale=1, max_zoom=None, **kwargs):
tile = Tile(x, y, z)
data = {
"type": "FeatureCollection",
"features": [],
}
headers = {"Content-Type": "application/json"}
min_zoom = None
if max_zoom is not None:
min_zoom = z
[
data["features"].append(footprint)
for footprint in footprints.features_for_tile(
tile,
CATALOGS[renderer],
scale=scale,
min_zoom=min_zoom,
max_zoom=max_zoom)
]
return jsonify(data), 200, headers
"""Module tests"""
from mercantile import Tile
import pytest
import mbtiles.worker
@pytest.mark.parametrize(
"tile", [Tile(36, 73, 7), Tile(0, 0, 0),
Tile(1, 1, 1)])
@pytest.mark.parametrize("filename", ["RGB.byte.tif", "RGBA.byte.tif"])
def test_process_tile(data, filename, tile):
sourcepath = str(data.join(filename))
mbtiles.worker.init_worker(
sourcepath,
{
"driver": "PNG",
"dtype": "uint8",
"nodata": 0,
"height": 256,
"width": 256,
"count": 3,
"crs": "EPSG:3857",
},
"nearest",
{},
{},
)
t, contents = mbtiles.worker.process_tile(tile)
assert t.x == tile.x
def make_labels(dest_folder, zoom, country, classes, ml_type, bounding_box,
sparse, **kwargs):
"""Create label data from OSM QA tiles for specified classes
Perform the following operations:
- If necessary, re-tile OSM QA Tiles to the specified zoom level
- Iterate over all tiles within the bounding box and produce a label for each
- Save the label file as labels.npz
- Create an output for previewing the labels (GeoJSON or PNG depending upon ml_type)
Parameters
------------
dest_folder: str
Folder to save labels and example tiles into
zoom: int
The zoom level to create tiles at
country: str
The OSM QA Tile extract to download. The value should be a country string matching a value found in
`label_maker/countries.txt`
classes: list
A list of classes for machine learning training. Each class is defined as a dict
with two required properties:
- name: class name
- filter: A Mapbox GL Filter.
See the README for more details
ml_type: str
Defines the type of machine learning. One of "classification", "object-detection", or "segmentation"
bounding_box: list
The bounding box to create images from. This should be given in the form: `[xmin, ymin, xmax, ymax]`
as longitude and latitude values between `[-180, 180]` and `[-90, 90]` respectively
sparse: boolean
Limit the total background tiles to write based on `background_ratio` kwarg.
geojson: str
Filepath to optional geojson label input
**kwargs: dict
Other properties from CLI config passed as keywords to other utility functions
"""
mbtiles_file = op.join(dest_folder, '{}.mbtiles'.format(country))
mbtiles_file_zoomed = op.join(dest_folder,
'{}-z{!s}.mbtiles'.format(country, zoom))
if not op.exists(mbtiles_file_zoomed):
filtered_geo = kwargs.get('geojson') or op.join(
dest_folder, '{}.geojson'.format(country))
fast_parse = []
if not op.exists(filtered_geo):
fast_parse = ['-P']
print('Retiling QA Tiles to zoom level {} (takes a bit)'.format(
zoom))
ps = Popen(['tippecanoe-decode', '-c', '-f', mbtiles_file],
stdout=PIPE)
stream_filter_fpath = op.join(op.dirname(label_maker.__file__),
'stream_filter.py')
run([
sys.executable, stream_filter_fpath,
json.dumps(bounding_box)
],
stdin=ps.stdout,
stdout=open(filtered_geo, 'w'))
ps.wait()
run(['tippecanoe', '--no-feature-limit', '--no-tile-size-limit'] +
fast_parse + [
'-l', 'osm', '-f', '-z',
str(zoom), '-Z',
str(zoom), '-o', mbtiles_file_zoomed, filtered_geo
])
# Call tilereduce
print('Determining labels for each tile')
mbtiles_to_reduce = mbtiles_file_zoomed
tilereduce(
dict(zoom=zoom,
source=mbtiles_to_reduce,
bbox=bounding_box,
args=dict(ml_type=ml_type, classes=classes)), _mapper, _callback,
_done)
# Add empty labels to any tiles which didn't have data
empty_label = _create_empty_label(ml_type, classes)
for tile in tiles(*bounding_box, [zoom]):
index = '-'.join([str(i) for i in tile])
global tile_results
if tile_results.get(index) is None:
tile_results[index] = empty_label
# Print a summary of the labels
_tile_results_summary(ml_type, classes)
# If the --sparse flag is provided, limit the total background tiles to write
if sparse:
pos_examples, neg_examples = [], []
for k in tile_results.keys():
# if we don't match any class, this is a negative example
if not sum([
class_match(ml_type, tile_results[k], i + 1)
for i, c in enumerate(classes)
]):
neg_examples.append(k)
else:
pos_examples.append(k)
# Choose random subset of negative examples
n_neg_ex = int(kwargs['background_ratio'] * len(pos_examples))
neg_examples = np.random.choice(neg_examples, n_neg_ex,
replace=False).tolist()
tile_results = {
k: tile_results.get(k)
for k in pos_examples + neg_examples
}
print('Using sparse mode; subselected {} background tiles'.format(
n_neg_ex))
# write out labels as numpy arrays
labels_file = op.join(dest_folder, 'labels.npz')
print('Writing out labels to {}'.format(labels_file))
np.savez(labels_file, **tile_results)
# write out labels as GeoJSON or PNG
if ml_type == 'classification':
features = []
for tile, label in tile_results.items():
feat = feature(Tile(*[int(t) for t in tile.split('-')]))
features.append(
Feature(geometry=feat['geometry'],
properties=dict(label=label.tolist())))
json.dump(fc(features),
open(op.join(dest_folder, 'classification.geojson'), 'w'))
elif ml_type == 'object-detection':
label_folder = op.join(dest_folder, 'labels')
if not op.isdir(label_folder):
makedirs(label_folder)
for tile, label in tile_results.items():
# if we have at least one bounding box label
if bool(label.shape[0]):
label_file = '{}.png'.format(tile)
img = Image.new('RGB', (256, 256))
draw = ImageDraw.Draw(img)
for box in label:
draw.rectangle(((box[0], box[1]), (box[2], box[3])),
outline=class_color(box[4]))
print('Writing {}'.format(label_file))
img.save(op.join(label_folder, label_file))
elif ml_type == 'segmentation':
label_folder = op.join(dest_folder, 'labels')
if not op.isdir(label_folder):
makedirs(label_folder)
for tile, label in tile_results.items():
# if we have any class pixels
if np.sum(label):
label_file = '{}.png'.format(tile)
visible_label = np.array([
class_color(l) for l in np.nditer(label)
]).reshape(256, 256, 3)
img = Image.fromarray(visible_label.astype(np.uint8))
print('Writing {}'.format(label_file))
img.save(op.join(label_folder, label_file))
default=["file://./"],
nargs="+",
help="Target path/URI(s) for archives")
args = parser.parse_args()
if args.verbose:
logger.setLevel(logging.DEBUG)
if len(args.targets) > 1 and not args.skip_meta:
logger.error(
"Outputting meta.json with more than one target is not supported yet."
)
sys.exit(1)
root = Tile(args.x, args.y, args.zoom)
concurrency = args.concurrency
min_zoom = args.zoom
max_zoom = args.max_zoom
metatile = args.metatile
materialize_zooms = args.materialize or []
scale = args.scale
materialize_zooms = list(filter(lambda z: z <= max_zoom,
materialize_zooms))
if args.cache_sources:
logger.info(
"Caching sources for root tile %s from zoom %d to %d",
root,
min_zoom,
def get_scaled_tile(id, z, x, y, scale):
tile = read_tile(id, Tile(x, y, z), scale=scale)
return tile, 200, {
'Content-Type': 'image/png'
}
