def get_sources(self, bounds, resolution):
bounds, bounds_crs = bounds
zoom = get_zoom(max(resolution))
self._log.info("Resolution: %s; equivalent zoom: %d", resolution, zoom)
left, bottom, right, top = warp.transform_bounds(
bounds_crs, WGS84_CRS, *bounds)
# account for rounding errors when converting between tiles and coords
left += 0.000001
bottom += 0.000001
right -= 0.000001
top -= 0.000001
if (self._bounds[0] <= left <= self._bounds[2]
or self._bounds[0] <= right <= self._bounds[2]) and (
self._bounds[1] <= bottom <= self._bounds[3]
or self._bounds[1] <= top <= self._bounds[3]) and (
self._minzoom <= zoom <= self._maxzoom):
tile = mercantile.bounding_tile(left, bottom, right, top)
with requests.get(
self.endpoint.format(x=tile.x, y=tile.y, z=tile.z)) as rsp:
if not rsp:
self._log.warn("%s failed: %s", rsp.url, rsp.text)
return
for source in rsp.json():
yield Source(**source)
def __init__(self, uri):
try:
if uri.startswith("s3://"):
url = urlparse(uri)
obj = S3.get_object(Bucket=url.netloc, Key=url.path[1:])
oin_meta = json.loads(obj["Body"].read().decode("utf-8"))
elif uri.startswith(("http://", "https://")):
oin_meta = requests.get(uri).json()
else:
raise NoCatalogAvailable()
except Exception:
raise NoCatalogAvailable()
self._meta = oin_meta
self._metadata_url = uri
self._name = oin_meta.get("title")
self._provider = oin_meta.get("provider")
self._source = oin_meta.get("uuid")
with get_source(self._source) as src:
self._bounds = warp.transform_bounds(src.crs, WGS84_CRS,
*src.bounds)
self._resolution = get_resolution_in_meters(
Bounds(src.bounds, src.crs), (src.height, src.width))
approximate_zoom = get_zoom(max(self._resolution), op=math.ceil)
if src.meta["dtype"] != "uint8":
global_min = src.get_tag_item("TIFFTAG_MINSAMPLEVALUE")
global_max = src.get_tag_item("TIFFTAG_MAXSAMPLEVALUE")
for band in range(0, src.count):
self._meta["values"] = self._meta.get("values", {})
self._meta["values"][band] = {}
min_val = src.get_tag_item("STATISTICS_MINIMUM",
bidx=band + 1)
max_val = src.get_tag_item("STATISTICS_MAXIMUM",
bidx=band + 1)
mean_val = src.get_tag_item("STATISTICS_MEAN",
bidx=band + 1)
if min_val is not None:
self._meta["values"][band]["min"] = float(min_val)
elif global_min is not None:
self._meta["values"][band]["min"] = float(global_min)
if max_val is not None:
self._meta["values"][band]["max"] = float(max_val)
elif global_max is not None:
self._meta["values"][band]["max"] = float(global_max)
if mean_val is not None:
self._meta["values"][band]["mean"] = float(mean_val)
self._center = [
(self._bounds[0] + self.bounds[2]) / 2,
(self._bounds[1] + self.bounds[3]) / 2,
approximate_zoom - 3,
]
self._maxzoom = approximate_zoom + 3
self._minzoom = approximate_zoom - 10
def _candidates(self, bounds, resolution):
cursor = self.conn.cursor()
zoom = get_zoom(max(resolution))
if bounds.crs == WGS84_CRS:
left, bottom, right, top = bounds.bounds
else:
left, bottom, right, top = warp.transform_bounds(
bounds.crs, WGS84_CRS, *bounds.bounds)
left = left if left != Infinity else -180
bottom = bottom if bottom != Infinity else -90
right = right if right != Infinity else 180
top = top if top != Infinity else 90
try:
cursor.execute(
"""
WITH bbox AS (
SELECT SetSRID(
GeomFromText('BOX({minx} {miny}, {maxx} {maxy})'),
4326) geom
),
sources AS (
SELECT
url,
source,
resolution,
coalesce(band_info, '{{}}') band_info,
coalesce(meta, '{{}}') meta,
coalesce(recipes, '{{}}') recipes,
acquired_at,
priority,
ST_Multi(footprints.geom) geom,
min_zoom,
max_zoom
FROM footprints
JOIN bbox ON ST_Intersects(footprints.geom, bbox.geom)
WHERE ? BETWEEN min_zoom AND max_zoom
)
SELECT
url,
source,
resolution,
band_info,
meta,
recipes,
acquired_at,
null band,
priority
FROM sources
""".format(minx=left, miny=bottom, maxx=right, maxy=top), (zoom, ))
for record in cursor:
yield Source(*record)
except Exception as e:
print(e)
LOG.warn(e)
finally:
cursor.close()
def __init__(self, uri):
rsp = requests.get(uri)
if not rsp.ok:
raise NoDataAvailable()
oin_meta = rsp.json()
self._meta = oin_meta
self._metadata_url = uri
self._name = oin_meta.get('title')
self._provider = oin_meta.get('provider')
self._source = oin_meta.get('uuid')
with get_source(self._source) as src:
self._bounds = warp.transform_bounds(src.crs, WGS84_CRS,
*src.bounds)
self._resolution = get_resolution_in_meters(
Bounds(src.bounds, src.crs), (src.height, src.width))
approximate_zoom = get_zoom(max(self._resolution), op=math.ceil)
self._center = [(self._bounds[0] + self.bounds[2]) / 2,
(self._bounds[1] + self.bounds[3]) / 2,
approximate_zoom - 3]
self._maxzoom = approximate_zoom + 3
self._minzoom = approximate_zoom - 10
def __init__(self, uri, rgb=None, nodata=None, linear_stretch=None, resample=None):
self._uri = uri
if rgb:
self._rgb = rgb
if nodata:
self._nodata = nodata
if linear_stretch:
self._linear_stretch = linear_stretch
try:
# test whether provided resampling method is valid
Resampling[resample]
self._resample = resample
except KeyError:
self._resample = None
self._meta = {}
with get_source(self._uri) as src:
self._bounds = warp.transform_bounds(src.crs, WGS84_CRS, *src.bounds)
self._resolution = get_resolution_in_meters(
Bounds(src.bounds, src.crs), (src.height, src.width)
)
approximate_zoom = get_zoom(max(self._resolution), op=math.ceil)
global_min = src.get_tag_item("TIFFTAG_MINSAMPLEVALUE")
global_max = src.get_tag_item("TIFFTAG_MAXSAMPLEVALUE")
for band in range(0, src.count):
self._meta["values"] = self._meta.get("values", {})
self._meta["values"][band] = {}
min_val = src.get_tag_item("STATISTICS_MINIMUM", bidx=band + 1)
max_val = src.get_tag_item("STATISTICS_MAXIMUM", bidx=band + 1)
mean_val = src.get_tag_item("STATISTICS_MEAN", bidx=band + 1)
if min_val is not None:
self._meta["values"][band]["min"] = float(min_val)
elif global_min is not None:
self._meta["values"][band]["min"] = float(global_min)
if max_val is not None:
self._meta["values"][band]["max"] = float(max_val)
elif global_max is not None:
self._meta["values"][band]["max"] = float(global_max)
if mean_val is not None:
self._meta["values"][band]["mean"] = float(mean_val)
self._center = [
(self._bounds[0] + self.bounds[2]) / 2,
(self._bounds[1] + self.bounds[3]) / 2,
approximate_zoom - 3,
]
self._maxzoom = approximate_zoom + 3
self._minzoom = approximate_zoom - 10
def get_sources(self, bounds, resolution):
bounds, bounds_crs = bounds
zoom = get_zoom(max(resolution))
left, bottom, right, top = warp.transform_bounds(
bounds_crs, WGS84_CRS, *bounds)
if (self._bounds[0] <= left <= self._bounds[2]
or self._bounds[0] <= right <= self._bounds[2]) and (
self._bounds[1] <= bottom <= self._bounds[3]
or self._bounds[1] <= top <= self._bounds[3]) and (
self._minzoom <= zoom <= self._maxzoom):
yield Source(self._source, self._name, self._resolution, {}, {},
{"imagery": True})
def get_sources(self, bounds, resolution):
from shapely.geometry import box
bounds, bounds_crs = bounds
results = []
zoom = get_zoom(max(resolution))
((left, right),
(bottom, top)) = warp.transform(bounds_crs, WGS84_CRS, bounds[::2],
bounds[1::2])
bounds_geom = box(left, bottom, right, top)
bounds_centroid = bounds_geom.centroid
# Filter by zoom level and intersecting geometries
for candidate in self._sources:
(geom, attr) = candidate
if attr['min_zoom'] <= zoom < attr['max_zoom'] and \
geom.intersects(bounds_geom):
results.append(candidate)
# Sort by resolution and centroid distance
results = sorted(results,
key=lambda (geom, attr): (
attr['priority'],
int(attr['resolution']),
bounds_centroid.distance(geom.centroid),
))
# Remove duplicate URLs
# From https://stackoverflow.com/a/480227
seen = set()
seen_add = seen.add
results = [
x for x in results
if not (x[1]['url'] in seen or seen_add(x[1]['url']))
]
# Pick only the attributes we care about
results = [(a['url'], a['source'], a['resolution'])
for (_, a) in results]
return results
def get_sources(self, bounds, resolution):
cursor = self.conn.cursor()
# TODO this is becoming relatively standard catalog boilerplate
zoom = get_zoom(max(resolution))
if bounds.crs == WGS84_CRS:
left, bottom, right, top = bounds.bounds
else:
left, bottom, right, top = warp.transform_bounds(
bounds.crs, WGS84_CRS, *bounds.bounds)
left = left if left != Infinity else -180
bottom = bottom if bottom != Infinity else -90
right = right if right != Infinity else 180
top = top if top != Infinity else 90
try:
query = """
WITH bbox AS (
SELECT SetSRID(GeomFromGeoJSON(?), 4326) geom
),
uncovered AS (
SELECT SetSRID(GeomFromGeoJSON(?), 4326) geom
),
date_range AS (
SELECT
COALESCE(min(acquired_at), date('1970-01-01')) min,
COALESCE(max(acquired_at), date('1970-01-01')) max
FROM footprints
)
SELECT
url,
source,
resolution,
coalesce(band_info, '{{}}') band_info,
coalesce(meta, '{{}}') meta,
coalesce(recipes, '{{}}') recipes,
acquired_at,
null band, -- for Source constructor compatibility
priority,
ST_Area(ST_Intersection(uncovered.geom, ST_Difference(footprints.geom, footprints.mask))) /
ST_Area(bbox.geom) coverage,
AsGeoJSON(ST_Intersection(bbox.geom, footprints.geom)) geom,
AsGeoJSON(ST_Intersection(footprints.mask, bbox.geom)) mask,
AsGeoJSON(ST_Difference(uncovered.geom, ST_Difference(footprints.geom, footprints.mask))) uncovered
FROM bbox, date_range, footprints
JOIN uncovered ON ST_Intersects(footprints.geom, uncovered.geom)
WHERE footprints.source || ' - ' || footprints.url NOT IN ({id_placeholders})
AND ? BETWEEN min_zoom AND max_zoom
ORDER BY
10 * coalesce(footprints.priority, 0.5) *
.1 * (1 - (strftime('%s') -
strftime('%s', COALESCE(acquired_at, date('2000-01-01')))) /
(strftime('%s') - strftime('%s', date_range.min))) *
50 *
-- de-prioritize over-zoomed sources
CASE WHEN ? / footprints.resolution >= 1
THEN 1
ELSE 1 / footprints.resolution
END *
ST_Area(
ST_Intersection(bbox.geom, footprints.geom)) /
ST_Area(bbox.geom) DESC
LIMIT 1
"""
bbox = json.dumps({
"type":
"Polygon",
"coordinates": [[
[left, bottom],
[left, top],
[right, top],
[right, bottom],
[left, bottom],
]],
})
uncovered = bbox
ids = set()
while True:
id_placeholders = ", ".join("?" * len(ids))
cursor.execute(
query.format(id_placeholders=id_placeholders),
(bbox, uncovered) + tuple(ids) + (zoom, min(resolution)),
)
count = 0
for record in cursor:
count += 1
(
url,
source,
res,
band_info,
meta,
recipes,
acquired_at,
band,
priority,
coverage,
_,
mask,
uncovered,
) = record
if mask is not None:
mask = json.loads(mask)
yield Source(
url,
source,
res,
json.loads(band_info),
json.loads(meta),
json.loads(recipes),
acquired_at,
band,
priority,
coverage,
mask=mask,
)
ids.add(source + " - " + url)
if count == 0 or uncovered is None:
break
except Exception as e:
LOG.exception(e)
finally:
cursor.close()
def _fill_bounds(self, bounds, resolution, include_geometries=False):
zoom = get_zoom(max(resolution))
query = """
WITH RECURSIVE bbox AS (
SELECT ST_SetSRID(
'BOX(%(minx)s %(miny)s, %(maxx)s %(maxy)s)'::box2d,
4326) geom
),
date_range AS (
SELECT
COALESCE(min(acquired_at), '1970-01-01') min,
COALESCE(max(acquired_at), '1970-01-01') max,
age(COALESCE(max(acquired_at), '1970-01-01'),
COALESCE(min(acquired_at), '1970-01-01')) "interval"
FROM {table}
),
sources AS (
SELECT * FROM (
SELECT
1 iterations,
ARRAY[url] urls,
ARRAY[source] sources,
ARRAY[resolution] resolutions,
ARRAY[coalesce(bands, '{{}}'::jsonb)] bands,
ARRAY[coalesce(meta, '{{}}'::jsonb)] metas,
ARRAY[coalesce(recipes, '{{}}'::jsonb)] recipes,
ARRAY[acquired_at] acquisition_dates,
ARRAY[priority] priorities,
ARRAY[ST_Area(ST_Intersection(bbox.geom, footprints.geom)) /
ST_Area(bbox.geom)] coverages,
ARRAY[ST_Multi(footprints.geom)] geometries,
ST_Multi(footprints.geom) geom,
ST_Difference(bbox.geom, footprints.geom) uncovered
FROM date_range, {table} footprints
JOIN bbox ON footprints.geom && bbox.geom
WHERE %(zoom)s BETWEEN min_zoom AND max_zoom
AND footprints.enabled = true
ORDER BY
10 * coalesce(footprints.priority, 0.5) *
.1 * (1 - (extract(
EPOCH FROM (current_timestamp - COALESCE(
acquired_at, '2000-01-01'))) /
extract(
EPOCH FROM (current_timestamp - date_range.min)))) *
50 *
-- de-prioritize over-zoomed sources
CASE WHEN %(resolution)s / footprints.resolution >= 1
THEN 1
ELSE 1 / footprints.resolution
END *
ST_Area(
ST_Intersection(bbox.geom, footprints.geom)) /
ST_Area(bbox.geom) DESC
LIMIT 1
) AS _
UNION ALL
SELECT * FROM (
SELECT
sources.iterations + 1,
sources.urls || url urls,
sources.sources || source sources,
sources.resolutions || resolution resolutions,
sources.bands || coalesce(
footprints.bands, '{{}}'::jsonb) bands,
sources.metas || coalesce(meta, '{{}}'::jsonb) metas,
sources.recipes || coalesce(
footprints.recipes, '{{}}'::jsonb) recipes,
sources.acquisition_dates || footprints.acquired_at
acquisition_dates,
sources.priorities || footprints.priority priorities,
sources.coverages || ST_Area(
ST_Intersection(sources.uncovered, footprints.geom)) /
ST_Area(bbox.geom) coverages,
sources.geometries || footprints.geom,
ST_Collect(sources.geom, footprints.geom) geom,
ST_Difference(sources.uncovered, footprints.geom) uncovered
FROM bbox, date_range, {table} footprints
-- use proper intersection to prevent voids from irregular
-- footprints
JOIN sources ON ST_Intersects(
footprints.geom, sources.uncovered)
WHERE NOT (footprints.url = ANY(sources.urls))
AND %(zoom)s BETWEEN min_zoom AND max_zoom
AND footprints.enabled = true
ORDER BY
10 * coalesce(footprints.priority, 0.5) *
.1 * (1 - (extract(
EPOCH FROM (current_timestamp - COALESCE(
acquired_at, '2000-01-01'))) /
extract(
EPOCH FROM (current_timestamp - date_range.min)))) *
50 *
-- de-prioritize over-zoomed sources
CASE WHEN %(resolution)s / footprints.resolution >= 1
THEN 1
ELSE 1 / footprints.resolution
END *
ST_Area(
ST_Intersection(sources.uncovered, footprints.geom)) /
ST_Area(bbox.geom) DESC
LIMIT 1
) AS _
),
candidates AS (
SELECT *
FROM sources
ORDER BY iterations DESC
LIMIT 1
), candidate_rows AS (
SELECT
unnest(urls) url,
unnest(sources) source,
unnest(resolutions) resolution,
unnest(bands) bands,
unnest(metas) meta,
unnest(recipes) recipes,
unnest(acquisition_dates) acquired_at,
unnest(priorities) priority,
unnest(coverages) coverage,
unnest(geometries) geom
FROM candidates
)
SELECT
url,
source,
resolution,
bands,
meta,
recipes,
acquired_at,
null band,
priority,
coverage,
CASE WHEN {include_geometries}
THEN ST_AsGeoJSON(geom)
ELSE 'null'
END geom
FROM candidate_rows
""".format(table=self.table,
geometry_column=self.geometry_column,
include_geometries=bool(include_geometries))
if bounds.crs == WGS84_CRS:
left, bottom, right, top = bounds.bounds
else:
left, bottom, right, top = warp.transform_bounds(
bounds.crs, WGS84_CRS, *bounds.bounds)
connection = self._pool.getconn()
try:
with connection as conn, conn.cursor() as cur:
cur.execute(
query, {
"minx": left if left != Infinity else -180,
"miny": bottom if bottom != Infinity else -90,
"maxx": right if right != Infinity else 180,
"maxy": top if top != Infinity else 90,
"zoom": zoom,
"resolution": min(resolution),
})
for record in cur:
yield Source(*record[:-1], geom=json.loads(record[-1]))
except Exception as e:
self._log.error(e)
finally:
self._pool.putconn(connection)
def transform(self, pixels):
data, (bounds, crs), _ = pixels
(count, height, width) = data.shape
if count != 1:
raise Exception("Can't hillshade from multiple bands")
(dx, dy) = get_resolution_in_meters(pixels.bounds, (height, width))
zoom = get_zoom(max(dx, dy))
# invert resolutions for hillshading purposes
dy *= -1
data = apply_latitude_adjustments(pixels).data
resample_factor = RESAMPLING.get(zoom, 1.0)
aff = transform.from_bounds(*bounds, width=width, height=height)
if self.resample and resample_factor != 1.0:
# resample data according to Tom Paterson's chart
# create an empty target array that's the shape of the resampled
# tile (e.g. 80% of 260x260px)
resampled_height = int(round(height * resample_factor))
resampled_width = int(round(width * resample_factor))
resampled = np.empty(
shape=(resampled_height, resampled_width), dtype=data.dtype)
resampled_mask = np.empty(shape=(resampled.shape))
newaff = transform.from_bounds(
*bounds, width=resampled_width, height=resampled_height)
# downsample using GDAL's reprojection functionality (which gives
# us access to different resampling algorithms)
warp.reproject(
data,
resampled,
src_transform=aff,
dst_transform=newaff,
src_crs=crs,
dst_crs=crs,
resampling=Resampling.bilinear, )
# reproject / resample the mask so that intermediate operations
# can also use it
if np.any(data.mask):
warp.reproject(
data.mask.astype(np.uint8),
resampled_mask,
src_transform=aff,
dst_transform=newaff,
src_crs=crs,
dst_crs=crs,
resampling=Resampling.nearest, )
resampled = np.ma.masked_array(resampled, mask=resampled_mask)
else:
resampled = np.ma.masked_array(resampled)
hs = _hillshade(
resampled,
dx=dx,
dy=dy,
vert_exag=EXAGGERATION.get(zoom, 1.0), )
if self.add_slopeshade:
ss = slopeshade(
resampled,
dx=dx,
dy=dy,
vert_exag=EXAGGERATION.get(zoom, 1.0))
hs *= ss
# scale hillshade values (0.0-1.0) to integers (0-255)
hs = (255.0 * hs).astype(np.uint8)
# create an empty target array that's the shape of the target tile
# + buffers (e.g. 260x260px)
resampled_hs = np.empty(shape=data.shape, dtype=hs.dtype)
# upsample (invert the previous reprojection)
warp.reproject(
hs.data,
resampled_hs,
src_transform=newaff,
dst_transform=aff,
src_crs=crs,
dst_crs=crs,
resampling=Resampling.bilinear, )
hs = np.ma.masked_array(resampled_hs, mask=data.mask)
else:
hs = _hillshade(
data[0],
dx=dx,
dy=dy,
vert_exag=EXAGGERATION.get(zoom, 1.0), )
if self.add_slopeshade:
ss = slopeshade(
data[0],
dx=dx,
dy=dy,
vert_exag=EXAGGERATION.get(zoom, 1.0))
# hs *= 0.8
hs *= ss
hs = np.ma.masked_array(hs[np.newaxis], mask=data.mask)
# scale hillshade values (0.0-1.0) to integers (0-255)
hs = (255.0 * hs).astype(np.uint8)
hs.fill_value = 0
return PixelCollection(hs, pixels.bounds), "raw"
def __init__(self, uri, rgb=None, nodata=None, linear_stretch=None, resample=None,
dst_max=None, dst_min=None, force_cast=None, to_vis=None):
self._uri = uri
self._rgb = rgb
self._nodata = nodata
self._linear_stretch = linear_stretch
self._dst_min = dst_min
self._dst_max = dst_max
self._force_cast = force_cast
self._to_vis = to_vis
try:
# test whether provided resampling method is valid
Resampling[resample]
self._resample = resample
except KeyError:
self._resample = None
self._meta = {}
self.src_meta = {}
with get_source(self._uri) as src:
self.src_meta = snake_case_to_camel_case_keys_of_dict(src.tags())
self.src_meta["bandCount"] = src.count
self._bounds = warp.transform_bounds(src.crs, WGS84_CRS, *src.bounds)
self._resolution = get_resolution_in_meters(
Bounds(src.bounds, src.crs), (src.height, src.width)
)
approximate_zoom = get_zoom(max(self._resolution), op=math.ceil)
global_min = src.get_tag_item("TIFFTAG_MINSAMPLEVALUE")
global_max = src.get_tag_item("TIFFTAG_MAXSAMPLEVALUE")
band_order = src.get_tag_item("BAND_ORDER")
if band_order is not None:
band_order = band_order.split(',')
if str(self._rgb).lower() == "metadata":
if band_order is not None:
def get_band_from_band_order(band_order, band_name, fallback):
if band_name in band_order:
return str(band_order.index(band_name) + 1)
else:
return fallback
red_band = get_band_from_band_order(band_order, "RED", "1")
green_band = get_band_from_band_order(band_order, "GRE", "2")
blue_band = get_band_from_band_order(band_order, "BLU", "3")
self._rgb = ",".join([red_band, green_band, blue_band])
else:
# Fallback
if src.count >= 3:
self._rgb = "1,2,3"
else:
self._rgb = "1,1,1"
self.src_meta["bandMetadata"] = {}
# FarmLens specific
band_assignments = band_order
if band_order is None:
band_assignments = range(0, src.count)
for band in xrange(0, src.count):
self.src_meta["bandMetadata"][band_assignments[band]] = snake_case_to_camel_case_keys_of_dict(src.tags(bidx=band+1))
self._meta["values"] = self._meta.get("values", {})
self._meta["values"][band] = {}
min_val = src.get_tag_item("STATISTICS_MINIMUM", bidx=band + 1)
max_val = src.get_tag_item("STATISTICS_MAXIMUM", bidx=band + 1)
mean_val = src.get_tag_item("STATISTICS_MEAN", bidx=band + 1)
stddev_val = src.get_tag_item("STATISTICS_STDDEV", bidx=band + 1)
if min_val is not None:
self._meta["values"][band]["min"] = float(min_val)
elif global_min is not None:
self._meta["values"][band]["min"] = float(global_min)
if max_val is not None:
self._meta["values"][band]["max"] = float(max_val)
elif global_max is not None:
self._meta["values"][band]["max"] = float(global_max)
if mean_val is not None:
self._meta["values"][band]["mean"] = float(mean_val)
if stddev_val is not None:
self._meta["values"][band]["stddev"] = float(stddev_val)
self._center = [
(self._bounds[0] + self.bounds[2]) / 2,
(self._bounds[1] + self.bounds[3]) / 2,
approximate_zoom - 3,
]
self._maxzoom = approximate_zoom + 3
self._minzoom = approximate_zoom - 10
