def get_size_estimate(provider, bbox, srs='3857'):
"""
Args:
provider: The name of the provider, corresponds to the name field in Export Provider model
bbox: A bbox in the format of an array
srs: An EPSG code for the map being used.
Returns: Estimated size in GB
"""
try:
provider = ExportProvider.objects.get(name=provider)
except ObjectDoesNotExist:
return None
levels = range(provider.level_from, provider.level_to + 1)
req_srs = mapproxy_srs.SRS(srs)
bbox = mapproxy_grid.grid_bbox(bbox, mapproxy_srs.SRS(4326), req_srs)
tile_size = (256, 256)
tile_grid = mapproxy_grid.TileGrid(srs,
tile_size=tile_size,
levels=len(levels))
total_tiles = 0
tiles = []
for level in levels:
# get_affected_level_tiles() returns a list with three
# things: first a tuple with the bounding box, second
# the height and width in tiles, and third a list of tile
# coordinate tuples. result[1] gives the tuple with
# the width/height of the desired set of tiles.
result = tile_grid.get_affected_level_tiles(bbox, int(level))
total_tiles += result[1][0] * result[1][1]
tiles.append(result[1][0] * result[1][1])
return [total_tiles, get_gb_estimate(total_tiles), tiles]
def get_vector_estimate(provider, bbox, srs="4326"):
"""
:param provider: The DataProvider to test
:param bbox: The bounding box of the request
:param srs: The SRS of the bounding box
:return: (estimate in mbs, object w/ metadata about how it was generated)
"""
# TODO tile_grid params should be serialized on all_stats object
tile_grid = ek_stats.get_default_tile_grid()
req_bbox = mapproxy_grid.grid_bbox(bbox, mapproxy_srs.SRS(srs),
tile_grid.srs)
req_area = ek_stats.get_area_bbox(req_bbox)
# Compute estimate
size_per_km, method = ek_stats.query(
provider.slug,
field=Stats.Fields.SIZE,
statistic_name=Stats.MEAN,
bbox=bbox,
bbox_srs=srs,
gap_fill_thresh=0.1,
default_value=0,
)
method["size_per_km"] = size_per_km
return req_area * size_per_km, method
def get_vector_estimate(provider, bbox, srs='4326'):
"""
:param provider: The DataProvider to test
:param bbox: The bounding box of the request
:param srs: The SRS of the bounding box
:return: (estimate in mbs, object w/ metadata about how it was generated)
"""
# TODO tile_grid params should be serialized on all_stats object
tile_grid = ek_stats.get_default_tile_grid()
req_bbox = mapproxy_grid.grid_bbox(bbox, mapproxy_srs.SRS(srs),
tile_grid.srs)
req_area = ek_stats.get_area_bbox(req_bbox)
# Compute estimate
size_per_km, method = ek_stats.query(
provider.export_provider_type.type_name,
'size',
'mean',
bbox,
srs,
grouping='provider_type',
gap_fill_thresh=0.1,
default_value=0)
method['size_per_km'] = size_per_km
return req_area * size_per_km, method
def test_unknown_srs(self):
try:
srs.SRS(2339)
except RuntimeError:
pass
else:
assert False, 'RuntimeError expected'
def get_tile_stats(parent,
tile_grid,
bbox,
create_if_absent=False,
cache_key=None):
"""
Intersects the bbox with the tile grid, returning all of the corresponding objects that hold
data samples for those tiles
:param parent: Parent dictionary
:param tile_grid: The tile grid defining per-tile statistics to generate
:param bbox: Query bounding box
:param create_if_absent: True if objects should be created if not-currently there
:param tid_cache: Optional cache storing results of bbox query on grid
:param cid: Optional id used to access/store results of bbox query on grid, tid_cache MUST be provided when used
:return: The list of objects intersecting the bbox, or an empty array
"""
run_bbox = mapproxy_grid.grid_bbox(bbox,
bbox_srs=mapproxy_srs.SRS(4326),
srs=tile_grid.srs)
affected_tiles = tile_grid.get_affected_level_tiles(
run_bbox, tile_grid.levels - 1) # Use highest res grid
tile_coords = []
for tile_coord in affected_tiles[2]:
tile_coords += [tile_coord]
tile_stats = list(
map(lambda t: get_tile_stat(parent, t, create_if_absent), tile_coords))
if create_if_absent:
return tile_stats # We won't have None entries
else:
return [x for x in tile_stats if x is not None]
def get_total_num_pixels(tile_grid, bbox, srs='4326', with_clipping=True):
"""
Determine the number of pixels in the tile_grid (across all levels) that are within the specified bounding box
:param tile_grid:
:param bbox: The bounding box defining the export region
:param srs: The SRS of the bounding box
:param with_clipping: When True tiles within the bbox will also have any pixels outside
the bbox excluded. False will use all pixels from tiles that intersect the bbox
:return: Total number of pixels
"""
grid_bbox = mapproxy_grid.grid_bbox(bbox, mapproxy_srs.SRS(srs),
tile_grid.srs)
# Determine all affected tiles across all provider levels
total_pixels = 0
px_per_tile = tile_grid.tile_size[0] * tile_grid.tile_size[1]
for lvl in range(0, tile_grid.levels):
result = tile_grid.get_affected_level_tiles(grid_bbox, int(lvl))
tile_coords = result[2]
if with_clipping:
# Determine number of pixels contained in the bbox ONLY
for tile_coord in tile_coords:
tile_bbox = tile_grid.tile_bbox(tile_coord, True)
i = get_bbox_intersect(grid_bbox, tile_bbox)
# Assume uniform spacing of pixels in the tile
tile_num_px = (get_area_bbox(i) /
get_area_bbox(tile_bbox)) * px_per_tile
total_pixels += tile_num_px
else:
num_tiles = result[1][0] * result[1][1] # xdim * ydim
total_pixels += px_per_tile * num_tiles
return total_pixels
def test_unknown_srs(self):
with pytest.raises(RuntimeError):
srs.SRS(2339)
def test_dummy_srs(self):
srs.SRS(1234)
def test_known_srs(self):
srs.SRS(4326)
def query(group_name,
field,
statistic_name,
bbox,
bbox_srs,
gap_fill_thresh=0.1,
default_value=None,
custom_stats=None,
grouping='provider_name'):
"""
Finds the highest resolution of the requested statistic:
1. Within group and bbox region (leveraging tile grid)
2. Within group
3. Global
4. Default value
:param group_name: Must match grouping semantics (e.g. grouping='provider_type' then group_name in wms, osm, ...)
:param field: The field to query (e.g. size, mpp, duration)
:param statistic_name: The name of the statistic (e.g. mean, ci_90, ..)
:param bbox: The bounding box defining the region to estimate
:param bbox_srs: The srs of bbox
:param gap_fill_thresh: If bbox has less that this % overlap with tile-level statistics then fill-gaps using group
level or global level size/sq.km, otherwise gap-fill using the mean of observed tile-level
stats
:param default_value: A default value to return if no statistics exist
:param custom_stats: Use custom statistics dictionary rather than cached (see get_statistics or compute_statistics)
:param grouping: iff custom_stats is None, defines which stat grouping to use (must match group_name semantics)
:return: (Highest resolution of statistic, object w/ metadata about how it was generated incl. resolution)
"""
if custom_stats is not None:
all_stats = custom_stats
else:
all_stats = get_statistics(grouping=grouping)
method = {
'stat': statistic_name,
}
def get_single_value(o):
fld = o.get(field)
if fld:
return fld.get(statistic_name)
def get_upper_ci_value(o):
fld = o.get(field)
if fld and fld.get(
statistic_name): # Can be missing (e.g. 1 data sample)
return fld[statistic_name][1] # Get the upper bound
get_value = get_upper_ci_value if statistic_name.startswith(
'ci_') else get_single_value
stat_value = None
if all_stats:
group_stats = all_stats.get(group_name)
if group_stats:
# TODO tile_grid params should be serialized on all_stats object
# We have some statistics specific to this group (e.g. osm, wms, etc)
tile_grid = get_default_tile_grid()
req_bbox = mapproxy_grid.grid_bbox(bbox,
mapproxy_srs.SRS(bbox_srs),
tile_grid.srs)
req_area = get_area_bbox(req_bbox)
affected_tiles = get_tile_stats(group_stats, tile_grid, req_bbox)
if affected_tiles and len(affected_tiles) > 0:
# We have some stats specific to this group, at tiles within the user-defined region
# We want to weight tile-specific statistics based on its % overlap with the bbox
# (e.g. tile1 accounts for 50% of bbox area, its stats should be weighted at 50%
total_weight = 0
values = []
stat_value = 0
for tile_stat in affected_tiles:
t_val = get_value(tile_stat)
if t_val is not None:
tile_coord = tile_stat['tile_coord']
inter = get_bbox_intersect(
req_bbox, tile_grid.tile_bbox(tile_coord, True))
weight = get_area_bbox(inter) / req_area
stat_value += weight * t_val
values += [t_val]
total_weight += weight
if total_weight > 1.0:
# Shouldn't happen since tile-grid is disjoint...
stat_value /= total_weight
elif total_weight < gap_fill_thresh:
# If the overlap was very minor than gap fill using the group-wide stats
stat_value += (1.0 - total_weight) * get_value(group_stats)
else:
# Otherwise, gap fill using the average from the tiles we did see
stat_value += (1.0 -
total_weight) * statistics.mean(values)
if total_weight > 0:
method['group'] = '{}_tiles'.format(group_name)
method['tiles'] = {
'count':
len(affected_tiles),
'total_weight':
100 * total_weight,
'gap_fill':
group_name
if total_weight < gap_fill_thresh else 'tile_mean'
}
else:
# It's possible that none of the tiles had the stat we were looking for in which case gap_fill
# is essentially the same as using the group-level statistic
method['group'] = group_name
else:
# No overlapping tiles, use group specific stats
method['group'] = group_name
stat_value = get_value(group_stats)
elif 'GLOBAL' in all_stats:
# No group-specific data, use statistics computed across all groups (i.e. every completed job)
method['group'] = 'GLOBAL'
stat_value = get_value(all_stats['GLOBAL'])
if stat_value is None:
# No statistics... use default
method['group'] = 'None'
stat_value = default_value
return stat_value, method
