def calc_buffered_bounds(format, bounds, meters_per_pixel_dim, layer_name,
geometry_type, buffer_cfg):
"""
Calculate the buffered bounds per format per layer based on config.
"""
if not buffer_cfg:
return bounds
format_buffer_cfg = buffer_cfg.get(format.extension)
if format_buffer_cfg is None:
return bounds
geometry_type = normalize_geometry_type(geometry_type)
per_layer_cfg = format_buffer_cfg.get('layer', {}).get(layer_name)
if per_layer_cfg is not None:
layer_geom_pixels = per_layer_cfg.get(geometry_type)
if layer_geom_pixels is not None:
assert isinstance(layer_geom_pixels, Number)
result = bounds_buffer(bounds,
meters_per_pixel_dim * layer_geom_pixels)
return result
by_geometry_pixels = format_buffer_cfg.get('geometry',
{}).get(geometry_type)
if by_geometry_pixels is not None:
assert isinstance(by_geometry_pixels, Number)
result = bounds_buffer(bounds,
meters_per_pixel_dim * by_geometry_pixels)
return result
return bounds
def _cut_coord(feature_layers, unpadded_bounds, meters_per_pixel_dim,
buffer_cfg):
cut_feature_layers = []
for feature_layer in feature_layers:
features = feature_layer['features']
padded_bounds_fn = create_query_bounds_pad_fn(buffer_cfg,
feature_layer['name'])
padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim)
cut_features = []
for feature in features:
shape, props, feature_id = feature
geom_type_bounds = padded_bounds[normalize_geometry_type(
shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
props_copy = props.copy()
cut_feature = shape, props_copy, feature_id
cut_features.append(cut_feature)
cut_feature_layer = dict(
name=feature_layer['name'],
layer_datum=feature_layer['layer_datum'],
features=cut_features,
padded_bounds=padded_bounds,
)
cut_feature_layers.append(cut_feature_layer)
return cut_feature_layers
def calc_buffered_bounds(
format, bounds, meters_per_pixel_dim, layer_name, geometry_type,
buffer_cfg):
"""
Calculate the buffered bounds per format per layer based on config.
"""
if not buffer_cfg:
return bounds
format_buffer_cfg = buffer_cfg.get(format.extension)
if format_buffer_cfg is None:
return bounds
geometry_type = normalize_geometry_type(geometry_type)
per_layer_cfg = format_buffer_cfg.get('layer', {}).get(layer_name)
if per_layer_cfg is not None:
layer_geom_pixels = per_layer_cfg.get(geometry_type)
if layer_geom_pixels is not None:
assert isinstance(layer_geom_pixels, Number)
result = bounds_buffer(
bounds, meters_per_pixel_dim * layer_geom_pixels)
return result
by_geometry_pixels = format_buffer_cfg.get('geometry', {}).get(
geometry_type)
if by_geometry_pixels is not None:
assert isinstance(by_geometry_pixels, Number)
result = bounds_buffer(
bounds, meters_per_pixel_dim * by_geometry_pixels)
return result
return bounds
def _cut_coord(
feature_layers, unpadded_bounds, meters_per_pixel_dim, buffer_cfg):
cut_feature_layers = []
for feature_layer in feature_layers:
features = feature_layer['features']
padded_bounds_fn = create_query_bounds_pad_fn(
buffer_cfg, feature_layer['name'])
padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim)
cut_features = []
for feature in features:
shape, props, feature_id = feature
geom_type_bounds = padded_bounds[
normalize_geometry_type(shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
props_copy = props.copy()
cut_feature = shape, props_copy, feature_id
cut_features.append(cut_feature)
cut_feature_layer = dict(
name=feature_layer['name'],
layer_datum=feature_layer['layer_datum'],
features=cut_features,
padded_bounds=padded_bounds,
)
cut_feature_layers.append(cut_feature_layer)
return cut_feature_layers
def _preprocess_data(feature_layers):
preproc_feature_layers = []
extra_data = dict(size={})
for feature_layer in feature_layers:
layer_datum = feature_layer['layer_datum']
geometry_types = layer_datum['geometry_types']
padded_bounds = feature_layer['padded_bounds']
features = []
features_size = 0
for row in feature_layer['features']:
wkb = row.pop('__geometry__')
shape = loads(wkb)
if shape.is_empty:
continue
if not shape.is_valid:
continue
if geometry_types is not None:
if shape.type not in geometry_types:
continue
# since a bounding box intersection is used, we
# perform a more accurate check here to filter out
# any extra features
# the formatter specific transformations will take
# care of any additional filtering
geom_type_bounds = padded_bounds[normalize_geometry_type(
shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
feature_id = row.pop('__id__')
props = dict()
feature_size = getsizeof(feature_id) + len(wkb)
for k, v in row.iteritems():
if k == 'mz_properties':
for output_key, output_val in v.items():
if output_val is not None:
# all other tags are utf8 encoded, encode
# these the same way to be consistent
if isinstance(output_key, unicode):
output_key = output_key.encode('utf-8')
if isinstance(output_val, unicode):
output_val = output_val.encode('utf-8')
props[output_key] = output_val
feature_size += len(output_key) + \
_sizeof(output_val)
else:
props[k] = v
feature_size += len(k) + _sizeof(v)
feature = shape, props, feature_id
features.append(feature)
features_size += feature_size
extra_data['size'][layer_datum['name']] = features_size
preproc_feature_layer = dict(
name=layer_datum['name'],
layer_datum=layer_datum,
features=features,
padded_bounds=padded_bounds,
)
preproc_feature_layers.append(preproc_feature_layer)
return preproc_feature_layers, extra_data
def process_coord_no_format(feature_layers, nominal_zoom, unpadded_bounds,
post_process_data, output_calc_mapping):
extra_data = dict(size={})
processed_feature_layers = []
# filter, and then transform each layer as necessary
for feature_layer in feature_layers:
layer_datum = feature_layer['layer_datum']
layer_name = layer_datum['name']
geometry_types = layer_datum['geometry_types']
padded_bounds = feature_layer['padded_bounds']
transform_fn_names = layer_datum['transform_fn_names']
if transform_fn_names:
transform_fns = resolve_transform_fns(transform_fn_names)
layer_transform_fn = make_transform_fn(transform_fns)
else:
layer_transform_fn = None
layer_output_calc = output_calc_mapping.get(layer_name)
assert layer_output_calc, 'output_calc_mapping missing layer: %s' % \
layer_name
features = []
features_size = 0
for row in feature_layer['features']:
wkb = row.pop('__geometry__')
shape = loads(wkb)
if shape.is_empty:
continue
if not shape.is_valid:
continue
if geometry_types is not None:
if shape.type not in geometry_types:
continue
# since a bounding box intersection is used, we
# perform a more accurate check here to filter out
# any extra features
# the formatter specific transformations will take
# care of any additional filtering
geom_type_bounds = padded_bounds[normalize_geometry_type(
shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
feature_id = row.pop('__id__')
props = {}
feature_size = getsizeof(feature_id) + len(wkb)
label = row.pop('__label__', None)
if label:
# TODO probably formalize as part of the feature
props['mz_label_placement'] = label
feature_size += len('__label__') + _sizeof(label)
# first ensure that all strings are utf-8 encoded
# it would be better for it all to be unicode instead, but
# some downstream transforms / formatters might be
# expecting utf-8
row = utils.encode_utf8(row)
query_props = row.pop('__properties__')
feature_size += len('__properties__') + _sizeof(query_props)
# TODO:
# Right now this is hacked to map the particular source,
# which all relevant queries include, back to another
# metadata property
# The reason for this is to support the same yaml syntax
# for python output calculation and sql min zoom function
# generation.
# This is done in python here to avoid having to update
# all the queries in the jinja file with redundant
# information.
meta = meta_for_properties(query_props)
# set the "tags" key
# some transforms expect to be able to read it from this location
# longer term, we might want to separate the notion of
# "input" and "output" properties as a part of the feature
props['tags'] = query_props
output_props = layer_output_calc(shape, query_props, feature_id,
meta)
assert output_props, 'No output calc rule matched'
# a feature can belong to more than one layer
# this check ensures that it only appears in the
# layers it should
# NOTE: the min zoom can be calculated by the yaml, so
# this check must happen after that
min_zoom = output_props.get('min_zoom')
assert min_zoom is not None, \
'Missing min_zoom in layer %s' % layer_name
# TODO would be better if 16 wasn't hard coded here
if nominal_zoom < 16 and min_zoom >= nominal_zoom + 1:
continue
for k, v in output_props.items():
if v is not None:
props[k] = v
if layer_transform_fn:
shape, props, feature_id = layer_transform_fn(
shape, props, feature_id, nominal_zoom)
feature = shape, props, feature_id
features.append(feature)
features_size += feature_size
extra_data['size'][layer_datum['name']] = features_size
sort_fn_name = layer_datum['sort_fn_name']
if sort_fn_name:
sort_fn = resolve(sort_fn_name)
features = sort_fn(features, nominal_zoom)
feature_layer = dict(
name=layer_name,
features=features,
layer_datum=layer_datum,
padded_bounds=padded_bounds,
)
processed_feature_layers.append(feature_layer)
# post-process data here, before it gets formatted
processed_feature_layers = _postprocess_data(processed_feature_layers,
post_process_data,
nominal_zoom, unpadded_bounds)
return processed_feature_layers, extra_data
def _preprocess_data(feature_layers):
preproc_feature_layers = []
extra_data = dict(size={})
for feature_layer in feature_layers:
layer_datum = feature_layer['layer_datum']
geometry_types = layer_datum['geometry_types']
padded_bounds = feature_layer['padded_bounds']
features = []
features_size = 0
for row in feature_layer['features']:
wkb = row.pop('__geometry__')
shape = loads(wkb)
if shape.is_empty:
continue
if not shape.is_valid:
continue
if geometry_types is not None:
if shape.type not in geometry_types:
continue
# since a bounding box intersection is used, we
# perform a more accurate check here to filter out
# any extra features
# the formatter specific transformations will take
# care of any additional filtering
geom_type_bounds = padded_bounds[
normalize_geometry_type(shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
feature_id = row.pop('__id__')
props = dict()
feature_size = getsizeof(feature_id) + len(wkb)
for k, v in row.iteritems():
if k == 'mz_properties':
for output_key, output_val in v.items():
if output_val is not None:
# all other tags are utf8 encoded, encode
# these the same way to be consistent
if isinstance(output_key, unicode):
output_key = output_key.encode('utf-8')
if isinstance(output_val, unicode):
output_val = output_val.encode('utf-8')
props[output_key] = output_val
feature_size += len(output_key) + \
_sizeof(output_val)
else:
props[k] = v
feature_size += len(k) + _sizeof(v)
feature = shape, props, feature_id
features.append(feature)
features_size += feature_size
extra_data['size'][layer_datum['name']] = features_size
preproc_feature_layer = dict(
name=layer_datum['name'],
layer_datum=layer_datum,
features=features,
padded_bounds=padded_bounds,
)
preproc_feature_layers.append(preproc_feature_layer)
return preproc_feature_layers, extra_data
def process_coord_no_format(
feature_layers, nominal_zoom, unpadded_bounds, post_process_data,
output_calc_mapping, log_fn=None):
extra_data = dict(size={})
processed_feature_layers = []
# filter, and then transform each layer as necessary
for feature_layer in feature_layers:
layer_datum = feature_layer['layer_datum']
layer_name = layer_datum['name']
geometry_types = layer_datum['geometry_types']
padded_bounds = feature_layer['padded_bounds']
transform_fn_names = layer_datum['transform_fn_names']
if transform_fn_names:
transform_fns = resolve_transform_fns(transform_fn_names)
layer_transform_fn = make_transform_fn(transform_fns)
else:
layer_transform_fn = None
layer_output_calc = output_calc_mapping.get(layer_name)
assert layer_output_calc, 'output_calc_mapping missing layer: %s' % \
layer_name
features = []
features_size = 0
for row in feature_layer['features']:
wkb = row.pop('__geometry__')
shape = loads(wkb)
if shape.is_empty:
continue
if not shape.is_valid:
continue
if geometry_types is not None:
if shape.type not in geometry_types:
continue
# since a bounding box intersection is used, we
# perform a more accurate check here to filter out
# any extra features
# the formatter specific transformations will take
# care of any additional filtering
geom_type_bounds = padded_bounds[
normalize_geometry_type(shape.type)]
shape_padded_bounds = geometry.box(*geom_type_bounds)
if not shape_padded_bounds.intersects(shape):
continue
feature_id = row.pop('__id__')
props = {}
feature_size = getsizeof(feature_id) + len(wkb)
label = row.pop('__label__', None)
if label:
# TODO probably formalize as part of the feature
props['mz_label_placement'] = label
feature_size += len('__label__') + _sizeof(label)
# first ensure that all strings are utf-8 encoded
# it would be better for it all to be unicode instead, but
# some downstream transforms / formatters might be
# expecting utf-8
row = utils.encode_utf8(row)
query_props = row.pop('__properties__')
feature_size += len('__properties__') + _sizeof(query_props)
# TODO:
# Right now this is hacked to map the particular source,
# which all relevant queries include, back to another
# metadata property
# The reason for this is to support the same yaml syntax
# for python output calculation and sql min zoom function
# generation.
# This is done in python here to avoid having to update
# all the queries in the jinja file with redundant
# information.
meta = meta_for_properties(query_props)
# set the "tags" key
# some transforms expect to be able to read it from this location
# longer term, we might want to separate the notion of
# "input" and "output" properties as a part of the feature
props['tags'] = query_props
output_props = layer_output_calc(
shape, query_props, feature_id, meta)
assert output_props, 'No output calc rule matched'
# a feature can belong to more than one layer
# this check ensures that it only appears in the
# layers it should
# NOTE: the min zoom can be calculated by the yaml, so
# this check must happen after that
min_zoom = output_props.get('min_zoom')
assert min_zoom is not None, \
'Missing min_zoom in layer %s' % layer_name
# TODO would be better if 16 wasn't hard coded here
if nominal_zoom < 16 and min_zoom >= nominal_zoom + 1:
continue
for k, v in output_props.items():
if v is not None:
props[k] = v
if layer_transform_fn:
shape, props, feature_id = layer_transform_fn(
shape, props, feature_id, nominal_zoom)
feature = shape, props, feature_id
features.append(feature)
features_size += feature_size
extra_data['size'][layer_datum['name']] = features_size
sort_fn_name = layer_datum['sort_fn_name']
if sort_fn_name:
sort_fn = resolve(sort_fn_name)
features = sort_fn(features, nominal_zoom)
feature_layer = dict(
name=layer_name,
features=features,
layer_datum=layer_datum,
padded_bounds=padded_bounds,
)
processed_feature_layers.append(feature_layer)
# post-process data here, before it gets formatted
processed_feature_layers = _postprocess_data(
processed_feature_layers, post_process_data, nominal_zoom,
unpadded_bounds, log_fn)
return processed_feature_layers, extra_data
