def calc_buffered_bounds(format, bounds, meters_per_pixel_dim, layer_name, geometry_type, buffer_cfg): """ Calculate the buffered bounds per format per layer based on config. """ if not buffer_cfg: return bounds format_buffer_cfg = buffer_cfg.get(format.extension) if format_buffer_cfg is None: return bounds geometry_type = normalize_geometry_type(geometry_type) per_layer_cfg = format_buffer_cfg.get('layer', {}).get(layer_name) if per_layer_cfg is not None: layer_geom_pixels = per_layer_cfg.get(geometry_type) if layer_geom_pixels is not None: assert isinstance(layer_geom_pixels, Number) result = bounds_buffer(bounds, meters_per_pixel_dim * layer_geom_pixels) return result by_geometry_pixels = format_buffer_cfg.get('geometry', {}).get(geometry_type) if by_geometry_pixels is not None: assert isinstance(by_geometry_pixels, Number) result = bounds_buffer(bounds, meters_per_pixel_dim * by_geometry_pixels) return result return bounds
def _cut_coord(feature_layers, unpadded_bounds, meters_per_pixel_dim, buffer_cfg): cut_feature_layers = [] for feature_layer in feature_layers: features = feature_layer['features'] padded_bounds_fn = create_query_bounds_pad_fn(buffer_cfg, feature_layer['name']) padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim) cut_features = [] for feature in features: shape, props, feature_id = feature geom_type_bounds = padded_bounds[normalize_geometry_type( shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue props_copy = props.copy() cut_feature = shape, props_copy, feature_id cut_features.append(cut_feature) cut_feature_layer = dict( name=feature_layer['name'], layer_datum=feature_layer['layer_datum'], features=cut_features, padded_bounds=padded_bounds, ) cut_feature_layers.append(cut_feature_layer) return cut_feature_layers
def calc_buffered_bounds( format, bounds, meters_per_pixel_dim, layer_name, geometry_type, buffer_cfg): """ Calculate the buffered bounds per format per layer based on config. """ if not buffer_cfg: return bounds format_buffer_cfg = buffer_cfg.get(format.extension) if format_buffer_cfg is None: return bounds geometry_type = normalize_geometry_type(geometry_type) per_layer_cfg = format_buffer_cfg.get('layer', {}).get(layer_name) if per_layer_cfg is not None: layer_geom_pixels = per_layer_cfg.get(geometry_type) if layer_geom_pixels is not None: assert isinstance(layer_geom_pixels, Number) result = bounds_buffer( bounds, meters_per_pixel_dim * layer_geom_pixels) return result by_geometry_pixels = format_buffer_cfg.get('geometry', {}).get( geometry_type) if by_geometry_pixels is not None: assert isinstance(by_geometry_pixels, Number) result = bounds_buffer( bounds, meters_per_pixel_dim * by_geometry_pixels) return result return bounds
def _cut_coord( feature_layers, unpadded_bounds, meters_per_pixel_dim, buffer_cfg): cut_feature_layers = [] for feature_layer in feature_layers: features = feature_layer['features'] padded_bounds_fn = create_query_bounds_pad_fn( buffer_cfg, feature_layer['name']) padded_bounds = padded_bounds_fn(unpadded_bounds, meters_per_pixel_dim) cut_features = [] for feature in features: shape, props, feature_id = feature geom_type_bounds = padded_bounds[ normalize_geometry_type(shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue props_copy = props.copy() cut_feature = shape, props_copy, feature_id cut_features.append(cut_feature) cut_feature_layer = dict( name=feature_layer['name'], layer_datum=feature_layer['layer_datum'], features=cut_features, padded_bounds=padded_bounds, ) cut_feature_layers.append(cut_feature_layer) return cut_feature_layers
def _preprocess_data(feature_layers): preproc_feature_layers = [] extra_data = dict(size={}) for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] geometry_types = layer_datum['geometry_types'] padded_bounds = feature_layer['padded_bounds'] features = [] features_size = 0 for row in feature_layer['features']: wkb = row.pop('__geometry__') shape = loads(wkb) if shape.is_empty: continue if not shape.is_valid: continue if geometry_types is not None: if shape.type not in geometry_types: continue # since a bounding box intersection is used, we # perform a more accurate check here to filter out # any extra features # the formatter specific transformations will take # care of any additional filtering geom_type_bounds = padded_bounds[normalize_geometry_type( shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue feature_id = row.pop('__id__') props = dict() feature_size = getsizeof(feature_id) + len(wkb) for k, v in row.iteritems(): if k == 'mz_properties': for output_key, output_val in v.items(): if output_val is not None: # all other tags are utf8 encoded, encode # these the same way to be consistent if isinstance(output_key, unicode): output_key = output_key.encode('utf-8') if isinstance(output_val, unicode): output_val = output_val.encode('utf-8') props[output_key] = output_val feature_size += len(output_key) + \ _sizeof(output_val) else: props[k] = v feature_size += len(k) + _sizeof(v) feature = shape, props, feature_id features.append(feature) features_size += feature_size extra_data['size'][layer_datum['name']] = features_size preproc_feature_layer = dict( name=layer_datum['name'], layer_datum=layer_datum, features=features, padded_bounds=padded_bounds, ) preproc_feature_layers.append(preproc_feature_layer) return preproc_feature_layers, extra_data
def process_coord_no_format(feature_layers, nominal_zoom, unpadded_bounds, post_process_data, output_calc_mapping): extra_data = dict(size={}) processed_feature_layers = [] # filter, and then transform each layer as necessary for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] layer_name = layer_datum['name'] geometry_types = layer_datum['geometry_types'] padded_bounds = feature_layer['padded_bounds'] transform_fn_names = layer_datum['transform_fn_names'] if transform_fn_names: transform_fns = resolve_transform_fns(transform_fn_names) layer_transform_fn = make_transform_fn(transform_fns) else: layer_transform_fn = None layer_output_calc = output_calc_mapping.get(layer_name) assert layer_output_calc, 'output_calc_mapping missing layer: %s' % \ layer_name features = [] features_size = 0 for row in feature_layer['features']: wkb = row.pop('__geometry__') shape = loads(wkb) if shape.is_empty: continue if not shape.is_valid: continue if geometry_types is not None: if shape.type not in geometry_types: continue # since a bounding box intersection is used, we # perform a more accurate check here to filter out # any extra features # the formatter specific transformations will take # care of any additional filtering geom_type_bounds = padded_bounds[normalize_geometry_type( shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue feature_id = row.pop('__id__') props = {} feature_size = getsizeof(feature_id) + len(wkb) label = row.pop('__label__', None) if label: # TODO probably formalize as part of the feature props['mz_label_placement'] = label feature_size += len('__label__') + _sizeof(label) # first ensure that all strings are utf-8 encoded # it would be better for it all to be unicode instead, but # some downstream transforms / formatters might be # expecting utf-8 row = utils.encode_utf8(row) query_props = row.pop('__properties__') feature_size += len('__properties__') + _sizeof(query_props) # TODO: # Right now this is hacked to map the particular source, # which all relevant queries include, back to another # metadata property # The reason for this is to support the same yaml syntax # for python output calculation and sql min zoom function # generation. # This is done in python here to avoid having to update # all the queries in the jinja file with redundant # information. meta = meta_for_properties(query_props) # set the "tags" key # some transforms expect to be able to read it from this location # longer term, we might want to separate the notion of # "input" and "output" properties as a part of the feature props['tags'] = query_props output_props = layer_output_calc(shape, query_props, feature_id, meta) assert output_props, 'No output calc rule matched' # a feature can belong to more than one layer # this check ensures that it only appears in the # layers it should # NOTE: the min zoom can be calculated by the yaml, so # this check must happen after that min_zoom = output_props.get('min_zoom') assert min_zoom is not None, \ 'Missing min_zoom in layer %s' % layer_name # TODO would be better if 16 wasn't hard coded here if nominal_zoom < 16 and min_zoom >= nominal_zoom + 1: continue for k, v in output_props.items(): if v is not None: props[k] = v if layer_transform_fn: shape, props, feature_id = layer_transform_fn( shape, props, feature_id, nominal_zoom) feature = shape, props, feature_id features.append(feature) features_size += feature_size extra_data['size'][layer_datum['name']] = features_size sort_fn_name = layer_datum['sort_fn_name'] if sort_fn_name: sort_fn = resolve(sort_fn_name) features = sort_fn(features, nominal_zoom) feature_layer = dict( name=layer_name, features=features, layer_datum=layer_datum, padded_bounds=padded_bounds, ) processed_feature_layers.append(feature_layer) # post-process data here, before it gets formatted processed_feature_layers = _postprocess_data(processed_feature_layers, post_process_data, nominal_zoom, unpadded_bounds) return processed_feature_layers, extra_data
def _preprocess_data(feature_layers): preproc_feature_layers = [] extra_data = dict(size={}) for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] geometry_types = layer_datum['geometry_types'] padded_bounds = feature_layer['padded_bounds'] features = [] features_size = 0 for row in feature_layer['features']: wkb = row.pop('__geometry__') shape = loads(wkb) if shape.is_empty: continue if not shape.is_valid: continue if geometry_types is not None: if shape.type not in geometry_types: continue # since a bounding box intersection is used, we # perform a more accurate check here to filter out # any extra features # the formatter specific transformations will take # care of any additional filtering geom_type_bounds = padded_bounds[ normalize_geometry_type(shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue feature_id = row.pop('__id__') props = dict() feature_size = getsizeof(feature_id) + len(wkb) for k, v in row.iteritems(): if k == 'mz_properties': for output_key, output_val in v.items(): if output_val is not None: # all other tags are utf8 encoded, encode # these the same way to be consistent if isinstance(output_key, unicode): output_key = output_key.encode('utf-8') if isinstance(output_val, unicode): output_val = output_val.encode('utf-8') props[output_key] = output_val feature_size += len(output_key) + \ _sizeof(output_val) else: props[k] = v feature_size += len(k) + _sizeof(v) feature = shape, props, feature_id features.append(feature) features_size += feature_size extra_data['size'][layer_datum['name']] = features_size preproc_feature_layer = dict( name=layer_datum['name'], layer_datum=layer_datum, features=features, padded_bounds=padded_bounds, ) preproc_feature_layers.append(preproc_feature_layer) return preproc_feature_layers, extra_data
def process_coord_no_format( feature_layers, nominal_zoom, unpadded_bounds, post_process_data, output_calc_mapping, log_fn=None): extra_data = dict(size={}) processed_feature_layers = [] # filter, and then transform each layer as necessary for feature_layer in feature_layers: layer_datum = feature_layer['layer_datum'] layer_name = layer_datum['name'] geometry_types = layer_datum['geometry_types'] padded_bounds = feature_layer['padded_bounds'] transform_fn_names = layer_datum['transform_fn_names'] if transform_fn_names: transform_fns = resolve_transform_fns(transform_fn_names) layer_transform_fn = make_transform_fn(transform_fns) else: layer_transform_fn = None layer_output_calc = output_calc_mapping.get(layer_name) assert layer_output_calc, 'output_calc_mapping missing layer: %s' % \ layer_name features = [] features_size = 0 for row in feature_layer['features']: wkb = row.pop('__geometry__') shape = loads(wkb) if shape.is_empty: continue if not shape.is_valid: continue if geometry_types is not None: if shape.type not in geometry_types: continue # since a bounding box intersection is used, we # perform a more accurate check here to filter out # any extra features # the formatter specific transformations will take # care of any additional filtering geom_type_bounds = padded_bounds[ normalize_geometry_type(shape.type)] shape_padded_bounds = geometry.box(*geom_type_bounds) if not shape_padded_bounds.intersects(shape): continue feature_id = row.pop('__id__') props = {} feature_size = getsizeof(feature_id) + len(wkb) label = row.pop('__label__', None) if label: # TODO probably formalize as part of the feature props['mz_label_placement'] = label feature_size += len('__label__') + _sizeof(label) # first ensure that all strings are utf-8 encoded # it would be better for it all to be unicode instead, but # some downstream transforms / formatters might be # expecting utf-8 row = utils.encode_utf8(row) query_props = row.pop('__properties__') feature_size += len('__properties__') + _sizeof(query_props) # TODO: # Right now this is hacked to map the particular source, # which all relevant queries include, back to another # metadata property # The reason for this is to support the same yaml syntax # for python output calculation and sql min zoom function # generation. # This is done in python here to avoid having to update # all the queries in the jinja file with redundant # information. meta = meta_for_properties(query_props) # set the "tags" key # some transforms expect to be able to read it from this location # longer term, we might want to separate the notion of # "input" and "output" properties as a part of the feature props['tags'] = query_props output_props = layer_output_calc( shape, query_props, feature_id, meta) assert output_props, 'No output calc rule matched' # a feature can belong to more than one layer # this check ensures that it only appears in the # layers it should # NOTE: the min zoom can be calculated by the yaml, so # this check must happen after that min_zoom = output_props.get('min_zoom') assert min_zoom is not None, \ 'Missing min_zoom in layer %s' % layer_name # TODO would be better if 16 wasn't hard coded here if nominal_zoom < 16 and min_zoom >= nominal_zoom + 1: continue for k, v in output_props.items(): if v is not None: props[k] = v if layer_transform_fn: shape, props, feature_id = layer_transform_fn( shape, props, feature_id, nominal_zoom) feature = shape, props, feature_id features.append(feature) features_size += feature_size extra_data['size'][layer_datum['name']] = features_size sort_fn_name = layer_datum['sort_fn_name'] if sort_fn_name: sort_fn = resolve(sort_fn_name) features = sort_fn(features, nominal_zoom) feature_layer = dict( name=layer_name, features=features, layer_datum=layer_datum, padded_bounds=padded_bounds, ) processed_feature_layers.append(feature_layer) # post-process data here, before it gets formatted processed_feature_layers = _postprocess_data( processed_feature_layers, post_process_data, nominal_zoom, unpadded_bounds, log_fn) return processed_feature_layers, extra_data