def parse_neighbourhood_meta_csv(csv_line_generator, placetype): reader = csv.reader(csv_line_generator) it = iter(reader) header = it.next() lbl_lat_idx = header.index('lbl_latitude') lbl_lng_idx = header.index('lbl_longitude') name_idx = header.index('name') wof_id_idx = header.index('id') hash_idx = header.index('file_hash') superseded_by_idx = header.index('superseded_by') min_row_length = (max( lbl_lat_idx, lbl_lng_idx, name_idx, wof_id_idx, hash_idx, superseded_by_idx) + 1) for row in it: if len(row) < min_row_length: continue superseded_by = row[superseded_by_idx] if superseded_by: continue wof_id_str = row[wof_id_idx] if not wof_id_str: continue try: wof_id = int(wof_id_str) except ValueError: continue name = row[name_idx] if not name: continue lat_str = row[lbl_lat_idx] lng_str = row[lbl_lng_idx] try: lat = float(lat_str) lng = float(lng_str) except ValueError: continue file_hash = row[hash_idx] label_x, label_y = reproject_lnglat_to_mercator(lng, lat) label_position = shapely.geometry.Point(label_x, label_y) neighbourhood_meta = NeighbourhoodMeta( wof_id, placetype, name, file_hash, label_position) yield neighbourhood_meta
def create_neighbourhood_from_json(json_data, neighbourhood_meta): def failure(reason): return NeighbourhoodFailure(neighbourhood_meta.wof_id, reason, json.dumps(json_data)) if not isinstance(json_data, dict): return failure('Unexpected json') props = json_data.get('properties') if props is None or not isinstance(props, dict): return failure('Missing properties') superseded_by = props.get('wof:superseded_by') # these often show up as empty lists, so we do a truthy test # instead of expicitly checking for None if superseded_by: return NeighbourhoodFailure(neighbourhood_meta.wof_id, 'superseded_by: %s' % superseded_by, json.dumps(json_data), superseded=True) geometry = json_data.get('geometry') if geometry is None: return failure('Missing geometry') try: shape_lnglat = shapely.geometry.shape(geometry) except Exception: return failure('Unexpected geometry') shape_mercator = shapely.ops.transform(reproject_lnglat_to_mercator, shape_lnglat) # ignore any features that are marked as funky is_funky = props.get('mz:is_funky') if is_funky is not None: try: is_funky = int(is_funky) except ValueError: return failure('Unexpected mz:is_funky value %s' % is_funky) if is_funky != 0: return NeighbourhoodFailure(neighbourhood_meta.wof_id, 'mz:is_funky value is not 0: %s' % is_funky, json.dumps(json_data), funky=True) wof_id = props.get('wof:id') if wof_id is None: return failure('Missing wof:id') try: wof_id = int(wof_id) except ValueError: return failure('wof_id is not an int: %s' % wof_id) name = props.get('wof:name') if name is None: return failure('Missing name') n_photos = props.get('misc:photo_sum') if n_photos is not None: try: n_photos = int(n_photos) except ValueError: return failure('misc:photo_sum is not an int: %s' % n_photos) label_lat = props.get('lbl:latitude') label_lng = props.get('lbl:longitude') if label_lat is None or label_lng is None: # first, try to fall back to geom:* when lbl:* is missing. we'd prefer # to have lbl:*, but it's better to have _something_ than nothing. label_lat = props.get('geom:latitude') label_lng = props.get('geom:longitude') if label_lat is None or label_lng is None: return failure('Missing lbl:latitude or lbl:longitude and ' + 'geom:latitude or geom:longitude') try: label_lat = float(label_lat) label_lng = float(label_lng) except ValueError: return failure('lbl:latitude or lbl:longitude not float') label_merc_x, label_merc_y = reproject_lnglat_to_mercator( label_lng, label_lat) label_position = shapely.geometry.Point(label_merc_x, label_merc_y) placetype = props.get('wof:placetype') if placetype is None: return failure('Missing wof:placetype') default_min_zoom = 15 default_max_zoom = 16 min_zoom = props.get('mz:min_zoom') if min_zoom is None: min_zoom = default_min_zoom else: try: min_zoom = float(min_zoom) except ValueError: return failure('mz:min_zoom not float: %s' % min_zoom) max_zoom = props.get('mz:max_zoom') if max_zoom is None: max_zoom = default_max_zoom else: try: max_zoom = float(max_zoom) except ValueError: return failure('mz:max_zoom not float: %s' % max_zoom) is_landuse_aoi = props.get('mz:is_landuse_aoi') if is_landuse_aoi is not None: try: is_landuse_aoi = int(is_landuse_aoi) except ValueError: return failure('is_landuse_aoi not int: %s' % is_landuse_aoi) is_landuse_aoi = is_landuse_aoi != 0 if shape_mercator.type in ('Polygon', 'MultiPolygon'): area = int(shape_mercator.area) else: area = None # for the purposes of display, we only care about the times when something # should first start to be shown, and the time when it should stop # showing. edtf_inception = _normalize_edtf(props.get('edtf:inception')) edtf_cessation = _normalize_edtf(props.get('edtf:cessation')) edtf_deprecated = _normalize_edtf(props.get('edtf:deprecated')) # check that the dates are valid first to return back a better error inception_earliest = edtf_inception.lower_fuzzy() cessation_latest = edtf_cessation.upper_fuzzy() deprecated_latest = edtf_deprecated.upper_fuzzy() if inception_earliest is None: return failure('invalid edtf:inception: %s' % props.get('edtf:inception')) if cessation_latest is None: return failure('invalid edtf:cessation: %s' % props.get('edtf:cessation')) if deprecated_latest is None: return failure('invalid edtf:deprecated: %s' % props.get('edtf:deprecated')) # the 'edtf:inception' property gives us approximately the former and we # take the earliest date it could mean. the 'edtf:cessation' and # 'edtf:deprecated' would both stop the item showing, so we take the # earliest of each's latest possible date. inception = inception_earliest cessation = min(cessation_latest, deprecated_latest) # grab any names in other languages lang_suffix_size = len('_preferred') l10n_names = {} for k, v in props.iteritems(): if not v: continue if not k.startswith('name:') or not k.endswith('_preferred'): continue if isinstance(v, list): v = v[0] lang = k[:-lang_suffix_size] l10n_names[lang] = v if not l10n_names: l10n_names = None wikidata = None # get wikidata ID concordance, if there is one concordances = props.get('wof:concordances') if concordances: wikidata = concordances.get('wd:id') neighbourhood = Neighbourhood(wof_id, placetype, name, neighbourhood_meta.hash, label_position, shape_mercator, n_photos, area, min_zoom, max_zoom, is_landuse_aoi, inception, cessation, l10n_names, wikidata) return neighbourhood
def point(id, position, tags): x, y = reproject_lnglat_to_mercator(*position) return Feature(id, Point(x, y), _tags_to_utf8(tags))
def create_neighbourhood_from_json(json_data, neighbourhood_meta): def failure(reason): return NeighbourhoodFailure( neighbourhood_meta.wof_id, reason, json.dumps(json_data)) if not isinstance(json_data, dict): return failure('Unexpected json') props = json_data.get('properties') if props is None or not isinstance(props, dict): return failure('Missing properties') superseded_by = props.get('wof:superseded_by') # these often show up as empty lists, so we do a truthy test # instead of expicitly checking for None if superseded_by: return NeighbourhoodFailure( neighbourhood_meta.wof_id, 'superseded_by: %s' % superseded_by, json.dumps(json_data), superseded=True) geometry = json_data.get('geometry') if geometry is None: return failure('Missing geometry') try: shape_lnglat = shapely.geometry.shape(geometry) except Exception: return failure('Unexpected geometry') shape_mercator = shapely.ops.transform( reproject_lnglat_to_mercator, shape_lnglat) # ignore any features that are marked as funky is_funky = props.get('mz:is_funky') if is_funky is not None: try: is_funky = int(is_funky) except ValueError: return failure('Unexpected mz:is_funky value %s' % is_funky) if is_funky != 0: return NeighbourhoodFailure( neighbourhood_meta.wof_id, 'mz:is_funky value is not 0: %s' % is_funky, json.dumps(json_data), funky=True) wof_id = props.get('wof:id') if wof_id is None: return failure('Missing wof:id') try: wof_id = int(wof_id) except ValueError: return failure('wof_id is not an int: %s' % wof_id) name = props.get('wof:name') if name is None: return failure('Missing name') n_photos = props.get('misc:photo_sum') if n_photos is not None: try: n_photos = int(n_photos) except ValueError: return failure('misc:photo_sum is not an int: %s' % n_photos) label_lat = props.get('lbl:latitude') label_lng = props.get('lbl:longitude') if label_lat is None or label_lng is None: # first, try to fall back to geom:* when lbl:* is missing. we'd prefer # to have lbl:*, but it's better to have _something_ than nothing. label_lat = props.get('geom:latitude') label_lng = props.get('geom:longitude') if label_lat is None or label_lng is None: return failure('Missing lbl:latitude or lbl:longitude and ' + 'geom:latitude or geom:longitude') try: label_lat = float(label_lat) label_lng = float(label_lng) except ValueError: return failure('lbl:latitude or lbl:longitude not float') label_merc_x, label_merc_y = reproject_lnglat_to_mercator( label_lng, label_lat) label_position = shapely.geometry.Point(label_merc_x, label_merc_y) placetype = props.get('wof:placetype') if placetype is None: return failure('Missing wof:placetype') default_min_zoom = 15 default_max_zoom = 16 min_zoom = props.get('mz:min_zoom') if min_zoom is None: min_zoom = default_min_zoom else: try: min_zoom = float(min_zoom) except ValueError: return failure('mz:min_zoom not float: %s' % min_zoom) max_zoom = props.get('mz:max_zoom') if max_zoom is None: max_zoom = default_max_zoom else: try: max_zoom = float(max_zoom) except ValueError: return failure('mz:max_zoom not float: %s' % max_zoom) is_landuse_aoi = props.get('mz:is_landuse_aoi') if is_landuse_aoi is not None: try: is_landuse_aoi = int(is_landuse_aoi) except ValueError: return failure('is_landuse_aoi not int: %s' % is_landuse_aoi) is_landuse_aoi = is_landuse_aoi != 0 if shape_mercator.type in ('Polygon', 'MultiPolygon'): area = int(shape_mercator.area) else: area = None # for the purposes of display, we only care about the times when something # should first start to be shown, and the time when it should stop # showing. edtf_inception = _normalize_edtf(props.get('edtf:inception')) edtf_cessation = _normalize_edtf(props.get('edtf:cessation')) edtf_deprecated = _normalize_edtf(props.get('edtf:deprecated')) # check that the dates are valid first to return back a better error inception_earliest = edtf_inception.lower_fuzzy() cessation_latest = edtf_cessation.upper_fuzzy() deprecated_latest = edtf_deprecated.upper_fuzzy() if inception_earliest is None: return failure('invalid edtf:inception: %s' % props.get('edtf:inception')) if cessation_latest is None: return failure('invalid edtf:cessation: %s' % props.get('edtf:cessation')) if deprecated_latest is None: return failure('invalid edtf:deprecated: %s' % props.get('edtf:deprecated')) # the 'edtf:inception' property gives us approximately the former and we # take the earliest date it could mean. the 'edtf:cessation' and # 'edtf:deprecated' would both stop the item showing, so we take the # earliest of each's latest possible date. inception = inception_earliest cessation = min(cessation_latest, deprecated_latest) # grab any names in other languages lang_suffix_size = len('_preferred') l10n_names = {} for k, v in props.iteritems(): if not v: continue if not k.startswith('name:') or not k.endswith('_preferred'): continue if isinstance(v, list): v = v[0] lang = k[:-lang_suffix_size] l10n_names[lang] = v if not l10n_names: l10n_names = None wikidata = None # get wikidata ID concordance, if there is one concordances = props.get('wof:concordances') if concordances: wikidata = concordances.get('wd:id') neighbourhood = Neighbourhood( wof_id, placetype, name, neighbourhood_meta.hash, label_position, shape_mercator, n_photos, area, min_zoom, max_zoom, is_landuse_aoi, inception, cessation, l10n_names, wikidata) return neighbourhood
def test_reproject_with_z(self): from tilequeue.tile import reproject_lnglat_to_mercator coord = reproject_lnglat_to_mercator(0, 0, 0) self.assertAlmostEqual(0, coord[0]) self.assertAlmostEqual(0, coord[1])