def parse_neighbourhood_meta_csv(csv_line_generator, placetype):
reader = csv.reader(csv_line_generator)
it = iter(reader)
header = it.next()
lbl_lat_idx = header.index('lbl_latitude')
lbl_lng_idx = header.index('lbl_longitude')
name_idx = header.index('name')
wof_id_idx = header.index('id')
hash_idx = header.index('file_hash')
superseded_by_idx = header.index('superseded_by')
min_row_length = (max(
lbl_lat_idx, lbl_lng_idx, name_idx, wof_id_idx, hash_idx,
superseded_by_idx) + 1)
for row in it:
if len(row) < min_row_length:
continue
superseded_by = row[superseded_by_idx]
if superseded_by:
continue
wof_id_str = row[wof_id_idx]
if not wof_id_str:
continue
try:
wof_id = int(wof_id_str)
except ValueError:
continue
name = row[name_idx]
if not name:
continue
lat_str = row[lbl_lat_idx]
lng_str = row[lbl_lng_idx]
try:
lat = float(lat_str)
lng = float(lng_str)
except ValueError:
continue
file_hash = row[hash_idx]
label_x, label_y = reproject_lnglat_to_mercator(lng, lat)
label_position = shapely.geometry.Point(label_x, label_y)
neighbourhood_meta = NeighbourhoodMeta(
wof_id, placetype, name, file_hash, label_position)
yield neighbourhood_meta
def parse_neighbourhood_meta_csv(csv_line_generator, placetype):
reader = csv.reader(csv_line_generator)
it = iter(reader)
header = it.next()
lbl_lat_idx = header.index('lbl_latitude')
lbl_lng_idx = header.index('lbl_longitude')
name_idx = header.index('name')
wof_id_idx = header.index('id')
hash_idx = header.index('file_hash')
superseded_by_idx = header.index('superseded_by')
min_row_length = (max(
lbl_lat_idx, lbl_lng_idx, name_idx, wof_id_idx, hash_idx,
superseded_by_idx) + 1)
for row in it:
if len(row) < min_row_length:
continue
superseded_by = row[superseded_by_idx]
if superseded_by:
continue
wof_id_str = row[wof_id_idx]
if not wof_id_str:
continue
try:
wof_id = int(wof_id_str)
except ValueError:
continue
name = row[name_idx]
if not name:
continue
lat_str = row[lbl_lat_idx]
lng_str = row[lbl_lng_idx]
try:
lat = float(lat_str)
lng = float(lng_str)
except ValueError:
continue
file_hash = row[hash_idx]
label_x, label_y = reproject_lnglat_to_mercator(lng, lat)
label_position = shapely.geometry.Point(label_x, label_y)
neighbourhood_meta = NeighbourhoodMeta(
wof_id, placetype, name, file_hash, label_position)
yield neighbourhood_meta
def create_neighbourhood_from_json(json_data, neighbourhood_meta):
def failure(reason):
return NeighbourhoodFailure(neighbourhood_meta.wof_id, reason,
json.dumps(json_data))
if not isinstance(json_data, dict):
return failure('Unexpected json')
props = json_data.get('properties')
if props is None or not isinstance(props, dict):
return failure('Missing properties')
superseded_by = props.get('wof:superseded_by')
# these often show up as empty lists, so we do a truthy test
# instead of expicitly checking for None
if superseded_by:
return NeighbourhoodFailure(neighbourhood_meta.wof_id,
'superseded_by: %s' % superseded_by,
json.dumps(json_data),
superseded=True)
geometry = json_data.get('geometry')
if geometry is None:
return failure('Missing geometry')
try:
shape_lnglat = shapely.geometry.shape(geometry)
except Exception:
return failure('Unexpected geometry')
shape_mercator = shapely.ops.transform(reproject_lnglat_to_mercator,
shape_lnglat)
# ignore any features that are marked as funky
is_funky = props.get('mz:is_funky')
if is_funky is not None:
try:
is_funky = int(is_funky)
except ValueError:
return failure('Unexpected mz:is_funky value %s' % is_funky)
if is_funky != 0:
return NeighbourhoodFailure(neighbourhood_meta.wof_id,
'mz:is_funky value is not 0: %s' %
is_funky,
json.dumps(json_data),
funky=True)
wof_id = props.get('wof:id')
if wof_id is None:
return failure('Missing wof:id')
try:
wof_id = int(wof_id)
except ValueError:
return failure('wof_id is not an int: %s' % wof_id)
name = props.get('wof:name')
if name is None:
return failure('Missing name')
n_photos = props.get('misc:photo_sum')
if n_photos is not None:
try:
n_photos = int(n_photos)
except ValueError:
return failure('misc:photo_sum is not an int: %s' % n_photos)
label_lat = props.get('lbl:latitude')
label_lng = props.get('lbl:longitude')
if label_lat is None or label_lng is None:
# first, try to fall back to geom:* when lbl:* is missing. we'd prefer
# to have lbl:*, but it's better to have _something_ than nothing.
label_lat = props.get('geom:latitude')
label_lng = props.get('geom:longitude')
if label_lat is None or label_lng is None:
return failure('Missing lbl:latitude or lbl:longitude and ' +
'geom:latitude or geom:longitude')
try:
label_lat = float(label_lat)
label_lng = float(label_lng)
except ValueError:
return failure('lbl:latitude or lbl:longitude not float')
label_merc_x, label_merc_y = reproject_lnglat_to_mercator(
label_lng, label_lat)
label_position = shapely.geometry.Point(label_merc_x, label_merc_y)
placetype = props.get('wof:placetype')
if placetype is None:
return failure('Missing wof:placetype')
default_min_zoom = 15
default_max_zoom = 16
min_zoom = props.get('mz:min_zoom')
if min_zoom is None:
min_zoom = default_min_zoom
else:
try:
min_zoom = float(min_zoom)
except ValueError:
return failure('mz:min_zoom not float: %s' % min_zoom)
max_zoom = props.get('mz:max_zoom')
if max_zoom is None:
max_zoom = default_max_zoom
else:
try:
max_zoom = float(max_zoom)
except ValueError:
return failure('mz:max_zoom not float: %s' % max_zoom)
is_landuse_aoi = props.get('mz:is_landuse_aoi')
if is_landuse_aoi is not None:
try:
is_landuse_aoi = int(is_landuse_aoi)
except ValueError:
return failure('is_landuse_aoi not int: %s' % is_landuse_aoi)
is_landuse_aoi = is_landuse_aoi != 0
if shape_mercator.type in ('Polygon', 'MultiPolygon'):
area = int(shape_mercator.area)
else:
area = None
# for the purposes of display, we only care about the times when something
# should first start to be shown, and the time when it should stop
# showing.
edtf_inception = _normalize_edtf(props.get('edtf:inception'))
edtf_cessation = _normalize_edtf(props.get('edtf:cessation'))
edtf_deprecated = _normalize_edtf(props.get('edtf:deprecated'))
# check that the dates are valid first to return back a better error
inception_earliest = edtf_inception.lower_fuzzy()
cessation_latest = edtf_cessation.upper_fuzzy()
deprecated_latest = edtf_deprecated.upper_fuzzy()
if inception_earliest is None:
return failure('invalid edtf:inception: %s' %
props.get('edtf:inception'))
if cessation_latest is None:
return failure('invalid edtf:cessation: %s' %
props.get('edtf:cessation'))
if deprecated_latest is None:
return failure('invalid edtf:deprecated: %s' %
props.get('edtf:deprecated'))
# the 'edtf:inception' property gives us approximately the former and we
# take the earliest date it could mean. the 'edtf:cessation' and
# 'edtf:deprecated' would both stop the item showing, so we take the
# earliest of each's latest possible date.
inception = inception_earliest
cessation = min(cessation_latest, deprecated_latest)
# grab any names in other languages
lang_suffix_size = len('_preferred')
l10n_names = {}
for k, v in props.iteritems():
if not v:
continue
if not k.startswith('name:') or not k.endswith('_preferred'):
continue
if isinstance(v, list):
v = v[0]
lang = k[:-lang_suffix_size]
l10n_names[lang] = v
if not l10n_names:
l10n_names = None
wikidata = None
# get wikidata ID concordance, if there is one
concordances = props.get('wof:concordances')
if concordances:
wikidata = concordances.get('wd:id')
neighbourhood = Neighbourhood(wof_id, placetype, name,
neighbourhood_meta.hash, label_position,
shape_mercator, n_photos, area, min_zoom,
max_zoom, is_landuse_aoi, inception,
cessation, l10n_names, wikidata)
return neighbourhood
def point(id, position, tags):
x, y = reproject_lnglat_to_mercator(*position)
return Feature(id, Point(x, y), _tags_to_utf8(tags))
def create_neighbourhood_from_json(json_data, neighbourhood_meta):
def failure(reason):
return NeighbourhoodFailure(
neighbourhood_meta.wof_id, reason, json.dumps(json_data))
if not isinstance(json_data, dict):
return failure('Unexpected json')
props = json_data.get('properties')
if props is None or not isinstance(props, dict):
return failure('Missing properties')
superseded_by = props.get('wof:superseded_by')
# these often show up as empty lists, so we do a truthy test
# instead of expicitly checking for None
if superseded_by:
return NeighbourhoodFailure(
neighbourhood_meta.wof_id,
'superseded_by: %s' % superseded_by,
json.dumps(json_data), superseded=True)
geometry = json_data.get('geometry')
if geometry is None:
return failure('Missing geometry')
try:
shape_lnglat = shapely.geometry.shape(geometry)
except Exception:
return failure('Unexpected geometry')
shape_mercator = shapely.ops.transform(
reproject_lnglat_to_mercator, shape_lnglat)
# ignore any features that are marked as funky
is_funky = props.get('mz:is_funky')
if is_funky is not None:
try:
is_funky = int(is_funky)
except ValueError:
return failure('Unexpected mz:is_funky value %s' % is_funky)
if is_funky != 0:
return NeighbourhoodFailure(
neighbourhood_meta.wof_id,
'mz:is_funky value is not 0: %s' % is_funky,
json.dumps(json_data), funky=True)
wof_id = props.get('wof:id')
if wof_id is None:
return failure('Missing wof:id')
try:
wof_id = int(wof_id)
except ValueError:
return failure('wof_id is not an int: %s' % wof_id)
name = props.get('wof:name')
if name is None:
return failure('Missing name')
n_photos = props.get('misc:photo_sum')
if n_photos is not None:
try:
n_photos = int(n_photos)
except ValueError:
return failure('misc:photo_sum is not an int: %s' % n_photos)
label_lat = props.get('lbl:latitude')
label_lng = props.get('lbl:longitude')
if label_lat is None or label_lng is None:
# first, try to fall back to geom:* when lbl:* is missing. we'd prefer
# to have lbl:*, but it's better to have _something_ than nothing.
label_lat = props.get('geom:latitude')
label_lng = props.get('geom:longitude')
if label_lat is None or label_lng is None:
return failure('Missing lbl:latitude or lbl:longitude and ' +
'geom:latitude or geom:longitude')
try:
label_lat = float(label_lat)
label_lng = float(label_lng)
except ValueError:
return failure('lbl:latitude or lbl:longitude not float')
label_merc_x, label_merc_y = reproject_lnglat_to_mercator(
label_lng, label_lat)
label_position = shapely.geometry.Point(label_merc_x, label_merc_y)
placetype = props.get('wof:placetype')
if placetype is None:
return failure('Missing wof:placetype')
default_min_zoom = 15
default_max_zoom = 16
min_zoom = props.get('mz:min_zoom')
if min_zoom is None:
min_zoom = default_min_zoom
else:
try:
min_zoom = float(min_zoom)
except ValueError:
return failure('mz:min_zoom not float: %s' % min_zoom)
max_zoom = props.get('mz:max_zoom')
if max_zoom is None:
max_zoom = default_max_zoom
else:
try:
max_zoom = float(max_zoom)
except ValueError:
return failure('mz:max_zoom not float: %s' % max_zoom)
is_landuse_aoi = props.get('mz:is_landuse_aoi')
if is_landuse_aoi is not None:
try:
is_landuse_aoi = int(is_landuse_aoi)
except ValueError:
return failure('is_landuse_aoi not int: %s' % is_landuse_aoi)
is_landuse_aoi = is_landuse_aoi != 0
if shape_mercator.type in ('Polygon', 'MultiPolygon'):
area = int(shape_mercator.area)
else:
area = None
# for the purposes of display, we only care about the times when something
# should first start to be shown, and the time when it should stop
# showing.
edtf_inception = _normalize_edtf(props.get('edtf:inception'))
edtf_cessation = _normalize_edtf(props.get('edtf:cessation'))
edtf_deprecated = _normalize_edtf(props.get('edtf:deprecated'))
# check that the dates are valid first to return back a better error
inception_earliest = edtf_inception.lower_fuzzy()
cessation_latest = edtf_cessation.upper_fuzzy()
deprecated_latest = edtf_deprecated.upper_fuzzy()
if inception_earliest is None:
return failure('invalid edtf:inception: %s' %
props.get('edtf:inception'))
if cessation_latest is None:
return failure('invalid edtf:cessation: %s' %
props.get('edtf:cessation'))
if deprecated_latest is None:
return failure('invalid edtf:deprecated: %s' %
props.get('edtf:deprecated'))
# the 'edtf:inception' property gives us approximately the former and we
# take the earliest date it could mean. the 'edtf:cessation' and
# 'edtf:deprecated' would both stop the item showing, so we take the
# earliest of each's latest possible date.
inception = inception_earliest
cessation = min(cessation_latest, deprecated_latest)
# grab any names in other languages
lang_suffix_size = len('_preferred')
l10n_names = {}
for k, v in props.iteritems():
if not v:
continue
if not k.startswith('name:') or not k.endswith('_preferred'):
continue
if isinstance(v, list):
v = v[0]
lang = k[:-lang_suffix_size]
l10n_names[lang] = v
if not l10n_names:
l10n_names = None
wikidata = None
# get wikidata ID concordance, if there is one
concordances = props.get('wof:concordances')
if concordances:
wikidata = concordances.get('wd:id')
neighbourhood = Neighbourhood(
wof_id, placetype, name, neighbourhood_meta.hash, label_position,
shape_mercator, n_photos, area, min_zoom, max_zoom, is_landuse_aoi,
inception, cessation, l10n_names, wikidata)
return neighbourhood
def test_reproject_with_z(self):
from tilequeue.tile import reproject_lnglat_to_mercator
coord = reproject_lnglat_to_mercator(0, 0, 0)
self.assertAlmostEqual(0, coord[0])
self.assertAlmostEqual(0, coord[1])
def point(id, position, tags):
x, y = reproject_lnglat_to_mercator(*position)
return Feature(id, Point(x, y), _tags_to_utf8(tags))
