def disambiguate_titles(self, source):
entities = Entity.objects.filter(source=source)
inferred_names = {}
for entity in entities:
inferred_name = entity.metadata['osm']['tags'].get('name') or entity.metadata['osm']['tags'].get('operator')
if not inferred_name:
continue
if not inferred_name in inferred_names:
inferred_names[inferred_name] = set()
inferred_names[inferred_name].add(entity)
for inferred_name, entities in inferred_names.items():
if len(entities) > 1:
for entity in entities:
if entity.metadata['osm']['tags'].get('addr:street'):
entity.title = u"%s, %s" % (inferred_name, entity.metadata['osm']['tags'].get('addr:street'))
continue
try:
name = reverse_geocode(entity.location[0], entity.location[1])[0]['name']
if name:
entity.title = u"%s, %s" % (inferred_name, name)
entity.save()
except:
self.output.write("Couldn't geocode for %s\n" % inferred_name)
def disambiguate_titles(self, source):
entities = Entity.objects.filter(source=source)
inferred_names = {}
for entity in entities:
inferred_name = entity.metadata['osm']['tags'].get(
'name') or entity.metadata['osm']['tags'].get('operator')
if not inferred_name:
continue
if not inferred_name in inferred_names:
inferred_names[inferred_name] = set()
inferred_names[inferred_name].add(entity)
for inferred_name, entities in inferred_names.items():
if len(entities) > 1:
for entity in entities:
if entity.metadata['osm']['tags'].get('addr:street'):
entity.title = u"%s, %s" % (
inferred_name,
entity.metadata['osm']['tags'].get('addr:street'))
continue
try:
name = reverse_geocode(entity.location[0],
entity.location[1])[0]['name']
if name:
entity.title = u"%s, %s" % (inferred_name, name)
entity.save()
except:
self.output.write("Couldn't geocode for %s\n" %
inferred_name)
def clean(self):
cleaned_data = self.cleaned_data
if cleaned_data['method'] in ('html5', 'html5request', 'gears', 'manual', 'geocoded', 'other', 'favourite'):
if cleaned_data['method'] == 'geocoded':
results = geocode(cleaned_data['name'])
if len(results) > 0:
cleaned_data.update(results[0])
cleaned_data['longitude'], cleaned_data['latitude'] = cleaned_data['location']
cleaned_data['alternatives'] = results[1:]
else:
raise forms.ValidationError("Unable to find a location that matches '%s'." % cleaned_data['name'])
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified'])
if not self._errors:
cleaned_data['location'] = cleaned_data['longitude'], cleaned_data['latitude']
if not cleaned_data.get('name'):
try:
cleaned_data['name'] = reverse_geocode(
self.cleaned_data['longitude'],
self.cleaned_data['latitude'])[0]['name']
except:
cleaned_data['name'] = None
elif cleaned_data['method'] in ('denied', 'error'):
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified'])
else:
self._errors['method'] = ErrorList(['method is required'])
return cleaned_data
def disambiguate_titles(self, source):
lang_code = get_language()
entities = Entity.objects.filter(source=source)
inferred_names = {}
if '-' in lang_code:
tags_to_try = ('name-%s' % lang_code,
'name-%s' % lang_code.split('-')[0], 'name',
'operator')
else:
tags_to_try = ('name-%s' % lang_code, 'name', 'operator')
for entity in entities:
inferred_name = None
for tag_to_try in tags_to_try:
if entity.metadata['osm']['tags'].get(tag_to_try):
inferred_name = entity.metadata['osm']['tags'].get(
tag_to_try)
break
if not inferred_name:
continue
if not inferred_name in inferred_names:
inferred_names[inferred_name] = set()
inferred_names[inferred_name].add(entity)
for inferred_name, entities in inferred_names.items():
if len(entities) > 1:
for entity in entities:
if entity.metadata['osm']['tags'].get('addr_street'):
title = u"%s, %s" % (
inferred_name,
entity.metadata['osm']['tags'].get('addr_street'))
else:
try:
place_name = reverse_geocode(
entity.location[0],
entity.location[1])[0]['name']
if place_name:
title = u"%s, %s" % (inferred_name, place_name)
entity.save()
else:
title = inferred_name
except:
logger.info("Couldn't geocode for %s\n" %
inferred_name)
title = inferred_name
try:
name = entity.names.get(language_code=lang_code)
except EntityName.DoesNotExist:
name = entity.names.create(language_code=lang_code,
title=title)
else:
name.title = title
finally:
name.save()
def clean(self):
cleaned_data = self.cleaned_data
if cleaned_data['method'] in ('html5', 'html5request', 'gears',
'manual', 'geocoded', 'other',
'favourite'):
if cleaned_data['method'] == 'geocoded':
if not cleaned_data['name'].strip():
raise forms.ValidationError(_("You must enter a location"))
results = geocode(cleaned_data['name'])
if len(results) > 0:
cleaned_data.update(results[0])
cleaned_data['longitude'], cleaned_data[
'latitude'] = cleaned_data['location']
# Ignore alternatives for postcodes
if not re.match(POSTCODE_RE, cleaned_data['name'].upper()):
cleaned_data['alternatives'] = results[1:]
else:
cleaned_data['alternatives'] = []
else:
raise forms.ValidationError(
_("Unable to find a location that matches '%s'.") %
cleaned_data['name'])
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(
['method requires that ' + key + ' must be specified'])
if not self._errors:
cleaned_data['location'] = cleaned_data[
'longitude'], cleaned_data['latitude']
if not cleaned_data.get('name'):
try:
cleaned_data['name'] = reverse_geocode(
self.cleaned_data['longitude'],
self.cleaned_data['latitude'])[0]['name']
except:
cleaned_data['name'] = u"↝ %f, %f" % (
self.cleaned_data['longitude'],
self.cleaned_data['latitude'])
elif cleaned_data['method'] in ('denied', 'error'):
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(
['method requires that ' + key + ' must be specified'])
else:
self._errors['method'] = ErrorList(['method is required'])
return cleaned_data
def disambiguate_titles(self, source):
lang_code = get_language()
entities = Entity.objects.filter(source=source)
inferred_names = {}
if '-' in lang_code:
tags_to_try = ('name-%s' % lang_code, 'name-%s' % lang_code.split('-')[0], 'name', 'operator')
else:
tags_to_try = ('name-%s' % lang_code, 'name', 'operator')
for entity in entities:
inferred_name = None
for tag_to_try in tags_to_try:
if entity.metadata['osm']['tags'].get(tag_to_try):
inferred_name = entity.metadata['osm']['tags'].get(tag_to_try)
break
if not inferred_name:
continue
if not inferred_name in inferred_names:
inferred_names[inferred_name] = set()
inferred_names[inferred_name].add(entity)
for inferred_name, entities in inferred_names.items():
if len(entities) > 1:
for entity in entities:
if entity.metadata['osm']['tags'].get('addr_street'):
title = u"%s, %s" % (inferred_name, entity.metadata['osm']['tags'].get('addr_street'))
else:
try:
place_name = reverse_geocode(entity.location[0], entity.location[1])[0]['name']
if place_name:
title = u"%s, %s" % (inferred_name, place_name)
entity.save()
else:
title = inferred_name
except:
self.output.write("Couldn't geocode for %s\n" % inferred_name)
title = inferred_name
try:
name = entity.names.get(language_code=lang_code)
except EntityName.DoesNotExist:
name = entity.names.create(language_code=lang_code,
title=title)
else:
name.title = title
finally:
name.save()
def clean(self):
cleaned_data = self.cleaned_data
if cleaned_data['method'] in ('html5', 'html5request', 'gears',
'manual', 'geocoded', 'other',
'favourite'):
if cleaned_data['method'] == 'geocoded':
results = geocode(cleaned_data['name'])
if len(results) > 0:
cleaned_data.update(results[0])
cleaned_data['longitude'], cleaned_data[
'latitude'] = cleaned_data['location']
cleaned_data['alternatives'] = results[1:]
else:
raise forms.ValidationError(
"Unable to find a location that matches '%s'." %
cleaned_data['name'])
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(
['method requires that ' + key + ' must be specified'])
if not self._errors:
cleaned_data['location'] = cleaned_data[
'longitude'], cleaned_data['latitude']
if not cleaned_data.get('name'):
try:
cleaned_data['name'] = reverse_geocode(
self.cleaned_data['longitude'],
self.cleaned_data['latitude'])[0]['name']
except:
cleaned_data['name'] = None
elif cleaned_data['method'] in ('denied', 'error'):
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(
['method requires that ' + key + ' must be specified'])
else:
self._errors['method'] = ErrorList(['method is required'])
return cleaned_data
def clean(self):
cleaned_data = self.cleaned_data
if cleaned_data['method'] in ('html5', 'html5request', 'gears','manual', 'geocoded', 'other', 'favourite'):
if cleaned_data['method'] == 'geocoded':
if not cleaned_data['name'].strip():
raise forms.ValidationError(_("You must enter a location"))
results = geocode(cleaned_data['name'])
if len(results) > 0:
cleaned_data.update(results[0])
cleaned_data['longitude'], cleaned_data['latitude'] = cleaned_data['location']
# Ignore alternatives for postcodes
if not re.match(POSTCODE_RE, cleaned_data['name'].upper()):
cleaned_data['alternatives'] = results[1:]
else:
cleaned_data['alternatives'] = []
else:
raise forms.ValidationError(_("Unable to find a location that matches '%s'.") % cleaned_data['name'])
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified'])
if not self._errors:
cleaned_data['location'] = cleaned_data['longitude'], cleaned_data['latitude']
if not cleaned_data.get('name'):
try:
cleaned_data['name'] = reverse_geocode(
self.cleaned_data['longitude'],
self.cleaned_data['latitude'])[0]['name']
except:
cleaned_data['name'] = u"↝ %f, %f" % (self.cleaned_data['longitude'], self.cleaned_data['latitude'])
elif cleaned_data['method'] in ('denied', 'error'):
for key in ('latitude', 'longitude', 'accuracy'):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified'])
else:
self._errors['method'] = ErrorList(['method is required'])
return cleaned_data
def clean(self):
cleaned_data = self.cleaned_data
if cleaned_data["method"] in ("html5", "html5request", "gears", "manual", "geocoded", "other", "favourite"):
if cleaned_data["method"] == "geocoded":
results = geocode(cleaned_data["name"])
if len(results) > 0:
cleaned_data.update(results[0])
cleaned_data["longitude"], cleaned_data["latitude"] = cleaned_data["location"]
# Ignore alternatives for postcodes
if not re.match(POSTCODE_RE, cleaned_data["name"].upper()):
cleaned_data["alternatives"] = results[1:]
else:
cleaned_data["alternatives"] = []
else:
raise forms.ValidationError("Unable to find a location that matches '%s'." % cleaned_data["name"])
for key in ("latitude", "longitude", "accuracy"):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(["method requires that " + key + " must be specified"])
if not self._errors:
cleaned_data["location"] = cleaned_data["longitude"], cleaned_data["latitude"]
if not cleaned_data.get("name"):
try:
cleaned_data["name"] = reverse_geocode(
self.cleaned_data["longitude"], self.cleaned_data["latitude"]
)[0]["name"]
except:
cleaned_data["name"] = u"↝ %f, %f" % (
self.cleaned_data["longitude"],
self.cleaned_data["latitude"],
)
elif cleaned_data["method"] in ("denied", "error"):
for key in ("latitude", "longitude", "accuracy"):
if cleaned_data.get(key) is None:
self._errors[key] = ErrorList(["method requires that " + key + " must be specified"])
else:
self._errors["method"] = ErrorList(["method is required"])
return cleaned_data
def endElement(self, name):
if name in ('node','way') and self.valid:
try:
types = self.find_types(self.tags)
except ValueError:
self.ignore_count += 1
return
# Ignore ways that lay partly outside our bounding box
if name == 'way' and not all(id in self.node_locations for id in self.nodes):
return
# We already have these from OxPoints, so leave them alone.
if self.tags.get('amenity') == 'library' and self.tags.get('operator') == 'University of Oxford':
return
# Ignore disused and under-construction entities
if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get('disused') in ('1', 'yes', 'true'):
return
try:
entity = Entity.objects.get(source=self.source,
_identifiers__scheme='osm',
_identifiers__value=self.id)
created = True
except Entity.DoesNotExist:
entity = Entity(source=self.source)
created = False
if not 'osm' in entity.metadata or \
entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']:
if created:
self.create_count += 1
else:
self.modify_count += 1
if name == 'node':
entity.location = Point(self.node_location, srid=4326)
entity.geometry = entity.location
elif name == 'way':
cls = LinearRing if self.nodes[0] == self.nodes[-1] else LineString
entity.geometry = cls([self.node_locations[n] for n in self.nodes], srid=4326)
min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf'))
for lon, lat in [self.node_locations[n] for n in self.nodes]:
min_ = min(min_[0], lon), min(min_[1], lat)
max_ = max(max_[0], lon), max(max_[1], lat)
entity.location = Point( (min_[0]+max_[0])/2 , (min_[1]+max_[1])/2 , srid=4326)
else:
raise AssertionError("There should be no other types of entity we're to deal with.")
names = dict()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
if '-' in lang_code:
tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator')
else:
tags_to_try = ('name:%s' % lang_code, 'name', 'operator')
name = None
for tag_to_try in tags_to_try:
if self.tags.get(tag_to_try):
name = self.tags.get(tag_to_try)
break
if name is None:
try:
name = reverse_geocode(*entity.location)[0]['name']
if not name:
raise IndexError
name = u"↝ %s" % name
except IndexError:
name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0])
names[lang_code] = name
entity.metadata['osm'] = {
'attrs': dict(self.attrs),
'tags': dict(zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values()))
}
entity.primary_type = self.entity_types[types[0]]
entity.save(identifiers={'osm': self.id})
for lang_code, name in names.items():
set_name_in_language(entity, lang_code, title=name)
entity.all_types = [self.entity_types[et] for et in types]
entity.update_all_types_completion()
else:
self.unchanged_count += 1
def testReverseGeocode(self):
points = [(51.758504,-1.256055)]
for point in points:
geolocation.reverse_geocode(*point)
def endElement(self, name):
if name in ('node', 'way') and self.valid:
try:
types = self.find_types(self.tags)
except ValueError:
self.ignore_count += 1
return
# Ignore ways that lay partly outside our bounding box
if name == 'way' and not all(id in self.node_locations
for id in self.nodes):
return
# Ignore disused and under-construction entities
if self.tags.get('life_cycle',
'in_use') != 'in_use' or self.tags.get(
'disused') in ('1', 'yes', 'true'):
return
# Memory management in debug mode
reset_queries()
if self.id in self.identities:
entity = get_entity(*self.identities[self.id].split(':'))
entity.metadata['osm'] = {
'attrs':
dict(self.attrs),
'tags':
dict(
zip((k.replace(':', '_') for k in self.tags.keys()),
self.tags.values()))
}
identifiers = entity.identifiers
identifiers.update({'osm': self.id})
entity.save(identifiers=identifiers)
entity.all_types = set(entity.all_types.all()) | set(
self.entity_types[et] for et in types)
entity.update_all_types_completion()
self.ids.remove(self.id)
else:
try:
entity = Entity.objects.get(source=self.source,
_identifiers__scheme='osm',
_identifiers__value=self.id)
created = False
except Entity.DoesNotExist:
entity = Entity(source=self.source)
created = True
if not 'osm' in entity.metadata or \
entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']:
if created:
self.create_count += 1
else:
self.modify_count += 1
if name == 'node':
entity.location = Point(self.node_location, srid=4326)
entity.geometry = entity.location
elif name == 'way':
cls = LinearRing if self.nodes[0] == self.nodes[
-1] else LineString
entity.geometry = cls(
[self.node_locations[n] for n in self.nodes],
srid=4326)
min_, max_ = (float('inf'),
float('inf')), (float('-inf'),
float('-inf'))
for lon, lat in [
self.node_locations[n] for n in self.nodes
]:
min_ = min(min_[0], lon), min(min_[1], lat)
max_ = max(max_[0], lon), max(max_[1], lat)
entity.location = Point((min_[0] + max_[0]) / 2,
(min_[1] + max_[1]) / 2,
srid=4326)
else:
raise AssertionError(
"There should be no other types of entity we're to deal with."
)
names = dict()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
if '-' in lang_code:
tags_to_try = ('name:%s' % lang_code,
'name:%s' %
lang_code.split('-')[0], 'name',
'operator')
else:
tags_to_try = ('name:%s' % lang_code, 'name',
'operator')
name = None
for tag_to_try in tags_to_try:
if self.tags.get(tag_to_try):
name = self.tags.get(tag_to_try)
break
if name is None:
try:
name = reverse_geocode(
*entity.location)[0]['name']
if not name:
raise IndexError
name = u"↝ %s" % name
except IndexError:
name = u"↝ %f, %f" % (
self.node_location[1],
self.node_location[0])
names[lang_code] = name
entity.metadata['osm'] = {
'attrs':
dict(self.attrs),
'tags':
dict(
zip((k.replace(':', '_')
for k in self.tags.keys()),
self.tags.values()))
}
entity.primary_type = self.entity_types[types[0]]
identifiers = entity.identifiers
identifiers.update({'osm': self.id})
entity.save(identifiers=identifiers)
for lang_code, name in names.items():
set_name_in_language(entity, lang_code, title=name)
entity.all_types = [self.entity_types[et] for et in types]
entity.update_all_types_completion()
else:
self.unchanged_count += 1
def endElement(self, name):
if name in ('node', 'way') and self.valid:
try:
types = self.find_types(self.tags)
except ValueError:
self.ignore_count += 1
return
# Ignore ways that lay partly outside our bounding box
if name == 'way' and not all(id in self.node_locations
for id in self.nodes):
return
# We already have these from OxPoints, so leave them alone.
if self.tags.get('amenity') == 'library' and self.tags.get(
'operator') == 'University of Oxford':
return
# Ignore disused and under-construction entities
if self.tags.get('life_cycle',
'in_use') != 'in_use' or self.tags.get(
'disused') in ('1', 'yes', 'true'):
return
try:
entity = Entity.objects.get(source=self.source,
_identifiers__scheme='osm',
_identifiers__value=self.id)
created = True
except Entity.DoesNotExist:
entity = Entity(source=self.source)
created = False
if not 'osm' in entity.metadata or entity.metadata['osm'].get(
'attrs', {}).get('timestamp',
'') < self.attrs['timestamp']:
if created:
self.create_count += 1
else:
self.modify_count += 1
if name == 'node':
entity.location = Point(self.node_location, srid=4326)
entity.geometry = entity.location
elif name == 'way':
cls = LinearRing if self.nodes[0] == self.nodes[
-1] else LineString
entity.geometry = cls(
[self.node_locations[n] for n in self.nodes],
srid=4326)
min_, max_ = (float('inf'), float('inf')), (float('-inf'),
float('-inf'))
for lon, lat in [
self.node_locations[n] for n in self.nodes
]:
min_ = min(min_[0], lon), min(min_[1], lat)
max_ = max(max_[0], lon), max(max_[1], lat)
entity.location = Point((min_[0] + max_[0]) / 2,
(min_[1] + max_[1]) / 2,
srid=4326)
else:
raise AssertionError(
"There should be no other types of entity we're to deal with."
)
try:
name = self.tags.get('name') or self.tags['operator']
except (KeyError, AssertionError):
try:
name = reverse_geocode(*entity.location)[0]['name']
if not name:
raise IndexError
name = u"↝ %s" % name
except IndexError:
name = u"↝ %f, %f" % (self.node_location[1],
self.node_location[0])
entity.title = name
entity.metadata['osm'] = {
'attrs': dict(self.attrs),
'tags': self.tags
}
entity.primary_type = self.entity_types[types[0]]
if 'addr:postcode' in self.tags:
entity.post_code = self.tags['addr:postcode'].replace(
' ', '')
else:
entity.post_code = ""
entity.save(identifiers={'osm': self.id})
entity.all_types = [self.entity_types[et] for et in types]
entity.update_all_types_completion()
else:
self.unchanged_count += 1
def testReverseGeocode(self):
points = [(51.758504, -1.256055)]
for point in points:
reverse_geocode(*point)
