def disambiguate_titles(self, source): entities = Entity.objects.filter(source=source) inferred_names = {} for entity in entities: inferred_name = entity.metadata['osm']['tags'].get('name') or entity.metadata['osm']['tags'].get('operator') if not inferred_name: continue if not inferred_name in inferred_names: inferred_names[inferred_name] = set() inferred_names[inferred_name].add(entity) for inferred_name, entities in inferred_names.items(): if len(entities) > 1: for entity in entities: if entity.metadata['osm']['tags'].get('addr:street'): entity.title = u"%s, %s" % (inferred_name, entity.metadata['osm']['tags'].get('addr:street')) continue try: name = reverse_geocode(entity.location[0], entity.location[1])[0]['name'] if name: entity.title = u"%s, %s" % (inferred_name, name) entity.save() except: self.output.write("Couldn't geocode for %s\n" % inferred_name)
def disambiguate_titles(self, source): entities = Entity.objects.filter(source=source) inferred_names = {} for entity in entities: inferred_name = entity.metadata['osm']['tags'].get( 'name') or entity.metadata['osm']['tags'].get('operator') if not inferred_name: continue if not inferred_name in inferred_names: inferred_names[inferred_name] = set() inferred_names[inferred_name].add(entity) for inferred_name, entities in inferred_names.items(): if len(entities) > 1: for entity in entities: if entity.metadata['osm']['tags'].get('addr:street'): entity.title = u"%s, %s" % ( inferred_name, entity.metadata['osm']['tags'].get('addr:street')) continue try: name = reverse_geocode(entity.location[0], entity.location[1])[0]['name'] if name: entity.title = u"%s, %s" % (inferred_name, name) entity.save() except: self.output.write("Couldn't geocode for %s\n" % inferred_name)
def clean(self): cleaned_data = self.cleaned_data if cleaned_data['method'] in ('html5', 'html5request', 'gears', 'manual', 'geocoded', 'other', 'favourite'): if cleaned_data['method'] == 'geocoded': results = geocode(cleaned_data['name']) if len(results) > 0: cleaned_data.update(results[0]) cleaned_data['longitude'], cleaned_data['latitude'] = cleaned_data['location'] cleaned_data['alternatives'] = results[1:] else: raise forms.ValidationError("Unable to find a location that matches '%s'." % cleaned_data['name']) for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified']) if not self._errors: cleaned_data['location'] = cleaned_data['longitude'], cleaned_data['latitude'] if not cleaned_data.get('name'): try: cleaned_data['name'] = reverse_geocode( self.cleaned_data['longitude'], self.cleaned_data['latitude'])[0]['name'] except: cleaned_data['name'] = None elif cleaned_data['method'] in ('denied', 'error'): for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified']) else: self._errors['method'] = ErrorList(['method is required']) return cleaned_data
def disambiguate_titles(self, source): lang_code = get_language() entities = Entity.objects.filter(source=source) inferred_names = {} if '-' in lang_code: tags_to_try = ('name-%s' % lang_code, 'name-%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name-%s' % lang_code, 'name', 'operator') for entity in entities: inferred_name = None for tag_to_try in tags_to_try: if entity.metadata['osm']['tags'].get(tag_to_try): inferred_name = entity.metadata['osm']['tags'].get( tag_to_try) break if not inferred_name: continue if not inferred_name in inferred_names: inferred_names[inferred_name] = set() inferred_names[inferred_name].add(entity) for inferred_name, entities in inferred_names.items(): if len(entities) > 1: for entity in entities: if entity.metadata['osm']['tags'].get('addr_street'): title = u"%s, %s" % ( inferred_name, entity.metadata['osm']['tags'].get('addr_street')) else: try: place_name = reverse_geocode( entity.location[0], entity.location[1])[0]['name'] if place_name: title = u"%s, %s" % (inferred_name, place_name) entity.save() else: title = inferred_name except: logger.info("Couldn't geocode for %s\n" % inferred_name) title = inferred_name try: name = entity.names.get(language_code=lang_code) except EntityName.DoesNotExist: name = entity.names.create(language_code=lang_code, title=title) else: name.title = title finally: name.save()
def clean(self): cleaned_data = self.cleaned_data if cleaned_data['method'] in ('html5', 'html5request', 'gears', 'manual', 'geocoded', 'other', 'favourite'): if cleaned_data['method'] == 'geocoded': if not cleaned_data['name'].strip(): raise forms.ValidationError(_("You must enter a location")) results = geocode(cleaned_data['name']) if len(results) > 0: cleaned_data.update(results[0]) cleaned_data['longitude'], cleaned_data[ 'latitude'] = cleaned_data['location'] # Ignore alternatives for postcodes if not re.match(POSTCODE_RE, cleaned_data['name'].upper()): cleaned_data['alternatives'] = results[1:] else: cleaned_data['alternatives'] = [] else: raise forms.ValidationError( _("Unable to find a location that matches '%s'.") % cleaned_data['name']) for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList( ['method requires that ' + key + ' must be specified']) if not self._errors: cleaned_data['location'] = cleaned_data[ 'longitude'], cleaned_data['latitude'] if not cleaned_data.get('name'): try: cleaned_data['name'] = reverse_geocode( self.cleaned_data['longitude'], self.cleaned_data['latitude'])[0]['name'] except: cleaned_data['name'] = u"↝ %f, %f" % ( self.cleaned_data['longitude'], self.cleaned_data['latitude']) elif cleaned_data['method'] in ('denied', 'error'): for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList( ['method requires that ' + key + ' must be specified']) else: self._errors['method'] = ErrorList(['method is required']) return cleaned_data
def disambiguate_titles(self, source): lang_code = get_language() entities = Entity.objects.filter(source=source) inferred_names = {} if '-' in lang_code: tags_to_try = ('name-%s' % lang_code, 'name-%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name-%s' % lang_code, 'name', 'operator') for entity in entities: inferred_name = None for tag_to_try in tags_to_try: if entity.metadata['osm']['tags'].get(tag_to_try): inferred_name = entity.metadata['osm']['tags'].get(tag_to_try) break if not inferred_name: continue if not inferred_name in inferred_names: inferred_names[inferred_name] = set() inferred_names[inferred_name].add(entity) for inferred_name, entities in inferred_names.items(): if len(entities) > 1: for entity in entities: if entity.metadata['osm']['tags'].get('addr_street'): title = u"%s, %s" % (inferred_name, entity.metadata['osm']['tags'].get('addr_street')) else: try: place_name = reverse_geocode(entity.location[0], entity.location[1])[0]['name'] if place_name: title = u"%s, %s" % (inferred_name, place_name) entity.save() else: title = inferred_name except: self.output.write("Couldn't geocode for %s\n" % inferred_name) title = inferred_name try: name = entity.names.get(language_code=lang_code) except EntityName.DoesNotExist: name = entity.names.create(language_code=lang_code, title=title) else: name.title = title finally: name.save()
def clean(self): cleaned_data = self.cleaned_data if cleaned_data['method'] in ('html5', 'html5request', 'gears', 'manual', 'geocoded', 'other', 'favourite'): if cleaned_data['method'] == 'geocoded': results = geocode(cleaned_data['name']) if len(results) > 0: cleaned_data.update(results[0]) cleaned_data['longitude'], cleaned_data[ 'latitude'] = cleaned_data['location'] cleaned_data['alternatives'] = results[1:] else: raise forms.ValidationError( "Unable to find a location that matches '%s'." % cleaned_data['name']) for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList( ['method requires that ' + key + ' must be specified']) if not self._errors: cleaned_data['location'] = cleaned_data[ 'longitude'], cleaned_data['latitude'] if not cleaned_data.get('name'): try: cleaned_data['name'] = reverse_geocode( self.cleaned_data['longitude'], self.cleaned_data['latitude'])[0]['name'] except: cleaned_data['name'] = None elif cleaned_data['method'] in ('denied', 'error'): for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList( ['method requires that ' + key + ' must be specified']) else: self._errors['method'] = ErrorList(['method is required']) return cleaned_data
def clean(self): cleaned_data = self.cleaned_data if cleaned_data['method'] in ('html5', 'html5request', 'gears','manual', 'geocoded', 'other', 'favourite'): if cleaned_data['method'] == 'geocoded': if not cleaned_data['name'].strip(): raise forms.ValidationError(_("You must enter a location")) results = geocode(cleaned_data['name']) if len(results) > 0: cleaned_data.update(results[0]) cleaned_data['longitude'], cleaned_data['latitude'] = cleaned_data['location'] # Ignore alternatives for postcodes if not re.match(POSTCODE_RE, cleaned_data['name'].upper()): cleaned_data['alternatives'] = results[1:] else: cleaned_data['alternatives'] = [] else: raise forms.ValidationError(_("Unable to find a location that matches '%s'.") % cleaned_data['name']) for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified']) if not self._errors: cleaned_data['location'] = cleaned_data['longitude'], cleaned_data['latitude'] if not cleaned_data.get('name'): try: cleaned_data['name'] = reverse_geocode( self.cleaned_data['longitude'], self.cleaned_data['latitude'])[0]['name'] except: cleaned_data['name'] = u"↝ %f, %f" % (self.cleaned_data['longitude'], self.cleaned_data['latitude']) elif cleaned_data['method'] in ('denied', 'error'): for key in ('latitude', 'longitude', 'accuracy'): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(['method requires that ' + key + ' must be specified']) else: self._errors['method'] = ErrorList(['method is required']) return cleaned_data
def clean(self): cleaned_data = self.cleaned_data if cleaned_data["method"] in ("html5", "html5request", "gears", "manual", "geocoded", "other", "favourite"): if cleaned_data["method"] == "geocoded": results = geocode(cleaned_data["name"]) if len(results) > 0: cleaned_data.update(results[0]) cleaned_data["longitude"], cleaned_data["latitude"] = cleaned_data["location"] # Ignore alternatives for postcodes if not re.match(POSTCODE_RE, cleaned_data["name"].upper()): cleaned_data["alternatives"] = results[1:] else: cleaned_data["alternatives"] = [] else: raise forms.ValidationError("Unable to find a location that matches '%s'." % cleaned_data["name"]) for key in ("latitude", "longitude", "accuracy"): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(["method requires that " + key + " must be specified"]) if not self._errors: cleaned_data["location"] = cleaned_data["longitude"], cleaned_data["latitude"] if not cleaned_data.get("name"): try: cleaned_data["name"] = reverse_geocode( self.cleaned_data["longitude"], self.cleaned_data["latitude"] )[0]["name"] except: cleaned_data["name"] = u"↝ %f, %f" % ( self.cleaned_data["longitude"], self.cleaned_data["latitude"], ) elif cleaned_data["method"] in ("denied", "error"): for key in ("latitude", "longitude", "accuracy"): if cleaned_data.get(key) is None: self._errors[key] = ErrorList(["method requires that " + key + " must be specified"]) else: self._errors["method"] = ErrorList(["method is required"]) return cleaned_data
def endElement(self, name): if name in ('node','way') and self.valid: try: types = self.find_types(self.tags) except ValueError: self.ignore_count += 1 return # Ignore ways that lay partly outside our bounding box if name == 'way' and not all(id in self.node_locations for id in self.nodes): return # We already have these from OxPoints, so leave them alone. if self.tags.get('amenity') == 'library' and self.tags.get('operator') == 'University of Oxford': return # Ignore disused and under-construction entities if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get('disused') in ('1', 'yes', 'true'): return try: entity = Entity.objects.get(source=self.source, _identifiers__scheme='osm', _identifiers__value=self.id) created = True except Entity.DoesNotExist: entity = Entity(source=self.source) created = False if not 'osm' in entity.metadata or \ entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']: if created: self.create_count += 1 else: self.modify_count += 1 if name == 'node': entity.location = Point(self.node_location, srid=4326) entity.geometry = entity.location elif name == 'way': cls = LinearRing if self.nodes[0] == self.nodes[-1] else LineString entity.geometry = cls([self.node_locations[n] for n in self.nodes], srid=4326) min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf')) for lon, lat in [self.node_locations[n] for n in self.nodes]: min_ = min(min_[0], lon), min(min_[1], lat) max_ = max(max_[0], lon), max(max_[1], lat) entity.location = Point( (min_[0]+max_[0])/2 , (min_[1]+max_[1])/2 , srid=4326) else: raise AssertionError("There should be no other types of entity we're to deal with.") names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): if '-' in lang_code: tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name:%s' % lang_code, 'name', 'operator') name = None for tag_to_try in tags_to_try: if self.tags.get(tag_to_try): name = self.tags.get(tag_to_try) break if name is None: try: name = reverse_geocode(*entity.location)[0]['name'] if not name: raise IndexError name = u"↝ %s" % name except IndexError: name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0]) names[lang_code] = name entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict(zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } entity.primary_type = self.entity_types[types[0]] entity.save(identifiers={'osm': self.id}) for lang_code, name in names.items(): set_name_in_language(entity, lang_code, title=name) entity.all_types = [self.entity_types[et] for et in types] entity.update_all_types_completion() else: self.unchanged_count += 1
def testReverseGeocode(self): points = [(51.758504,-1.256055)] for point in points: geolocation.reverse_geocode(*point)
def endElement(self, name): if name in ('node', 'way') and self.valid: try: types = self.find_types(self.tags) except ValueError: self.ignore_count += 1 return # Ignore ways that lay partly outside our bounding box if name == 'way' and not all(id in self.node_locations for id in self.nodes): return # Ignore disused and under-construction entities if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get( 'disused') in ('1', 'yes', 'true'): return # Memory management in debug mode reset_queries() if self.id in self.identities: entity = get_entity(*self.identities[self.id].split(':')) entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict( zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } identifiers = entity.identifiers identifiers.update({'osm': self.id}) entity.save(identifiers=identifiers) entity.all_types = set(entity.all_types.all()) | set( self.entity_types[et] for et in types) entity.update_all_types_completion() self.ids.remove(self.id) else: try: entity = Entity.objects.get(source=self.source, _identifiers__scheme='osm', _identifiers__value=self.id) created = False except Entity.DoesNotExist: entity = Entity(source=self.source) created = True if not 'osm' in entity.metadata or \ entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']: if created: self.create_count += 1 else: self.modify_count += 1 if name == 'node': entity.location = Point(self.node_location, srid=4326) entity.geometry = entity.location elif name == 'way': cls = LinearRing if self.nodes[0] == self.nodes[ -1] else LineString entity.geometry = cls( [self.node_locations[n] for n in self.nodes], srid=4326) min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf')) for lon, lat in [ self.node_locations[n] for n in self.nodes ]: min_ = min(min_[0], lon), min(min_[1], lat) max_ = max(max_[0], lon), max(max_[1], lat) entity.location = Point((min_[0] + max_[0]) / 2, (min_[1] + max_[1]) / 2, srid=4326) else: raise AssertionError( "There should be no other types of entity we're to deal with." ) names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): if '-' in lang_code: tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name:%s' % lang_code, 'name', 'operator') name = None for tag_to_try in tags_to_try: if self.tags.get(tag_to_try): name = self.tags.get(tag_to_try) break if name is None: try: name = reverse_geocode( *entity.location)[0]['name'] if not name: raise IndexError name = u"↝ %s" % name except IndexError: name = u"↝ %f, %f" % ( self.node_location[1], self.node_location[0]) names[lang_code] = name entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict( zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } entity.primary_type = self.entity_types[types[0]] identifiers = entity.identifiers identifiers.update({'osm': self.id}) entity.save(identifiers=identifiers) for lang_code, name in names.items(): set_name_in_language(entity, lang_code, title=name) entity.all_types = [self.entity_types[et] for et in types] entity.update_all_types_completion() else: self.unchanged_count += 1
def endElement(self, name): if name in ('node', 'way') and self.valid: try: types = self.find_types(self.tags) except ValueError: self.ignore_count += 1 return # Ignore ways that lay partly outside our bounding box if name == 'way' and not all(id in self.node_locations for id in self.nodes): return # We already have these from OxPoints, so leave them alone. if self.tags.get('amenity') == 'library' and self.tags.get( 'operator') == 'University of Oxford': return # Ignore disused and under-construction entities if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get( 'disused') in ('1', 'yes', 'true'): return try: entity = Entity.objects.get(source=self.source, _identifiers__scheme='osm', _identifiers__value=self.id) created = True except Entity.DoesNotExist: entity = Entity(source=self.source) created = False if not 'osm' in entity.metadata or entity.metadata['osm'].get( 'attrs', {}).get('timestamp', '') < self.attrs['timestamp']: if created: self.create_count += 1 else: self.modify_count += 1 if name == 'node': entity.location = Point(self.node_location, srid=4326) entity.geometry = entity.location elif name == 'way': cls = LinearRing if self.nodes[0] == self.nodes[ -1] else LineString entity.geometry = cls( [self.node_locations[n] for n in self.nodes], srid=4326) min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf')) for lon, lat in [ self.node_locations[n] for n in self.nodes ]: min_ = min(min_[0], lon), min(min_[1], lat) max_ = max(max_[0], lon), max(max_[1], lat) entity.location = Point((min_[0] + max_[0]) / 2, (min_[1] + max_[1]) / 2, srid=4326) else: raise AssertionError( "There should be no other types of entity we're to deal with." ) try: name = self.tags.get('name') or self.tags['operator'] except (KeyError, AssertionError): try: name = reverse_geocode(*entity.location)[0]['name'] if not name: raise IndexError name = u"↝ %s" % name except IndexError: name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0]) entity.title = name entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': self.tags } entity.primary_type = self.entity_types[types[0]] if 'addr:postcode' in self.tags: entity.post_code = self.tags['addr:postcode'].replace( ' ', '') else: entity.post_code = "" entity.save(identifiers={'osm': self.id}) entity.all_types = [self.entity_types[et] for et in types] entity.update_all_types_completion() else: self.unchanged_count += 1
def testReverseGeocode(self): points = [(51.758504, -1.256055)] for point in points: reverse_geocode(*point)