def _load_from_csv(self, reader, entity_type, source): j = 0 for i, line in enumerate(reader): postcode_abbrev, (easting, northing) = line[0], line[10:12] if postcode_abbrev[-4] != ' ': postcode = '%s %s' % (postcode_abbrev[:-3], postcode_abbrev[-3:]) else: postcode = postcode_abbrev postcode_abbrev = postcode_abbrev.replace(' ', '') try: easting, northing = int(easting), int(northing) except ValueError: continue j += 1 try: entity = Entity.objects.get(source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev) except Entity.DoesNotExist: entity = Entity(source=source) entity.location = Point(easting, northing, srid=27700) entity.geometry = entity.location entity.primary_type = entity_type identifiers = { 'postcode': postcode_abbrev, 'postcode-canonical': postcode, } entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title = postcode) entity.all_types.add(entity_type) entity.update_all_types_completion()
def _get_entity_types(self): entity_types = {} new_entity_types = set() for slug, et in self._entity_types.items(): et_category, created = EntityTypeCategory.objects.get_or_create(name=et['category']) try: entity_type = EntityType.objects.get(slug=slug) created = False except EntityType.DoesNotExist: entity_type = EntityType(slug=slug) created = True entity_type.category = et_category entity_type.slug = slug if created: entity_type.show_in_nearby_list = et['show_in_nearby_list'] entity_type.show_in_category_list = et['show_in_category_list'] entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=_(et['verbose_name']), verbose_name_singular=_(et['verbose_name_singular']), verbose_name_plural=_(et['verbose_name_plural'])) new_entity_types.add(slug) entity_types[slug] = entity_type for slug in new_entity_types: subtype_of = self._entity_types[slug]['parent-types'] entity_types[slug].subtype_of.clear() for s in subtype_of: entity_types[slug].subtype_of.add(entity_types[s]) entity_types[slug].save() return entity_types
def _get_entity(self, stop_code, stop_name, source, entity_type): """Finds a bus stop entity or creates one if it cannot be found. If multiple entities are found we clean them up. """ scheme = 'naptan' try: entity = get_entity(scheme, stop_code) except: try: entity = Entity.objects.get(_identifiers__scheme=scheme, _identifiers__value=stop_code) logger.debug("Found Entity: %s" % entity) except Entity.DoesNotExist: logger.debug("Entity does not exist: %s-%s" % (stop_code, stop_name)) entity = Entity() except Entity.MultipleObjectsReturned: logger.warning("Multiple Entities found for : %s-%s" % (stop_code, stop_name)) Entity.objects.filter(_identifiers__scheme=scheme, _identifiers__value=stop_code).delete() entity = Entity() entity.primary_type = entity_type entity.source = source identifiers = {scheme: stop_code} set_name_in_language(entity, 'en', title=stop_name) entity.all_types = (entity_type, ) entity.save(identifiers=identifiers) return entity
def import_data(self, metadata, output): source, entity_type = self._get_source(), self._get_entity_type() parser = etree.XMLParser(load_dtd=True) parser.resolvers.add(BBCTPEGResolver()) xml = etree.parse(urllib.urlopen(self._tpeg_url), parser=parser) entities, seen = {}, set() for entity in Entity.objects.filter(source=source): if 'bbc-tpeg' in entity.identifiers: entities[entity.identifiers['bbc-tpeg']] = entity for message in xml.getroot().findall('tpeg_message'): id = message.find('road_traffic_message').attrib['message_id'] road_traffic_message = message.find('road_traffic_message') try: entity = entities[id] except KeyError: entity = Entity() entities[id] = entity entity.source = source entity.primary_type = entity_type locs = map( self._wgs84_to_point, road_traffic_message.findall( 'location_container/location_coordinates/WGS84')) if len(locs) > 1: entity.geometry = LineString(*locs) elif len(locs) == 1: entity.geometry = locs[0] else: continue entity.location = Point( sum(p.x for p in locs) / len(locs), sum(p.y for p in locs) / len(locs), srid=4326, ) entity.metadata['bbc_tpeg'] = { 'xml': etree.tostring(message), 'severity': road_traffic_message.attrib['severity_factor'], 'generated': road_traffic_message.attrib['message_generation_time'], 'version': int(road_traffic_message.attrib['version_number']), } entity.save(identifiers={'bbc-tpeg': id}) set_name_in_language(entity, 'en', title=message.find('summary').text) entity.all_types = [entity_type] entity.update_all_types_completion() seen.add(entity.pk) for entity in Entity.objects.filter(source=source): if not entity.pk in seen: entity.delete()
def _get_entity(self, stop_code, stop_name, source, entity_type): """Finds a bus stop entity or creates one if it cannot be found. If multiple entities are found we clean them up. """ scheme = 'naptan' try: entity = get_entity(scheme, stop_code) except: try: entity = Entity.objects.get(_identifiers__scheme=scheme, _identifiers__value=stop_code) logger.debug("Found Entity: %s" % entity) except Entity.DoesNotExist: logger.debug("Entity does not exist: %s-%s" % (stop_code, stop_name)) entity = Entity() except Entity.MultipleObjectsReturned: logger.warning("Multiple Entities found for : %s-%s" % (stop_code, stop_name)) Entity.objects.filter(_identifiers__scheme=scheme, _identifiers__value=stop_code).delete() entity = Entity() entity.primary_type = entity_type entity.source = source identifiers = {scheme: stop_code} set_name_in_language(entity, 'en', title=stop_name) entity.all_types = (entity_type,) entity.save(identifiers=identifiers) return entity
def _scrape(self, route, url, output): self._output.write(route) url += '&showall=1' service = etree.parse(urlopen(url), parser = etree.HTMLParser()) route.stops.clear() for i, tr in enumerate(service.find('.//table').findall('tr')[1:]): try: stop_code = tr[1][0].text except IndexError: # Stops on ACIS Live that don't have codes, e.g., out of county # stops stop_name = tr[3][0].text try: entity = Entity.objects.get(source=self._get_source(), _identifiers__scheme='acisroute', _identifiers__value=stop_name) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) entity_type = self._get_entity_type() entity.primary_type = entity_type identifiers = { 'acisroute': stop_name } entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=stop_name) entity.all_types = (entity_type,) entity.update_all_types_completion() else: # TODO: Change identifier lookup based on ACIS region try: entity = get_entity('naptan', stop_code) if entity.source == self._get_source(): # Raise Http404 if this is a bus stop we came up with, # so any name changes, etc, get processed raise Http404() except Http404: # Out of zone bus stops with NaPTAN codes - alternatively, # the fake bus stops Oxontime made up for the TUBE route try: entity = Entity.objects.get(source=self._get_source(), _identifiers__scheme='naptan', _identifiers__value=stop_code) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) identifiers = { 'naptan': stop_code } entity_type = self._get_entity_type() entity.primary_type = entity_type entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=tr[3][0].text) entity.all_types = (entity_type,) entity.update_all_types_completion() entity.save() StopOnRoute.objects.create(route=route, entity=entity, order=i)
def import_data(self, metadata, output): source, entity_type = self._get_source(), self._get_entity_type() parser = etree.XMLParser(load_dtd=True) parser.resolvers.add(BBCTPEGResolver()) xml = etree.parse(urllib.urlopen(self._tpeg_url), parser=parser) entities, seen = {}, set() for entity in Entity.objects.filter(source=source): if 'bbc-tpeg' in entity.identifiers: entities[entity.identifiers['bbc-tpeg']] = entity for message in xml.getroot().findall('tpeg_message'): id = message.find('road_traffic_message').attrib['message_id'] road_traffic_message = message.find('road_traffic_message') try: entity = entities[id] except KeyError: entity = Entity() entities[id] = entity entity.source = source entity.primary_type = entity_type locs = map(self._wgs84_to_point, road_traffic_message.findall('location_container/location_coordinates/WGS84')) if len(locs) > 1: entity.geometry = LineString(*locs) elif len(locs) == 1: entity.geometry = locs[0] else: continue entity.location = Point( sum(p.x for p in locs)/len(locs), sum(p.y for p in locs)/len(locs), srid=4326, ) entity.metadata['bbc_tpeg'] = { 'xml': etree.tostring(message), 'severity': road_traffic_message.attrib['severity_factor'], 'generated': road_traffic_message.attrib['message_generation_time'], 'version': int(road_traffic_message.attrib['version_number']), } entity.save(identifiers={'bbc-tpeg': id}) set_name_in_language(entity, 'en', title = message.find('summary').text) entity.all_types = [entity_type] entity.update_all_types_completion() seen.add(entity.pk) for entity in Entity.objects.filter(source=source): if not entity.pk in seen: entity.delete()
def _load_from_csv(self, reader, entity_type, source): j = 0 for i, line in enumerate(reader): postcode_abbrev, (easting, northing) = line[0], line[10:12] postcode_abbrev = postcode_abbrev.replace(' ', '') # Now try to figure out where to put the space in if re.match(r'[A-Z][0-9]{2}[A-Z]{2}', postcode_abbrev): # A9 9AA postcode = '%s %s' % (postcode_abbrev[:2], postcode_abbrev[2:]) elif re.match(r'[A-Z][0-9]{3}[A-Z]{2}', postcode_abbrev): # A99 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9]{2}[A-Z]{2}', postcode_abbrev): # AA9 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9]{3}[A-Z]{2}', postcode_abbrev): # AA99 9AA postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:]) elif re.match(r'[A-Z][0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev): # A9A 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev): # AA9A 9AA postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:]) else: postcode = postcode_abbrev try: easting, northing = int(easting), int(northing) except ValueError: continue j += 1 try: entity = Entity.objects.get( source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev) except Entity.DoesNotExist: entity = Entity(source=source) entity.location = Point(easting, northing, srid=27700) entity.geometry = entity.location entity.primary_type = entity_type identifiers = { 'postcode': postcode_abbrev, 'postcode-canonical': postcode, } entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=postcode) entity.all_types.add(entity_type) entity.update_all_types_completion()
def _load_from_csv(self, reader, entity_type, source): j = 0 for i, line in enumerate(reader): postcode_abbrev, (easting, northing) = line[0], line[10:12] postcode_abbrev = postcode_abbrev.replace(' ', '') # Now try to figure out where to put the space in if re.match(r'[A-Z][0-9]{2}[A-Z]{2}', postcode_abbrev): # A9 9AA postcode = '%s %s' % (postcode_abbrev[:2], postcode_abbrev[2:]) elif re.match(r'[A-Z][0-9]{3}[A-Z]{2}', postcode_abbrev): # A99 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9]{2}[A-Z]{2}', postcode_abbrev): # AA9 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9]{3}[A-Z]{2}', postcode_abbrev): # AA99 9AA postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:]) elif re.match(r'[A-Z][0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev): # A9A 9AA postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:]) elif re.match(r'[A-Z]{2}[0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev): # AA9A 9AA postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:]) else: postcode = postcode_abbrev try: easting, northing = int(easting), int(northing) except ValueError: continue j += 1 try: entity = Entity.objects.get(source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev) except Entity.DoesNotExist: entity = Entity(source=source) entity.location = Point(easting, northing, srid=27700) entity.geometry = entity.location entity.primary_type = entity_type identifiers = { 'postcode': postcode_abbrev, 'postcode-canonical': postcode, } entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title = postcode) entity.all_types.add(entity_type) entity.update_all_types_completion()
def endElement(self, name): self.name_stack.pop() if name == 'StopPoint': try: # Classify metro stops according to their particular system if self.meta['stop-type'] == 'MET': try: entity_type, is_entrance = self.entity_types[ self.meta['stop-type'] + ':' + self.meta['atco-code'][6:8]] except KeyError: entity_type, is_entrance = self.entity_types['MET'] else: entity_type, is_entrance = self.entity_types[ self.meta['stop-type']] except KeyError: pass else: entity = self.add_stop(self.meta, entity_type, self.source, is_entrance) if entity: self.entities.add(entity) elif name == 'StopAreaRef': self.stop_areas.append(self.meta['stop-area']) del self.meta['stop-area'] elif name == 'StopArea': if self.areas != None: in_area = False for area in self.areas: if self.meta['area-code'].startswith(area): in_area = True if not in_area: return sa, created = EntityGroup.objects.get_or_create( source=self.source, ref_code=self.meta['area-code']) sa.save() for lang_code, name in self.names.items(): if lang_code is None: lang_code = 'en' set_name_in_language(sa, lang_code, title=name) elif name == 'CommonName': if self.lang not in self.names: self.names[self.lang] = self.meta['common-name'] elif name == 'Name' and self.meta['name'] != '': if self.lang not in self.names: self.names[self.lang] = self.meta['name']
def endElement(self, name): self.name_stack.pop() if name == 'StopPoint': try: # Classify metro stops according to their particular system if self.meta['stop-type'] == 'MET': try: entity_type, is_entrance = self.entity_types[self.meta['stop-type'] + ':' + self.meta['atco-code'][6:8]] except KeyError: entity_type, is_entrance = self.entity_types['MET'] else: entity_type, is_entrance = self.entity_types[self.meta['stop-type']] except KeyError: pass else: entity = self.add_stop(self.meta, entity_type, self.source, is_entrance) if entity: self.entities.add(entity) elif name == 'StopAreaRef': self.stop_areas.append(self.meta['stop-area']) del self.meta['stop-area'] elif name == 'StopArea': if self.areas != None: in_area = False for area in self.areas: if self.meta['area-code'].startswith(area): in_area = True if not in_area: return sa, created = EntityGroup.objects.get_or_create( source=self.source, ref_code=self.meta['area-code']) sa.save() for lang_code, name in self.names.items(): if lang_code is None: lang_code = 'en' set_name_in_language(sa, lang_code, title=name) elif name == 'CommonName': if self.lang not in self.names: self.names[self.lang] = self.meta['common-name'] elif name == 'Name' and self.meta['name'] != '': if self.lang not in self.names: self.names[self.lang] = self.meta['name']
def _get_entity_type(self): category, created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Uncategorised')) entity_type, created = EntityType.objects.get_or_create( slug='post-code', category=category) entity_type.slug = 'post-code' if created: entity_type.show_in_nearby_list = False entity_type.show_in_category_list = False entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=_('postcode'), verbose_name_singular=_('a postcode'), verbose_name_plural=_('postcodes')) return entity_type
def _get_entity_types(self): entity_types = {} category, created = EntityTypeCategory.objects.get_or_create( name=_('Transport')) category.save() for stop_type in self.entity_type_definitions: et = self.entity_type_definitions[stop_type] try: entity_type = EntityType.objects.get(slug=et['slug']) except EntityType.DoesNotExist: entity_type = EntityType(slug=et['slug']) entity_type.category = category entity_type.uri = "http://mollyproject.org/schema/maps#%s" % et[ 'uri-local'] if created: entity_type.show_in_nearby_list = et['nearby'] entity_type.show_in_category_list = et['category'] entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=ugettext( et['verbose-name']), verbose_name_singular=ugettext( et['verbose-name-singular']), verbose_name_plural=ugettext( et['verbose-name-plural'])) entity_types[stop_type] = entity_type for stop_type, entity_type in entity_types.items(): if entity_type.slug == 'public-transport-access-node': continue entity_type.subtype_of.add(entity_types[None]) if stop_type.startswith( 'MET' ) and stop_type != 'MET' and entity_type.slug != self.RAIL_STATION_DEFINITION[ 'slug']: entity_type.subtype_of.add(entity_types['MET']) return entity_types
def _get_entity_type(self): category, created = EntityTypeCategory.objects.get_or_create( name=ugettext_noop('Uncategorised')) entity_type, created = EntityType.objects.get_or_create( slug='post-code', category=category) entity_type.slug = 'post-code' if created: entity_type.show_in_nearby_list = False entity_type.show_in_category_list = False entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=_('postcode'), verbose_name_singular=_('a postcode'), verbose_name_plural=_('postcodes')) return entity_type
def decode_category(self, attrib): if self._category is None: cat = 'Uncategorised' else: cat = self._category slug = slugify(cat) podcast_category, created = PodcastCategory.objects.get_or_create(slug=slug) set_name_in_language(podcast_category, self.lang_code, name=cat) try: podcast_category.order = self.CATEGORY_ORDERS[slug] except KeyError: self.CATEGORY_ORDERS[slug] = len(self.CATEGORY_ORDERS) podcast_category.order = self.CATEGORY_ORDERS[slug] podcast_category.save() return podcast_category
def _get_entity_type(self): try: entity_type = EntityType.objects.get(slug='travel-alert') created = False except EntityType.DoesNotExist: entity_type = EntityType(slug='travel-alert') created = True category, etc_created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Transport')) if created: entity_type.show_in_nearby_list = False entity_type.show_in_category_list = False entity_type.category = category entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=_('travel alert'), verbose_name_singular=_('a travel alert'), verbose_name_plural=_('travel alerts')) return entity_type
def decode_category(self, attrib): if self._category is None: cat = 'Uncategorised' else: cat = self._category slug = slugify(cat) podcast_category, created = PodcastCategory.objects.get_or_create( slug=slug) set_name_in_language(podcast_category, self.lang_code, name=cat) try: podcast_category.order = self.CATEGORY_ORDERS[slug] except KeyError: self.CATEGORY_ORDERS[slug] = len(self.CATEGORY_ORDERS) podcast_category.order = self.CATEGORY_ORDERS[slug] podcast_category.save() return podcast_category
def import_data(self, **metadata): atom = self.atom xml = etree.parse(urllib.urlopen(self.url)) rss_urls = [] category_elems = xml.getroot().findall(atom('entry')) for i, category_elem in enumerate(category_elems): link = category_elem.find(atom('link') + "[@rel='alternate']") slug = link.attrib['href'].split('/')[-1] category, created = PodcastCategory.objects.get_or_create( slug=slug) set_name_in_language(category, lang_code, name=category_elem.find(atom('title')).text) category.order = i category.save() category_xml = etree.parse(urllib.urlopen(link.attrib['href'])) for podcast_elem in category_xml.getroot().findall(atom('entry')): url = podcast_elem.find(atom('link') + "[@rel='alternate']").attrib['href'] slug = url.split('/')[-1] podcast, created = Podcast.objects.get_or_create( provider=self.class_path, slug=slug) podcast.rss_url = url podcast.category = category rss_urls.append(url) self.update_podcast.delay(podcast) for podcast in Podcast.objects.filter(provider=self.class_path): if not podcast.rss_url in rss_urls: podcast.delete() return metadata
def _get_entity_type(self): try: entity_type = EntityType.objects.get(slug='travel-alert') created = False except EntityType.DoesNotExist: entity_type = EntityType(slug='travel-alert') created = True category, etc_created = EntityTypeCategory.objects.get_or_create( name=ugettext_noop('Transport')) if created: entity_type.show_in_nearby_list = False entity_type.show_in_category_list = False entity_type.category = category entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=_('travel alert'), verbose_name_singular=_('a travel alert'), verbose_name_plural=_('travel alerts')) return entity_type
def _get_entity_types(self): entity_types = {} category, created = EntityTypeCategory.objects.get_or_create(name=_('Transport')) category.save() for stop_type in self.entity_type_definitions: et = self.entity_type_definitions[stop_type] try: entity_type = EntityType.objects.get(slug=et['slug']) except EntityType.DoesNotExist: entity_type = EntityType(slug=et['slug']) entity_type.category = category entity_type.uri = "http://mollyproject.org/schema/maps#%s" % et['uri-local'] if created: entity_type.show_in_nearby_list = et['nearby'] entity_type.show_in_category_list = et['category'] entity_type.save() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): set_name_in_language(entity_type, lang_code, verbose_name=ugettext(et['verbose-name']), verbose_name_singular=ugettext(et['verbose-name-singular']), verbose_name_plural=ugettext(et['verbose-name-plural'])) entity_types[stop_type] = entity_type for stop_type, entity_type in entity_types.items(): if entity_type.slug == 'public-transport-access-node': continue entity_type.subtype_of.add(entity_types[None]) if stop_type.startswith('MET') and stop_type != 'MET' and entity_type.slug != self.RAIL_STATION_DEFINITION['slug']: entity_type.subtype_of.add(entity_types['MET']) return entity_types
def import_data(self, **metadata): atom = self.atom xml = etree.parse(urllib.urlopen(self.url)) rss_urls = [] category_elems = xml.getroot().findall(atom('entry')) for i, category_elem in enumerate(category_elems): link = category_elem.find(atom('link')+"[@rel='alternate']") slug = link.attrib['href'].split('/')[-1] category, created = PodcastCategory.objects.get_or_create(slug=slug) set_name_in_language(category, lang_code, name=category_elem.find(atom('title')).text) category.order = i category.save() category_xml = etree.parse(urllib.urlopen(link.attrib['href'])) for podcast_elem in category_xml.getroot().findall(atom('entry')): url = podcast_elem.find(atom('link')+"[@rel='alternate']").attrib['href'] slug = url.split('/')[-1] podcast, created = Podcast.objects.get_or_create( provider=self.class_path, slug=slug) podcast.rss_url = url podcast.category = category rss_urls.append(url) self.update_podcast.delay(podcast) for podcast in Podcast.objects.filter(provider=self.class_path): if not podcast.rss_url in rss_urls: podcast.delete() return metadata
def endElement(self, name): if name in ('node','way') and self.valid: try: types = self.find_types(self.tags) except ValueError: self.ignore_count += 1 return # Ignore ways that lay partly outside our bounding box if name == 'way' and not all(id in self.node_locations for id in self.nodes): return # We already have these from OxPoints, so leave them alone. if self.tags.get('amenity') == 'library' and self.tags.get('operator') == 'University of Oxford': return # Ignore disused and under-construction entities if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get('disused') in ('1', 'yes', 'true'): return try: entity = Entity.objects.get(source=self.source, _identifiers__scheme='osm', _identifiers__value=self.id) created = True except Entity.DoesNotExist: entity = Entity(source=self.source) created = False if not 'osm' in entity.metadata or \ entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']: if created: self.create_count += 1 else: self.modify_count += 1 if name == 'node': entity.location = Point(self.node_location, srid=4326) entity.geometry = entity.location elif name == 'way': cls = LinearRing if self.nodes[0] == self.nodes[-1] else LineString entity.geometry = cls([self.node_locations[n] for n in self.nodes], srid=4326) min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf')) for lon, lat in [self.node_locations[n] for n in self.nodes]: min_ = min(min_[0], lon), min(min_[1], lat) max_ = max(max_[0], lon), max(max_[1], lat) entity.location = Point( (min_[0]+max_[0])/2 , (min_[1]+max_[1])/2 , srid=4326) else: raise AssertionError("There should be no other types of entity we're to deal with.") names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): if '-' in lang_code: tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name:%s' % lang_code, 'name', 'operator') name = None for tag_to_try in tags_to_try: if self.tags.get(tag_to_try): name = self.tags.get(tag_to_try) break if name is None: try: name = reverse_geocode(*entity.location)[0]['name'] if not name: raise IndexError name = u"↝ %s" % name except IndexError: name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0]) names[lang_code] = name entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict(zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } entity.primary_type = self.entity_types[types[0]] entity.save(identifiers={'osm': self.id}) for lang_code, name in names.items(): set_name_in_language(entity, lang_code, title=name) entity.all_types = [self.entity_types[et] for et in types] entity.update_all_types_completion() else: self.unchanged_count += 1
def _import_cif(self, cif): """ Parse a CIF file """ # Clear cache once per file - avoid high memory usage self._cache = EntityCache() # Also reset SQL queries log reset_queries() routes = [] this_journey = None for line in cif: if line[:2] == 'QS': # Journey header if this_journey is not None: routes[-1]['journies'].append(this_journey) if line[2] == 'D': this_journey = None continue this_journey = { 'operator-code': line[3:7], 'id': line[7:13], 'start-date': self._parse_cif_date(line[13:21]), 'end-date': self._parse_cif_date(line[21:29]), 'days': weekbool( line[29] == '1', # Monday line[30] == '1', # Tuesday line[31] == '1', # Wednesday line[32] == '1', # Thursday line[33] == '1', # Friday line[34] == '1', # Saturday line[35] == '1', # Sunday ), 'school-holidays': { 'S': 'term-time', 'H': 'holidays' }.get(line[36], 'all'), 'bank-holidays': { 'A': 'additional', 'B': 'holidays', 'X': 'non-holidays' }.get(line[37], 'all'), 'route': line[38:42], 'vehicle': line[48:56].strip(), 'direction': line[64], 'notes': [], 'stops': [], } elif line[:2] in ('QN', 'ZN'): # Notes this_journey['notes'].append(line[7:]) elif line[:2] == 'QO': # Journey start try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': None, 'std': self._parse_cif_time(line[14:18]), 'activity': 'O', 'estimated': line[22] == '0', 'fare-stage': line[24] == '1' }) except Http404: pass elif line[:2] == 'QI': # Journey intermediate stop try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': self._parse_cif_time(line[14:18]), 'std': self._parse_cif_time(line[18:22]), 'activity': line[22], 'estimated': line[27] == '0', 'fare-stage': line[29] == '1' }) except Http404: pass elif line[:2] == 'QT': # Journey complete try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': self._parse_cif_time(line[14:18]), 'std': None, 'activity': 'F', 'estimated': line[22] == '0', 'fare-stage': line[24] == '1' }) except Http404: pass elif line[:2] == 'ZL': # Route ID route_id = line[2:] elif line[:2] == 'ZD': # Days route ID route_id += line[18:-1] elif line[:2] == 'ZS': # Route if this_journey is not None: routes[-1]['journies'].append(this_journey) routes.append({ 'id': route_id, 'number': line[10:14].strip(), 'description': line[14:-1], 'stops': [], 'journies': [] }) elif line[:2] == 'ZA': stop_code = line[3:15].strip() try: entity = self._cache['atco:%s' % stop_code] if entity.source == self._get_source(): # Raise Http404 if this is a bus stop we came up with, # so any name changes, etc, get processed raise Http404() except Http404: # Out of zone bus stops with NaPTAN codes try: entity = Entity.objects.get( source=self._get_source(), _identifiers__scheme='atco', _identifiers__value=stop_code) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) identifiers = {'atco': stop_code} entity_type = self._entity_type entity.primary_type = entity_type entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=line[15:63].strip()) entity.all_types = (entity_type, ) entity.update_all_types_completion() entity.save() routes[-1]['stops'].append(entity) if this_journey is not None: routes[-1]['journies'].append(this_journey) return routes
def add_stop(self, meta, entity_type, source, is_entrance): # Check this entity is in an area if self.areas != None: in_area = False for area in self.areas: if meta['atco-code'].startswith(area): in_area = True if not in_area: return # See if we're updating an existing object, or creating a new one try: entity = Entity.objects.get(source=source, _identifiers__scheme='atco', _identifiers__value=meta['atco-code']) except Entity.DoesNotExist: entity = Entity(source=source) except Entity.MultipleObjectsReturned: # Handle clashes Entity.objects.filter(source=source, _identifiers__scheme='atco', _identifiers__value=meta['atco-code']).delete() entity = Entity(source=source) common_name, indicator, locality, street = [meta.get(k) for k in ('common-name', 'indicator', 'locality-ref', 'street')] if (common_name or '').endswith(' DEL') or \ (indicator or '').lower() == 'not in use' or \ 'to define route' in (common_name or '') or \ 'to def rte' in (common_name or '') or \ 'to def route' in (common_name or '') or \ 'def.rte' in (common_name or ''): # In the NaPTAN list, but indicates it's an unused stop return if self.meta['stop-type'] in ('MET','GAT','FER', 'RLY'): names = self.names else: names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): # Try and find one in our preferred order for lang in (lang_code, 'en', None): if lang in self.names: common_name = self.names[lang] break # Expand abbreviations in indicators if indicator is not None: parts = [] for part in indicator.split(): parts.append({ # Translators: This is referring to bus stop location descriptions 'op': ugettext('Opposite'), 'opp': ugettext('Opposite'), 'opposite': ugettext('Opposite'), # Translators: This is referring to bus stop location descriptions 'adj': ugettext('Adjacent'), # Translators: This is referring to bus stop location descriptions 'outside': ugettext('Outside'), 'o/s': ugettext('Outside'), # Translators: This is referring to bus stop location descriptions 'nr': ugettext('Near'), # Translators: This is referring to bus stop location descriptions 'inside': ugettext('Inside'), # Translators: This is referring to bus stop location descriptions 'stp': ugettext('Stop'), }.get(part.lower(), part)) indicator = ' '.join(parts) if indicator is None and self.meta['stop-type'] in ('AIR', 'FTD', 'RSE', 'TMU', 'BCE'): # Translators: This is referring to public transport entities title = ugettext('Entrance to %s') % common_name elif indicator is None and self.meta['stop-type'] in ('FBT',): # Translators: This is referring to ferry ports title = ugettext('Berth at %s') % common_name elif indicator is None and self.meta['stop-type'] in ('RPL','PLT'): # Translators: This is referring to rail and metro stations title = ugettext('Platform at %s') % common_name elif indicator is not None and indicator.lower() != 'none' \ and indicator not in common_name: title = indicator + ' ' + common_name else: title = common_name if street not in (None, '-', '---'): # Deal with all-caps street names if street.upper() == street: fixedstreet = '' wordstart = True for letter in street: if wordstart: wordstart = False fixedstreet += letter continue elif letter == ' ': wordstart = True fixedstreet += letter continue else: fixedstreet += letter.lower() street = fixedstreet if street not in title: title += ', ' + street locality_lang = self.nptg_localities.get(locality) if locality_lang != None: for lang in (lang_code, 'en', 'cy'): if lang in locality_lang: if locality_lang[lang] != street: title += ', ' + locality_lang[lang] break names[lang_code] = title entity.primary_type = entity_type entity.is_entrance = is_entrance if not entity.metadata: entity.metadata = {} entity.metadata['naptan'] = meta entity.location = Point(float(meta['longitude']), float(meta['latitude']), srid=4326) entity.geometry = entity.location if meta['atco-code'] in self.tube_references: entity.metadata['london-underground-identifiers'] = self.tube_references[meta['atco-code']] identifiers = { 'atco': meta['atco-code'], } if 'naptan-code' in meta: meta['naptan-code'] = ''.join(map(self.naptan_dial, meta['naptan-code'])) identifiers['naptan'] = meta['naptan-code'] if 'plate-code' in meta: identifiers['plate'] = meta['plate-code'] if 'crs' in meta: identifiers['crs'] = meta['crs'] if 'tiploc' in meta: identifiers['tiploc'] = meta['tiploc'] if indicator != None and re.match('Stop [A-Z]\d\d?', indicator): identifiers['stop'] = indicator[5:] entity.save(identifiers=identifiers) for lang_code, name in names.items(): # This is the NaPTAN, so default to English if lang_code is None: lang_code = 'en' set_name_in_language(entity, lang_code, title=name) entity.all_types = (entity_type,) entity.update_all_types_completion() entity.groups.clear() for stop_area in self.stop_areas: sa, created = EntityGroup.objects.get_or_create(source=source, ref_code=stop_area) entity.groups.add(sa) entity.save() return entity
def add_stop(self, meta, entity_type, source): # Check this entity is in an area if self.areas != None: in_area = False for area in self.areas: if meta['atco-code'].startswith(area): in_area = True if not in_area: return # See if we're updating an existing object, or creating a new one try: entity = Entity.objects.get(source=source, _identifiers__scheme='atco', _identifiers__value=meta['atco-code']) except Entity.DoesNotExist: entity = Entity(source=source) except Entity.MultipleObjectsReturned: # Handle clashes Entity.objects.filter( source=source, _identifiers__scheme='atco', _identifiers__value=meta['atco-code']).delete() entity = Entity(source=source) common_name, indicator, locality, street = [ meta.get(k) for k in ('common-name', 'indicator', 'locality-ref', 'street') ] if (common_name or '').endswith(' DEL') or \ (indicator or '').lower() == 'not in use' or \ 'to define route' in (common_name or '') or \ 'to def rte' in (common_name or '') or \ 'to def route' in (common_name or '') or \ 'def.rte' in (common_name or ''): # In the NaPTAN list, but indicates it's an unused stop return if self.meta['stop-type'] in ('MET', 'GAT', 'FER', 'RLY'): names = self.names else: names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): # Try and find one in our preferred order for lang in (lang_code, 'en', None): if lang in self.names: common_name = self.names[lang] break # Expand abbreviations in indicators if indicator is not None: parts = [] for part in indicator.split(): parts.append({ # Translators: This is referring to bus stop location descriptions 'op': ugettext('Opposite'), 'opp': ugettext('Opposite'), 'opposite': ugettext('Opposite'), # Translators: This is referring to bus stop location descriptions 'adj': ugettext('Adjacent'), # Translators: This is referring to bus stop location descriptions 'outside': ugettext('Outside'), 'o/s': ugettext('Outside'), # Translators: This is referring to bus stop location descriptions 'nr': ugettext('Near'), # Translators: This is referring to bus stop location descriptions 'inside': ugettext('Inside'), # Translators: This is referring to bus stop location descriptions 'stp': ugettext('Stop'), }.get(part.lower(), part)) indicator = ' '.join(parts) if indicator is None and self.meta['stop-type'] in ( 'AIR', 'FTD', 'RSE', 'TMU', 'BCE'): # Translators: This is referring to public transport entities title = ugettext('Entrance to %s') % common_name elif indicator is None and self.meta['stop-type'] in ( 'FBT', ): # Translators: This is referring to ferry ports title = ugettext('Berth at %s') % common_name elif indicator is None and self.meta['stop-type'] in ( 'RPL', 'PLT'): # Translators: This is referring to rail and metro stations title = ugettext('Platform at %s') % common_name elif indicator is not None and indicator.lower() != 'none' \ and indicator not in common_name: title = indicator + ' ' + common_name else: title = common_name if street != None and street != '-': # Deal with all-caps street names if street.upper() == street: fixedstreet = '' wordstart = True for letter in street: if wordstart: wordstart = False fixedstreet += letter continue elif letter == ' ': wordstart = True fixedstreet += letter continue else: fixedstreet += letter.lower() street = fixedstreet if street not in title: title += ', ' + street locality_lang = self.nptg_localities.get(locality) if locality_lang != None: for lang in (lang_code, 'en', 'cy'): if lang in locality_lang: if locality_lang[lang] != street: title += ', ' + locality_lang[lang] break names[lang_code] = title entity.primary_type = entity_type if not entity.metadata: entity.metadata = {} entity.metadata['naptan'] = meta entity.location = Point(float(meta['longitude']), float(meta['latitude']), srid=4326) entity.geometry = entity.location if meta['atco-code'] in self.tube_references: entity.metadata[ 'london-underground-identifiers'] = self.tube_references[ meta['atco-code']] identifiers = { 'atco': meta['atco-code'], } if 'naptan-code' in meta: meta['naptan-code'] = ''.join( map(self.naptan_dial, meta['naptan-code'])) identifiers['naptan'] = meta['naptan-code'] if 'plate-code' in meta: identifiers['plate'] = meta['plate-code'] if 'crs' in meta: identifiers['crs'] = meta['crs'] if indicator != None and re.match('Stop [A-Z]\d\d?', indicator): identifiers['stop'] = indicator[5:] entity.save(identifiers=identifiers) for lang_code, name in names.items(): # This is the NaPTAN, so default to English if lang_code is None: lang_code = 'en' set_name_in_language(entity, lang_code, title=name) entity.all_types = (entity_type, ) entity.update_all_types_completion() entity.groups.clear() for stop_area in self.stop_areas: sa, created = EntityGroup.objects.get_or_create(source=source, ref_code=stop_area) entity.groups.add(sa) entity.save() return entity
def _import_cif(self, cif): """ Parse a CIF file """ # Clear cache once per file - avoid high memory usage self._cache = EntityCache() # Also reset SQL queries log reset_queries() routes = [] this_journey = None for line in cif: if line[:2] == 'QS': # Journey header if this_journey is not None: routes[-1]['journies'].append(this_journey) if line[2] == 'D': this_journey = None continue this_journey = { 'operator-code': line[3:7], 'id': line[7:13], 'start-date': self._parse_cif_date(line[13:21]), 'end-date': self._parse_cif_date(line[21:29]), 'days': weekbool( line[29] == '1', # Monday line[30] == '1', # Tuesday line[31] == '1', # Wednesday line[32] == '1', # Thursday line[33] == '1', # Friday line[34] == '1', # Saturday line[35] == '1', # Sunday ), 'school-holidays': { 'S': 'term-time', 'H': 'holidays' }.get(line[36], 'all'), 'bank-holidays': { 'A': 'additional', 'B': 'holidays', 'X': 'non-holidays' }.get(line[37], 'all'), 'route': line[38:42], 'vehicle': line[48:56].strip(), 'direction': line[64], 'notes': [], 'stops': [], } elif line[:2] in ('QN', 'ZN'): # Notes this_journey['notes'].append(line[7:]) elif line[:2] == 'QO': # Journey start try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': None, 'std': self._parse_cif_time(line[14:18]), 'activity': 'O', 'estimated': line[22] == '0', 'fare-stage': line[24] == '1' }) except Http404: pass elif line[:2] == 'QI': # Journey intermediate stop try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': self._parse_cif_time(line[14:18]), 'std': self._parse_cif_time(line[18:22]), 'activity': line[22], 'estimated': line[27] == '0', 'fare-stage': line[29] == '1' }) except Http404: pass elif line[:2] == 'QT': # Journey complete try: this_journey['stops'].append({ 'entity': self._cache['atco:%s' % line[2:14].strip()], 'sta': self._parse_cif_time(line[14:18]), 'std': None, 'activity': 'F', 'estimated': line[22] == '0', 'fare-stage': line[24] == '1' }) except Http404: pass elif line[:2] == 'ZL': # Route ID route_id = line[2:] elif line[:2] == 'ZD': # Days route ID route_id += line[18:-1] elif line[:2] == 'ZS': # Route if this_journey is not None: routes[-1]['journies'].append(this_journey) routes.append({ 'id': route_id, 'number': line[10:14].strip(), 'description': line[14:-1], 'stops': [], 'journies': [] }) elif line[:2] == 'ZA': stop_code = line[3:15].strip() try: entity = self._cache['atco:%s' % stop_code] if entity.source == self._get_source(): # Raise Http404 if this is a bus stop we came up with, # so any name changes, etc, get processed raise Http404() except Http404: # Out of zone bus stops with NaPTAN codes try: entity = Entity.objects.get(source=self._get_source(), _identifiers__scheme='atco', _identifiers__value=stop_code) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) identifiers = { 'atco': stop_code } entity_type = self._entity_type entity.primary_type = entity_type entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=line[15:63].strip()) entity.all_types = (entity_type,) entity.update_all_types_completion() entity.save() routes[-1]['stops'].append(entity) if this_journey is not None: routes[-1]['journies'].append(this_journey) return routes
def _scrape(self, route, url, output): url += '&showall=1' service = etree.parse(urlopen(url), parser = etree.HTMLParser()) route.stops.clear() for i, tr in enumerate(service.find('.//table').findall('tr')[1:]): try: stop_code = tr[1][0].text except IndexError: # Stops on ACIS Live that don't have codes, e.g., out of county # stops stop_name = tr[3][0].text try: entity = Entity.objects.get(source=self._get_source(), _identifiers__scheme='acisroute', _identifiers__value=stop_name) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) entity_type = self._get_entity_type() entity.primary_type = entity_type identifiers = { 'acisroute': stop_name } entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=stop_name) entity.all_types = (entity_type,) entity.update_all_types_completion() else: if stop_code.startswith('693') or stop_code.startswith('272') \ or stop_code.startswith('734') or stop_code.startswith('282'): # Oxontime uses NaPTAN code scheme = 'naptan' elif stop_code.startswith('450'): # West Yorkshire uses plate code scheme = 'plate' else: # Everyone else uses ATCO scheme = 'atco' if stop_code.startswith('370'): # Except South Yorkshire, which mangles the code stop_code = '3700%s' % stop_code[3:] try: entity = get_entity(scheme, stop_code) if entity.source == self._get_source(): # Raise Http404 if this is a bus stop we came up with, # so any name changes, etc, get processed raise Http404() except Http404: # Out of zone bus stops with NaPTAN codes - alternatively, # the fake bus stops Oxontime made up for the TUBE route try: entity = Entity.objects.get(source=self._get_source(), _identifiers__scheme=scheme, _identifiers__value=stop_code) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) identifiers = {scheme: stop_code} entity_type = self._get_entity_type() entity.primary_type = entity_type entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=tr[3][0].text) entity.all_types = (entity_type,) entity.update_all_types_completion() entity.save() StopOnRoute.objects.create(route=route, entity=entity, order=i)
def endElement(self, name): if name in ('node', 'way') and self.valid: try: types = self.find_types(self.tags) except ValueError: self.ignore_count += 1 return # Ignore ways that lay partly outside our bounding box if name == 'way' and not all(id in self.node_locations for id in self.nodes): return # Ignore disused and under-construction entities if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get( 'disused') in ('1', 'yes', 'true'): return # Memory management in debug mode reset_queries() if self.id in self.identities: entity = get_entity(*self.identities[self.id].split(':')) entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict( zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } identifiers = entity.identifiers identifiers.update({'osm': self.id}) entity.save(identifiers=identifiers) entity.all_types = set(entity.all_types.all()) | set( self.entity_types[et] for et in types) entity.update_all_types_completion() self.ids.remove(self.id) else: try: entity = Entity.objects.get(source=self.source, _identifiers__scheme='osm', _identifiers__value=self.id) created = False except Entity.DoesNotExist: entity = Entity(source=self.source) created = True if not 'osm' in entity.metadata or \ entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']: if created: self.create_count += 1 else: self.modify_count += 1 if name == 'node': entity.location = Point(self.node_location, srid=4326) entity.geometry = entity.location elif name == 'way': cls = LinearRing if self.nodes[0] == self.nodes[ -1] else LineString entity.geometry = cls( [self.node_locations[n] for n in self.nodes], srid=4326) min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf')) for lon, lat in [ self.node_locations[n] for n in self.nodes ]: min_ = min(min_[0], lon), min(min_[1], lat) max_ = max(max_[0], lon), max(max_[1], lat) entity.location = Point((min_[0] + max_[0]) / 2, (min_[1] + max_[1]) / 2, srid=4326) else: raise AssertionError( "There should be no other types of entity we're to deal with." ) names = dict() for lang_code, lang_name in settings.LANGUAGES: with override(lang_code): if '-' in lang_code: tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator') else: tags_to_try = ('name:%s' % lang_code, 'name', 'operator') name = None for tag_to_try in tags_to_try: if self.tags.get(tag_to_try): name = self.tags.get(tag_to_try) break if name is None: try: name = reverse_geocode( *entity.location)[0]['name'] if not name: raise IndexError name = u"↝ %s" % name except IndexError: name = u"↝ %f, %f" % ( self.node_location[1], self.node_location[0]) names[lang_code] = name entity.metadata['osm'] = { 'attrs': dict(self.attrs), 'tags': dict( zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values())) } entity.primary_type = self.entity_types[types[0]] identifiers = entity.identifiers identifiers.update({'osm': self.id}) entity.save(identifiers=identifiers) for lang_code, name in names.items(): set_name_in_language(entity, lang_code, title=name) entity.all_types = [self.entity_types[et] for et in types] entity.update_all_types_completion() else: self.unchanged_count += 1
def _scrape(self, route, url, output): url += '&showall=1' service = etree.parse(urlopen(url), parser=etree.HTMLParser()) route.stops.clear() for i, tr in enumerate(service.find('.//table').findall('tr')[1:]): try: stop_code = tr[1][0].text except IndexError: # Stops on ACIS Live that don't have codes, e.g., out of county # stops stop_name = tr[3][0].text try: entity = Entity.objects.get( source=self._get_source(), _identifiers__scheme='acisroute', _identifiers__value=stop_name) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) entity_type = self._get_entity_type() entity.primary_type = entity_type identifiers = {'acisroute': stop_name} entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=stop_name) entity.all_types = (entity_type, ) entity.update_all_types_completion() else: if stop_code.startswith('693') or stop_code.startswith('272') \ or stop_code.startswith('734') or stop_code.startswith('282'): # Oxontime uses NaPTAN code scheme = 'naptan' elif stop_code.startswith('450'): # West Yorkshire uses plate code scheme = 'plate' else: # Everyone else uses ATCO scheme = 'atco' if stop_code.startswith('370'): # Except South Yorkshire, which mangles the code stop_code = '3700%s' % stop_code[3:] try: entity = get_entity(scheme, stop_code) if entity.source == self._get_source(): # Raise Http404 if this is a bus stop we came up with, # so any name changes, etc, get processed raise Http404() except Http404: # Out of zone bus stops with NaPTAN codes - alternatively, # the fake bus stops Oxontime made up for the TUBE route try: entity = Entity.objects.get( source=self._get_source(), _identifiers__scheme=scheme, _identifiers__value=stop_code) except Entity.DoesNotExist: entity = Entity(source=self._get_source()) identifiers = {scheme: stop_code} entity_type = self._get_entity_type() entity.primary_type = entity_type entity.save(identifiers=identifiers) set_name_in_language(entity, 'en', title=tr[3][0].text) entity.all_types = (entity_type, ) entity.update_all_types_completion() entity.save() StopOnRoute.objects.create(route=route, entity=entity, order=i)