コード例 #1
0
ファイル: bbc_tpeg.py プロジェクト: alexliyu/mobilesystem
 def import_data(self, metadata, output):
     source, entity_type = self._get_source(), self._get_entity_type()
     
     parser = etree.XMLParser(load_dtd=True)
     parser.resolvers.add(BBCTPEGResolver())
     xml = etree.parse(urllib.urlopen(self._tpeg_url), parser=parser)
     
     entities, seen = {}, set()
     for entity in Entity.objects.filter(source=source):
         if 'bbc-tpeg' in entity.identifiers:
             entities[entity.identifiers['bbc-tpeg']] = entity
     
     for message in xml.getroot().findall('tpeg_message'):
         id = message.find('road_traffic_message').attrib['message_id']
         road_traffic_message = message.find('road_traffic_message')
         
         try:
             entity = entities[id]
         except KeyError:
             entity = Entity()
             entities[id] = entity
         
         entity.source = source
         entity.primary_type = entity_type
         
         locs = map(self._wgs84_to_point, road_traffic_message.findall('location_container/location_coordinates/WGS84'))
         if len(locs) > 1:
             entity.geometry = LineString(*locs)
         elif len(locs) == 1:
             entity.geometry = locs[0]
         else:
             continue
         entity.location = Point(
             sum(p.x for p in locs) / len(locs),
             sum(p.y for p in locs) / len(locs),
             srid=4326,
         )
         
         entity.metadata['bbc_tpeg'] = {
             'xml': etree.tostring(message),
             'severity': road_traffic_message.attrib['severity_factor'],
             'generated': road_traffic_message.attrib['message_generation_time'],
             'version': int(road_traffic_message.attrib['version_number']),
         }
         
         entity.save(identifiers={'bbc-tpeg': id})
         set_name_in_language(entity, 'en',
                              title=message.find('summary').text)
         entity.all_types = [entity_type]
         entity.update_all_types_completion()
         seen.add(entity.pk)
     
     for entity in Entity.objects.filter(source=source):
         if not entity.pk in seen:
             entity.delete()
コード例 #2
0
ファイル: postcodes.py プロジェクト: alexliyu/mobilesystem
 def _load_from_csv(self, reader, entity_type, source):
     j = 0
     for i, line in enumerate(reader):
         postcode_abbrev, (easting, northing) = line[0], line[10:12]
         postcode_abbrev = postcode_abbrev.replace(' ', '')
         
         # Now try to figure out where to put the space in
         if re.match(r'[A-Z][0-9]{2}[A-Z]{2}', postcode_abbrev):
             # A9 9AA
             postcode = '%s %s' % (postcode_abbrev[:2], postcode_abbrev[2:])
         elif re.match(r'[A-Z][0-9]{3}[A-Z]{2}', postcode_abbrev):
             # A99 9AA
             postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
         elif re.match(r'[A-Z]{2}[0-9]{2}[A-Z]{2}', postcode_abbrev):
             # AA9 9AA
             postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
         elif re.match(r'[A-Z]{2}[0-9]{3}[A-Z]{2}', postcode_abbrev):
             # AA99 9AA
             postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:])
         elif re.match(r'[A-Z][0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev):
             # A9A 9AA
             postcode = '%s %s' % (postcode_abbrev[:3], postcode_abbrev[3:])
         elif re.match(r'[A-Z]{2}[0-9][A-Z][0-9][A-Z]{2}', postcode_abbrev):
             # AA9A 9AA
             postcode = '%s %s' % (postcode_abbrev[:4], postcode_abbrev[4:])
         else:
             postcode = postcode_abbrev
         
         try:
             easting, northing = int(easting), int(northing)
         except ValueError:
             continue
             
         j += 1
         
         try:
             entity = Entity.objects.get(source=source, _identifiers__scheme='postcode', _identifiers__value=postcode_abbrev)
         except Entity.DoesNotExist:
             entity = Entity(source=source)
         
         entity.location = Point(easting, northing, srid=3857)
         entity.geometry = entity.location
         entity.primary_type = entity_type
         
         identifiers = {
             'postcode': postcode_abbrev,
             'postcode-canonical': postcode,
         }
         entity.save(identifiers=identifiers)
         set_name_in_language(entity, 'en',
                              title=postcode)
         entity.all_types.add(entity_type)
         entity.update_all_types_completion()
コード例 #3
0
ファイル: osm.py プロジェクト: alexliyu/mobilesystem
 def endElement(self, name):
     if name in ('node', 'way') and self.valid:
         try:
             types = self.find_types(self.tags)
         except ValueError:
             self.ignore_count += 1
             return
         
         # Ignore ways that lay partly outside our bounding box
         if name == 'way' and not all(id in self.node_locations for id in self.nodes):
             return
         
         # We already have these from OxPoints, so leave them alone.
         if self.tags.get('amenity') == 'library' and self.tags.get('operator') == 'University of Oxford':
             return
         
         # Ignore disused and under-construction entities
         if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get('disused') in ('1', 'yes', 'true'):
             return
         
         reset_queries()
         
         try:
             entity = Entity.objects.get(source=self.source,
                                         _identifiers__scheme='osm',
                                         _identifiers__value=self.id)
             created = True
         except Entity.DoesNotExist:
             entity = Entity(source=self.source)
             created = False
         
         if not 'osm' in entity.metadata or \
           entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']:
             
             if created:
                 self.create_count += 1
             else:
                 self.modify_count += 1
             
             if name == 'node':
                 entity.location = Point(self.node_location, srid=4326)
                 entity.geometry = entity.location
             elif name == 'way':
                 cls = LinearRing if self.nodes[0] == self.nodes[-1] else LineString
                 entity.geometry = cls([self.node_locations[n] for n in self.nodes], srid=4326)
                 min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf'))
                 for lon, lat in [self.node_locations[n] for n in self.nodes]:
                     min_ = min(min_[0], lon), min(min_[1], lat) 
                     max_ = max(max_[0], lon), max(max_[1], lat)
                 entity.location = Point((min_[0] + max_[0]) / 2 , (min_[1] + max_[1]) / 2 , srid=4326)
             else:
                 raise AssertionError("There should be no other types of entity we're to deal with.")
             
             names = dict()
             
             for lang_code, lang_name in settings.LANGUAGES:
                 with override(lang_code):
                 
                     if '-' in lang_code:
                         tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator')
                     else:
                         tags_to_try = ('name:%s' % lang_code, 'name', 'operator')
                         name = None
                         for tag_to_try in tags_to_try:
                             if self.tags.get(tag_to_try):
                                 name = self.tags.get(tag_to_try)
                                 break
                     
                     if name is None:
                         try:
                             name = reverse_geocode(*entity.location)[0]['name']
                             if not name:
                                 raise IndexError
                             name = u"↝ %s" % name
                         except IndexError:
                             name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0])
                     
                     names[lang_code] = name
             
             entity.metadata['osm'] = {
                 'attrs': dict(self.attrs),
                 'tags': dict(zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values()))
             }
             entity.primary_type = self.entity_types[types[0]]
             
             entity.save(identifiers={'osm': self.id})
             
             for lang_code, name in names.items():
                 set_name_in_language(entity, lang_code, title=name)
             
             entity.all_types = [self.entity_types[et] for et in types]
             entity.update_all_types_completion()
         
         else:
             self.unchanged_count += 1
コード例 #4
0
ファイル: naptan.py プロジェクト: alexliyu/mobilesystem
    def add_stop(self, meta, entity_type, source):

        # Check this entity is in an area
        if self.areas != None:
            in_area = False
            for area in self.areas:
                if meta["atco-code"].startswith(area):
                    in_area = True
            if not in_area:
                return

        # See if we're updating an existing object, or creating a new one
        try:
            entity = Entity.objects.get(
                source=source, _identifiers__scheme="atco", _identifiers__value=meta["atco-code"]
            )
        except Entity.DoesNotExist:
            entity = Entity(source=source)
        except Entity.MultipleObjectsReturned:
            # Handle clashes
            Entity.objects.filter(
                source=source, _identifiers__scheme="atco", _identifiers__value=meta["atco-code"]
            ).delete()
            entity = Entity(source=source)

        common_name, indicator, locality, street = [
            meta.get(k) for k in ("common-name", "indicator", "locality-ref", "street")
        ]

        if (
            (common_name or "").endswith(" DEL")
            or (indicator or "").lower() == "not in use"
            or "to define route" in (common_name or "")
            or "to def rte" in (common_name or "")
            or "to def route" in (common_name or "")
            or "def.rte" in (common_name or "")
        ):
            # In the NaPTAN list, but indicates it's an unused stop
            return

        if self.meta["stop-type"] in ("MET", "GAT", "FER", "RLY"):
            names = self.names
        else:

            names = dict()

            for lang_code, lang_name in settings.LANGUAGES:
                with override(lang_code):

                    # Try and find one in our preferred order
                    for lang in (lang_code, "en", None):
                        if lang in self.names:
                            common_name = self.names[lang]
                            break

                    # Expand abbreviations in indicators
                    if indicator is not None:
                        parts = []
                        for part in indicator.split():
                            parts.append(
                                {
                                    # Translators: This is referring to bus stop location descriptions
                                    "op": ugettext("Opposite"),
                                    "opp": ugettext("Opposite"),
                                    "opposite": ugettext("Opposite"),
                                    # Translators: This is referring to bus stop location descriptions
                                    "adj": ugettext("Adjacent"),
                                    # Translators: This is referring to bus stop location descriptions
                                    "outside": ugettext("Outside"),
                                    "o/s": ugettext("Outside"),
                                    # Translators: This is referring to bus stop location descriptions
                                    "nr": ugettext("Near"),
                                    # Translators: This is referring to bus stop location descriptions
                                    "inside": ugettext("Inside"),
                                    # Translators: This is referring to bus stop location descriptions
                                    "stp": ugettext("Stop"),
                                }.get(part.lower(), part)
                            )
                        indicator = " ".join(parts)

                    if indicator is None and self.meta["stop-type"] in ("AIR", "FTD", "RSE", "TMU", "BCE"):
                        # Translators: This is referring to public transport entities
                        title = ugettext("Entrance to %s") % common_name

                    elif indicator is None and self.meta["stop-type"] in ("FBT",):
                        # Translators: This is referring to ferry ports
                        title = ugettext("Berth at %s") % common_name

                    elif indicator is None and self.meta["stop-type"] in ("RPL", "PLT"):
                        # Translators: This is referring to rail and metro stations
                        title = ugettext("Platform at %s") % common_name

                    elif indicator is not None and indicator.lower() != "none" and indicator not in common_name:
                        title = indicator + " " + common_name

                    else:
                        title = common_name

                    if street != None and street != "-":
                        # Deal with all-caps street names
                        if street.upper() == street:
                            fixedstreet = ""
                            wordstart = True
                            for letter in street:
                                if wordstart:
                                    wordstart = False
                                    fixedstreet += letter
                                    continue
                                elif letter == " ":
                                    wordstart = True
                                    fixedstreet += letter
                                    continue
                                else:
                                    fixedstreet += letter.lower()
                            street = fixedstreet

                        if street not in title:
                            title += ", " + street

                    locality_lang = self.nptg_localities.get(locality)
                    if locality_lang != None:
                        for lang in (lang_code, "en", "cy"):
                            if lang in locality_lang:
                                if locality_lang[lang] != street:
                                    title += ", " + locality_lang[lang]
                                break

                    names[lang_code] = title

        entity.primary_type = entity_type

        if not entity.metadata:
            entity.metadata = {}
        entity.metadata["naptan"] = meta
        entity.location = Point(float(meta["longitude"]), float(meta["latitude"]), srid=4326)
        entity.geometry = entity.location

        if meta["atco-code"] in self.tube_references:
            entity.metadata["london-underground-identifiers"] = self.tube_references[meta["atco-code"]]

        identifiers = {"atco": meta["atco-code"]}
        if "naptan-code" in meta:
            meta["naptan-code"] = "".join(map(self.naptan_dial, meta["naptan-code"]))
            identifiers["naptan"] = meta["naptan-code"]
        if "plate-code" in meta:
            identifiers["plate"] = meta["plate-code"]
        if "crs" in meta:
            identifiers["crs"] = meta["crs"]
        if indicator != None and re.match("Stop [A-Z]\d\d?", indicator):
            identifiers["stop"] = indicator[5:]

        entity.save(identifiers=identifiers)

        for lang_code, name in names.items():
            # This is the NaPTAN, so default to English
            if lang_code is None:
                lang_code = "en"
            set_name_in_language(entity, lang_code, title=name)

        entity.all_types = (entity_type,)
        entity.update_all_types_completion()
        entity.groups.clear()
        for stop_area in self.stop_areas:
            sa, created = EntityGroup.objects.get_or_create(source=source, ref_code=stop_area)
            entity.groups.add(sa)
        entity.save()

        return entity