def _get_entity_types(self):
entity_types = {}
new_entity_types = set()
for slug, et in self._entity_types.items():
et_category, created = EntityTypeCategory.objects.get_or_create(name=et['category'])
try:
entity_type = EntityType.objects.get(slug=slug)
created = False
except EntityType.DoesNotExist:
entity_type = EntityType(slug=slug)
created = True
entity_type.category = et_category
entity_type.slug = slug
if created:
entity_type.show_in_nearby_list = et['show_in_nearby_list']
entity_type.show_in_category_list = et['show_in_category_list']
entity_type.save()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
set_name_in_language(entity_type, lang_code,
verbose_name=_(et['verbose_name']),
verbose_name_singular=_(et['verbose_name_singular']),
verbose_name_plural=_(et['verbose_name_plural']))
new_entity_types.add(slug)
entity_types[slug] = entity_type
for slug in new_entity_types:
subtype_of = self._entity_types[slug]['parent-types']
entity_types[slug].subtype_of.clear()
for s in subtype_of:
entity_types[slug].subtype_of.add(entity_types[s])
entity_types[slug].save()
return entity_types
def _get_entity_type(self):
category, created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Uncategorised'))
entity_type, created = EntityType.objects.get_or_create(
slug='post-code', category=category)
entity_type.slug = 'post-code'
if created:
entity_type.show_in_nearby_list = False
entity_type.show_in_category_list = False
entity_type.save()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
set_name_in_language(entity_type, lang_code,
verbose_name=_('postcode'),
verbose_name_singular=_('a postcode'),
verbose_name_plural=_('postcodes'))
return entity_type
def _get_entity_types(self):
entity_types = {}
category, created = EntityTypeCategory.objects.get_or_create(name=_("Transport"))
category.save()
for stop_type in self.entity_type_definitions:
et = self.entity_type_definitions[stop_type]
try:
entity_type = EntityType.objects.get(slug=et["slug"])
except EntityType.DoesNotExist:
entity_type = EntityType(slug=et["slug"])
entity_type.category = category
entity_type.uri = "http://mollyproject.org/schema/maps#%s" % et["uri-local"]
if created:
entity_type.show_in_nearby_list = et["nearby"]
entity_type.show_in_category_list = et["category"]
entity_type.save()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
set_name_in_language(
entity_type,
lang_code,
verbose_name=ugettext(et["verbose-name"]),
verbose_name_singular=ugettext(et["verbose-name-singular"]),
verbose_name_plural=ugettext(et["verbose-name-plural"]),
)
entity_types[stop_type] = entity_type
for stop_type, entity_type in entity_types.items():
if entity_type.slug == "public-transport-access-node":
continue
entity_type.subtype_of.add(entity_types[None])
if (
stop_type.startswith("MET")
and stop_type != "MET"
and entity_type.slug != self.RAIL_STATION_DEFINITION["slug"]
):
entity_type.subtype_of.add(entity_types["MET"])
return entity_types
def _get_entity_type(self):
try:
entity_type = EntityType.objects.get(slug='travel-alert')
created = False
except EntityType.DoesNotExist:
entity_type = EntityType(slug='travel-alert')
created = True
category, etc_created = EntityTypeCategory.objects.get_or_create(name=ugettext_noop('Transport'))
if created:
entity_type.show_in_nearby_list = False
entity_type.show_in_category_list = False
entity_type.category = category
entity_type.save()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
set_name_in_language(entity_type, lang_code,
verbose_name=_('travel alert'),
verbose_name_singular=_('a travel alert'),
verbose_name_plural=_('travel alerts'))
return entity_type
def import_data(self, metadata, output):
"Imports places data from OpenStreetMap"
old_etag = metadata.get('etag', '')
request = AnyMethodRequest(self._url, method='HEAD')
response = urllib2.urlopen(request)
new_etag = response.headers['ETag'][1:-1]
self.output = output
if not settings.DEBUG and new_etag == old_etag:
output.write('OSM data not updated. Not updating.\n')
return
parser = make_parser(['xml.sax.xmlreader.IncrementalParser'])
parser.setContentHandler(OSMHandler(self._get_source(),
self._get_entity_types(),
lambda tags, type_list=None: self._find_types(tags, self._osm_tags if type_list is None else type_list),
output,
self._lat_north,
self._lat_south,
self._lon_west,
self._lon_east))
# Parse in 8k chunks
osm = urllib2.urlopen(self._url)
buffer = osm.read(8192)
bunzip = bz2.BZ2Decompressor()
while buffer:
parser.feed(bunzip.decompress(buffer))
buffer = osm.read(8192)
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
self.disambiguate_titles(self._get_source())
return {
'etag': new_etag,
}
def endElement(self, name):
if name in ('node', 'way') and self.valid:
try:
types = self.find_types(self.tags)
except ValueError:
self.ignore_count += 1
return
# Ignore ways that lay partly outside our bounding box
if name == 'way' and not all(id in self.node_locations for id in self.nodes):
return
# We already have these from OxPoints, so leave them alone.
if self.tags.get('amenity') == 'library' and self.tags.get('operator') == 'University of Oxford':
return
# Ignore disused and under-construction entities
if self.tags.get('life_cycle', 'in_use') != 'in_use' or self.tags.get('disused') in ('1', 'yes', 'true'):
return
reset_queries()
try:
entity = Entity.objects.get(source=self.source,
_identifiers__scheme='osm',
_identifiers__value=self.id)
created = True
except Entity.DoesNotExist:
entity = Entity(source=self.source)
created = False
if not 'osm' in entity.metadata or \
entity.metadata['osm'].get('attrs', {}).get('timestamp', '') < self.attrs['timestamp']:
if created:
self.create_count += 1
else:
self.modify_count += 1
if name == 'node':
entity.location = Point(self.node_location, srid=4326)
entity.geometry = entity.location
elif name == 'way':
cls = LinearRing if self.nodes[0] == self.nodes[-1] else LineString
entity.geometry = cls([self.node_locations[n] for n in self.nodes], srid=4326)
min_, max_ = (float('inf'), float('inf')), (float('-inf'), float('-inf'))
for lon, lat in [self.node_locations[n] for n in self.nodes]:
min_ = min(min_[0], lon), min(min_[1], lat)
max_ = max(max_[0], lon), max(max_[1], lat)
entity.location = Point((min_[0] + max_[0]) / 2 , (min_[1] + max_[1]) / 2 , srid=4326)
else:
raise AssertionError("There should be no other types of entity we're to deal with.")
names = dict()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
if '-' in lang_code:
tags_to_try = ('name:%s' % lang_code, 'name:%s' % lang_code.split('-')[0], 'name', 'operator')
else:
tags_to_try = ('name:%s' % lang_code, 'name', 'operator')
name = None
for tag_to_try in tags_to_try:
if self.tags.get(tag_to_try):
name = self.tags.get(tag_to_try)
break
if name is None:
try:
name = reverse_geocode(*entity.location)[0]['name']
if not name:
raise IndexError
name = u"↝ %s" % name
except IndexError:
name = u"↝ %f, %f" % (self.node_location[1], self.node_location[0])
names[lang_code] = name
entity.metadata['osm'] = {
'attrs': dict(self.attrs),
'tags': dict(zip((k.replace(':', '_') for k in self.tags.keys()), self.tags.values()))
}
entity.primary_type = self.entity_types[types[0]]
entity.save(identifiers={'osm': self.id})
for lang_code, name in names.items():
set_name_in_language(entity, lang_code, title=name)
entity.all_types = [self.entity_types[et] for et in types]
entity.update_all_types_completion()
else:
self.unchanged_count += 1
def add_stop(self, meta, entity_type, source):
# Check this entity is in an area
if self.areas != None:
in_area = False
for area in self.areas:
if meta["atco-code"].startswith(area):
in_area = True
if not in_area:
return
# See if we're updating an existing object, or creating a new one
try:
entity = Entity.objects.get(
source=source, _identifiers__scheme="atco", _identifiers__value=meta["atco-code"]
)
except Entity.DoesNotExist:
entity = Entity(source=source)
except Entity.MultipleObjectsReturned:
# Handle clashes
Entity.objects.filter(
source=source, _identifiers__scheme="atco", _identifiers__value=meta["atco-code"]
).delete()
entity = Entity(source=source)
common_name, indicator, locality, street = [
meta.get(k) for k in ("common-name", "indicator", "locality-ref", "street")
]
if (
(common_name or "").endswith(" DEL")
or (indicator or "").lower() == "not in use"
or "to define route" in (common_name or "")
or "to def rte" in (common_name or "")
or "to def route" in (common_name or "")
or "def.rte" in (common_name or "")
):
# In the NaPTAN list, but indicates it's an unused stop
return
if self.meta["stop-type"] in ("MET", "GAT", "FER", "RLY"):
names = self.names
else:
names = dict()
for lang_code, lang_name in settings.LANGUAGES:
with override(lang_code):
# Try and find one in our preferred order
for lang in (lang_code, "en", None):
if lang in self.names:
common_name = self.names[lang]
break
# Expand abbreviations in indicators
if indicator is not None:
parts = []
for part in indicator.split():
parts.append(
{
# Translators: This is referring to bus stop location descriptions
"op": ugettext("Opposite"),
"opp": ugettext("Opposite"),
"opposite": ugettext("Opposite"),
# Translators: This is referring to bus stop location descriptions
"adj": ugettext("Adjacent"),
# Translators: This is referring to bus stop location descriptions
"outside": ugettext("Outside"),
"o/s": ugettext("Outside"),
# Translators: This is referring to bus stop location descriptions
"nr": ugettext("Near"),
# Translators: This is referring to bus stop location descriptions
"inside": ugettext("Inside"),
# Translators: This is referring to bus stop location descriptions
"stp": ugettext("Stop"),
}.get(part.lower(), part)
)
indicator = " ".join(parts)
if indicator is None and self.meta["stop-type"] in ("AIR", "FTD", "RSE", "TMU", "BCE"):
# Translators: This is referring to public transport entities
title = ugettext("Entrance to %s") % common_name
elif indicator is None and self.meta["stop-type"] in ("FBT",):
# Translators: This is referring to ferry ports
title = ugettext("Berth at %s") % common_name
elif indicator is None and self.meta["stop-type"] in ("RPL", "PLT"):
# Translators: This is referring to rail and metro stations
title = ugettext("Platform at %s") % common_name
elif indicator is not None and indicator.lower() != "none" and indicator not in common_name:
title = indicator + " " + common_name
else:
title = common_name
if street != None and street != "-":
# Deal with all-caps street names
if street.upper() == street:
fixedstreet = ""
wordstart = True
for letter in street:
if wordstart:
wordstart = False
fixedstreet += letter
continue
elif letter == " ":
wordstart = True
fixedstreet += letter
continue
else:
fixedstreet += letter.lower()
street = fixedstreet
if street not in title:
title += ", " + street
locality_lang = self.nptg_localities.get(locality)
if locality_lang != None:
for lang in (lang_code, "en", "cy"):
if lang in locality_lang:
if locality_lang[lang] != street:
title += ", " + locality_lang[lang]
break
names[lang_code] = title
entity.primary_type = entity_type
if not entity.metadata:
entity.metadata = {}
entity.metadata["naptan"] = meta
entity.location = Point(float(meta["longitude"]), float(meta["latitude"]), srid=4326)
entity.geometry = entity.location
if meta["atco-code"] in self.tube_references:
entity.metadata["london-underground-identifiers"] = self.tube_references[meta["atco-code"]]
identifiers = {"atco": meta["atco-code"]}
if "naptan-code" in meta:
meta["naptan-code"] = "".join(map(self.naptan_dial, meta["naptan-code"]))
identifiers["naptan"] = meta["naptan-code"]
if "plate-code" in meta:
identifiers["plate"] = meta["plate-code"]
if "crs" in meta:
identifiers["crs"] = meta["crs"]
if indicator != None and re.match("Stop [A-Z]\d\d?", indicator):
identifiers["stop"] = indicator[5:]
entity.save(identifiers=identifiers)
for lang_code, name in names.items():
# This is the NaPTAN, so default to English
if lang_code is None:
lang_code = "en"
set_name_in_language(entity, lang_code, title=name)
entity.all_types = (entity_type,)
entity.update_all_types_completion()
entity.groups.clear()
for stop_area in self.stop_areas:
sa, created = EntityGroup.objects.get_or_create(source=source, ref_code=stop_area)
entity.groups.add(sa)
entity.save()
return entity