def update_or_create_from_xml(self, el,
default_language=settings.LANGUAGE_CODE, base_url='',
save=True):
""" Returns a tuple of (CREATED/UPDATED/UNMODIFIED, model_obj)"""
el_hash = hashlib.md5(etree.tostring(el)).hexdigest()
el = deepcopy(el)
jurisdiction_id, obj_id = el.findtext('id').split('/')
if not (jurisdiction_id and obj_id):
raise ValidationError(u"%s is not a valid Open511 ID" % el.findtext('id'))
el.remove(el.xpath('id')[0])
external_jurisdiction = el.xpath('link[@rel="jurisdiction"]')
if external_jurisdiction:
el.remove(external_jurisdiction[0])
try:
jurisdiction = Jurisdiction.objects.get(id=jurisdiction_id)
except Jurisdiction.DoesNotExist:
if not external_jurisdiction:
raise Exception("No jurisdiction URL provided for %s" % jurisdiction_id)
jurisdiction = Jurisdiction.objects.get_or_create_from_url(
urljoin(base_url, external_jurisdiction[0].get('href')))
try:
obj = self.get(id=obj_id, jurisdiction=jurisdiction)
if obj.last_import_hash == el_hash:
return ('UNMODIFIED', obj)
created = False
except ObjectDoesNotExist:
created = True
obj = self.model(id=obj_id, jurisdiction=jurisdiction, last_import_hash=el_hash)
self_link = el.xpath('link[@rel="self"]')
if self_link:
obj.external_url = urljoin(base_url, self_link[0].get('href'))
el.remove(self_link[0])
# Extract the geometry
geometry = el.xpath('geography')[0]
gml = etree.tostring(geometry[0], encoding='unicode')
ewkt = gml_to_ewkt(gml, force_2D=True)
obj.geom = geos_geom_from_string(ewkt)
# And regenerate the GML so it's consistent with the PostGIS representation
el.remove(geometry)
el.append(E.geography(geom_to_xml_element(obj.geom)))
# Push down the default language if necessary
if not el.get(XML_LANG):
el.set(XML_LANG, default_language)
obj.xml_elem = el
if save:
obj.save()
return ('CREATED' if created else 'UPDATED', obj)
def get_roadevent_from_summary(summary):
elem = E.event(E.id("%s/%s" % (JURISDICTION_ID, summary['id'])))
elem.append(
E.geography(
geom_to_xml_element(
Point(float(summary['lng']), float(summary['lat']), srid=4326)
)
)
)
url = BASE_DETAIL_URL + summary['id']
resp = urlopen(url)
time.sleep(0.5)
def set_val(tag, val):
if val not in (None, ''):
e = etree.Element(tag)
e.text = str(val)
elem.append(e)
set_val('status', 'ACTIVE')
set_val('event_type', 'CONSTRUCTION')
set_val('severity', 'MAJOR' if summary['id'].startswith('maj') else 'MODERATE')
root = lxml.html.fragment_fromstring(resp.read().decode('utf8'))
set_val('headline', _get_text_from_elems(root.cssselect('#tdIdentification')))
description = _get_text_from_elems(root.cssselect('#tdDescriptionEntrave,#tdDetail'))
affected_roads = _get_text_from_elems(root.cssselect('#tdLocalisation'))
traffic_restrictions = _get_text_from_elems(root.cssselect('#tdRestrictionCamionnage'))
if affected_roads:
description += u'\n\nLocalisation: ' + affected_roads
if traffic_restrictions:
description += u'\n\nRestrictions: ' + traffic_restrictions
set_val('description', description)
start_date = _get_text_from_elems(root.cssselect('#tdDebut'))
end_date = _get_text_from_elems(root.cssselect('#tdFin'))
if start_date:
sked = E.recurring_schedule()
sked.append(E.start_date(str(_str_to_date(start_date))))
if end_date:
sked.append(E.end_date(str(_str_to_date(end_date))))
elem.append(E.schedule(E.recurring_schedules(sked)))
return elem
def main():
logging.basicConfig()
base = get_base_open511_element(lang='fr')
cameras = E.cameras()
base.append(cameras)
english_names = get_english_names()
for camera_info in get_list_of_cameras():
camera = E.camera(
E.id(JURISDICTION_ID + '/' + unicode(camera_info['id'])),
E.name(camera_info['info'])
)
if english_names[camera_info['id']]:
ename = E.name(english_names[camera_info['id']])
ename.set(XML_LANG, 'en')
camera.append(ename)
camera.append(
E.geography(
geom_to_xml_element(
Point(float(camera_info['lng']), float(camera_info['lat']), srid=4326)
)
)
)
try:
camera.append(
E.media_files(
E.link(rel="related", href=get_image_url(camera_info['id']), type="image/jpeg")
)
)
except Exception as e:
logger.exception("Couldn't fetch image for camera #%s" % camera_info['id'])
continue
cameras.append(camera)
print etree.tostring(base, pretty_print=True)
def to_full_xml_element(self):
return E.geography(geom_to_xml_element(self.geom))
def feature_to_open511_element(feature):
"""Transform an OGR Feature from the KML input into an XML Element for a RoadEvent."""
# Using a hash of the geometry for an ID. For proper production use,
# there'll probably have to be some code in the importer
# that compares to existing entries in the DB to determine whether
# this is new or modified...
id = hashlib.md5(feature.geom.wkt.encode('ascii')).hexdigest()
while id in ids_seen:
id += 'x'
ids_seen.add(id)
elem = E.event()
def set_val(tag, val):
if val not in (None, ''):
e = etree.Element(tag)
e.text = str(val)
elem.append(e)
def maybe_decode(s):
if isinstance(s, str):
return s
return s.decode('utf8')
set_val('id', "%s/%s" % (JURISDICTION_ID, id))
set_val('status', 'ACTIVE')
set_val('event_type', 'CONSTRUCTION')
set_val('severity', 'UNKNOWN')
set_val('headline', maybe_decode(feature.get('Name')))
blob = lxml.html.fragment_fromstring(maybe_decode(feature.get('Description')),
create_parent='content')
description_label = blob.xpath('//strong[text()="Description"]')
localisation = blob.cssselect('div#localisation p')
if description_label or localisation:
description_bits = []
if description_label:
el = description_label[0].getnext()
while el.tag == 'p':
description_bits.append(_get_el_text(el))
el = el.getnext()
if localisation:
description_bits.append('Localisation: ' + '; '.join(
_get_el_text(el) for el in localisation))
set_val('description', '\n\n'.join(description_bits))
try:
link = blob.cssselect('#avis_residants a, #en_savoir_plus a')[0]
e = etree.Element('link')
e.set('rel', 'related')
e.set('href', link.get('href'))
if link.get('title'):
e.set('title', link.get('title'))
elem.append(E.attachments(e))
except IndexError:
pass
facultatif = blob.cssselect('div#itineraire_facult p')
if facultatif:
set_val('detour', '\n\n'.join(_get_el_text(el) for el in facultatif))
if blob.cssselect('div#dates strong'):
try:
start_date = blob.xpath(u'div[@id="dates"]/strong[text()="Date de d\xe9but"]')[0].tail
end_date = blob.xpath(u'div[@id="dates"]/strong[text()="Date de fin"]')[0].tail
start_date = _fr_string_to_date(start_date)
end_date = _fr_string_to_date(end_date)
if start_date:
sked = E.recurring_schedule(E.start_date(str(start_date)))
if end_date:
sked.append(E.end_date(str(end_date)))
elem.append(E.schedule(E.recurring_schedules(sked)))
except IndexError:
pass
elem.append(E.geography(
geom_to_xml_element(feature.geom)
))
return elem