def import_xcri_ox(force_update=False):
app = create_app()
url = app.config['XCRI_IMPORT_URL']
with app.blueprint_context(BLUEPRINT_NAME):
xcri = get_resource(url, force_update)
xcri_importer = XcriOxImporter(searcher, xcri, timeout=600)
xcri_importer.run()
def import_osm(previous_result=None, url=None, force_update=False):
"""Run the OSM importer if previous importer has succeeded
"""
if previous_result in (None, True):
try:
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['OSM_IMPORT_URL']
osm = get_resource(url, force_update)
if osm:
logger.info("OSM Downloaded - Stored here: %s" % osm)
osm = open(osm)
handler = OSMHandler(search_importer, 5)
parser = make_parser(['xml.sax.xmlreader.IncrementalParser'])
parser.setContentHandler(handler)
# Parse in 8k chunks
buffer = osm.read(8192)
bunzip = bz2.BZ2Decompressor()
while buffer:
parser.feed(bunzip.decompress(buffer))
buffer = osm.read(8192)
parser.close()
return True
else:
logger.info("OSM hasn't been imported - resource not loaded")
except:
logger.error("Error running OSM importer", exc_info=True)
return False
def import_sbb_stations(previous_result=None, url=None, force_update=False):
if previous_result not in [None, True]:
return False
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['SBB_IMPORT_URL']
db = get_resource(url, force_update)
if db:
sbb_importer = SbbStationImporter(searcher, 10, db, ['340'], 'identifiers')
sbb_importer.run()
else:
logger.info("SBB stations haven't been imported - resource not loaded")
return False
return True
def update_status(self, force_update=False):
xml = get_resource(self.url, force_update)
if not xml:
logger.warning('Failed to update river status')
return False
xml = etree.parse(open(xml), parser=etree.HTMLParser())
tbody = xml.getroot().find(".//div[@id='sidebar']/table")
status_map = {}
for i, tr in enumerate(tbody.findall('tr')):
name = tr[0].text.split(':')[0]
status = tr[1][0].attrib['src'][:-4].split('_')[-1]
status_map[name] = status
logger.info('River: %s - Status: %s' % (name, status))
self.set_status(status_map)
return True
def update_status(self, force_update=False):
xml = get_resource(self.url, force_update)
if not xml:
logger.warning('Failed to update river status')
return False
xml = etree.parse(open(xml), parser=etree.HTMLParser())
tbody = xml.getroot().find(".//div[@id='sidebar']/table")
status_map = {}
for i, tr in enumerate(tbody.findall('tr')):
name = tr[0].text.split(':')[0]
status = tr[1][0].attrib['src'][:-4].split('_')[-1]
status_map[name] = status
logger.info('River: %s - Status: %s' % (name, status))
self.set_status(status_map)
return True
def import_ox_library_data(previous_result=None, url=None, force_update=False):
if previous_result in (None, True):
try:
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['LIBRARY_DATA_IMPORT_URL']
library_data = get_resource(url, force_update)
if library_data:
file = open(library_data)
importer = OxLibraryDataImporter(search_importer, 10, file)
importer.run()
return True
else:
logger.info("OxLibraryData hasn't been imported - resource not loaded")
except:
logger.error("Error running OxLibraryData importer", exc_info=True)
return False
def import_naptan(previous_result=None, url=None, force_update=False):
if previous_result in (None, True):
try:
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['NAPTAN_IMPORT_URL']
naptan = get_resource(url, force_update)
if naptan:
archive = zipfile.ZipFile(open(naptan))
f = archive.open('NaPTAN.xml')
naptan = NaPTANImporter(search_importer, 10, f, ['340'], 'identifiers')
naptan.run()
return True
else:
logger.info("Naptan hasn't been imported - resource not loaded")
except:
logger.error("Error running NaPTAN importer", exc_info=True)
return False
def import_swiss_library_data(previous_result=None, url=None, force_update=False):
if previous_result not in [None, True]:
return False
app = create_app()
with app.blueprint_context(BLUEPRINT_NAME):
return True # XXX add some code
url = url or app.config['SWISS_LIBRARY_IMPORT_URL']
oxpoints = get_resource(url, force_update, media_type=RDF_MEDIA_TYPE)
if oxpoints:
logger.info("OxPoints Downloaded - Stored here: %s" % oxpoints)
oxpoints = open(oxpoints)
importer = OxpointsDescendantsImporter(kv_store, oxpoints, Org.subOrganizationOf,
rdf_media_type=RDF_MEDIA_TYPE)
importer.import_data()
else:
logger.info("OxPoints descendants hasn't been imported - resource not loaded")
return False
return True
def import_oxpoints_organisation_descendants(previous_result=None, url=None, force_update=False):
"""Run the OxPoints organisation descendants importer
"""
if previous_result in (None, True):
try:
app = create_app()
RDF_MEDIA_TYPE = 'text/turtle' # default RDF serialization
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['OXPOINTS_IMPORT_URL']
oxpoints = get_resource(url, force_update, media_type=RDF_MEDIA_TYPE)
if oxpoints:
logger.info("OxPoints Downloaded - Stored here: %s" % oxpoints)
oxpoints = open(oxpoints)
importer = OxpointsDescendantsImporter(kv_store, oxpoints, Org.subOrganizationOf,
rdf_media_type=RDF_MEDIA_TYPE)
importer.import_data()
return True
else:
logger.info("OxPoints descendants hasn't been imported - resource not loaded")
except:
logger.error("Error running OxPoints descendants importer", exc_info=True)
return False
def import_oxpoints(previous_result=None, url=None, force_update=False):
"""Run the OxPoints importer
"""
if previous_result in (None, True):
try:
app = create_app()
RDF_MEDIA_TYPE = 'text/turtle' # default RDF serialization
with app.blueprint_context(BLUEPRINT_NAME):
url = url or app.config['OXPOINTS_IMPORT_URL']
static_files_dir = app.config.get('STATIC_FILES_IMPORT_DIRECTORY', None)
oxpoints = get_resource(url, force_update, media_type=RDF_MEDIA_TYPE)
if 'OXPOINTS_SHAPES_URL' in app.config:
url_shapes = app.config['OXPOINTS_SHAPES_URL']
oxpoints_shape = get_resource(url_shapes, force_update, media_type=RDF_MEDIA_TYPE)
else:
oxpoints_shape = None
if 'OXPOINTS_ACCESSIBILITY_URL' in app.config:
url_accessibility = app.config['OXPOINTS_ACCESSIBILITY_URL']
oxpoints_accessibility = get_resource(url_accessibility,
force_update=force_update,
media_type=RDF_MEDIA_TYPE)
else:
oxpoints_accessibility = None
if 'OXPOINTS_COURSES_LOCATIONS_URL' in app.config:
url_courses = app.config['OXPOINTS_COURSES_LOCATIONS_URL']
oxpoints_courses = get_resource(url_courses,
force_update=force_update,
media_type=RDF_MEDIA_TYPE)
else:
oxpoints_courses = None
if oxpoints:
logger.info("OxPoints Downloaded - Stored here: %s" % oxpoints)
oxpoints = open(oxpoints)
if oxpoints_shape:
shapes = open(oxpoints_shape)
else:
shapes = None
if oxpoints_accessibility:
accessibility = open(oxpoints_accessibility)
else:
accessibility = None
if oxpoints_courses:
courses = open(oxpoints_courses)
else:
courses = None
importer = OxpointsImporter(search_importer, 10, oxpoints, shapes, accessibility, courses, static_files_dir,
rdf_media_type=RDF_MEDIA_TYPE)
importer.import_data()
return True
else:
logger.info("OxPoints hasn't been imported - resource not loaded")
except:
logger.error("Error running OxPoints importer", exc_info=True)
return False