def date_range(timeline):
"""Given a timeline, determine the start and end dates.
The timeline may be sparse (containing fewer entries than all the dates
between the min and max dates) and since it is a dict,
the dates may be in any order.
:param timeline: A dictionary of non-sequential dates (in YYYY-MM-DD) as
keys and values (representing ways collected on that day).
:type timeline: dict
:returns: A tuple containing two dates:
* start_date - a date object representing the earliest date in the
time line.
* end_date - a date object representing the newest date in the time
line.
:rtype: (date, date)
"""
start_date = None
end_date = None
for next_date in timeline.keys():
year, month, day = next_date.split('-')
message = 'Date: %s' % next_date
LOGGER.info(message)
timeline_date = date(int(year), int(month), int(day))
if start_date is None:
start_date = timeline_date
if end_date is None:
end_date = timeline_date
if timeline_date < start_date:
start_date = timeline_date
if timeline_date > end_date:
end_date = timeline_date
return start_date, end_date
def load_osm_document(file_path, url_path):
"""Load an osm document, refreshing it if the cached copy is stale.
To save bandwidth the file is not downloaded if it is less than 1 hour old.
:type file_path: basestring
:param file_path: The path on the filesystem to which the file should
be saved.
:param url_path: Path (relative to the ftp root) from which the file
should be retrieved.
:type url_path: str
:returns: A file object for the the downloaded file.
:rtype: file
Raises:
None
"""
elapsed_seconds = 0
if os.path.exists(file_path):
current_time = time.time() # in unix epoch
file_time = os.path.getmtime(file_path) # in unix epoch
elapsed_seconds = current_time - file_time
if elapsed_seconds > 3600:
os.remove(file_path)
if elapsed_seconds > 3600 or not os.path.exists(file_path):
fetch_osm(file_path, url_path)
message = ('fetched %s' % file_path)
LOGGER.info(message)
file_handle = open(file_path, 'rb')
return file_handle
def load_osm_document(file_path, url_path):
"""Load an osm document, refreshing it if the cached copy is stale.
To save bandwidth the file is not downloaded if it is less than 1 hour old.
:type file_path: basestring
:param file_path: The path on the filesystem to which the file should
be saved.
:param url_path: Path (relative to the ftp root) from which the file
should be retrieved.
:type url_path: str
:returns: A file object for the the downloaded file.
:rtype: file
Raises:
None
"""
elapsed_seconds = 0
if os.path.exists(file_path):
current_time = time.time() # in unix epoch
file_time = os.path.getmtime(file_path) # in unix epoch
elapsed_seconds = current_time - file_time
if elapsed_seconds > 3600:
os.remove(file_path)
if elapsed_seconds > 3600 or not os.path.exists(file_path):
fetch_osm(file_path, url_path)
message = ('fetched %s' % file_path)
LOGGER.info(message)
file_handle = open(file_path, 'rb')
return file_handle
def date_range(timeline):
"""Given a timeline, determine the start and end dates.
The timeline may be sparse (containing fewer entries than all the dates
between the min and max dates) and since it is a dict,
the dates may be in any order.
:param timeline: A dictionary of non-sequential dates (in YYYY-MM-DD) as
keys and values (representing ways collected on that day).
:type timeline: dict
:returns: A tuple containing two dates:
* start_date - a date object representing the earliest date in the
time line.
* end_date - a date object representing the newest date in the time
line.
:rtype: (date, date)
"""
start_date = None
end_date = None
for next_date in timeline.keys():
year, month, day = next_date.split('-')
message = 'Date: %s' % next_date
LOGGER.info(message)
timeline_date = date(int(year), int(month), int(day))
if start_date is None:
start_date = timeline_date
if end_date is None:
end_date = timeline_date
if timeline_date < start_date:
start_date = timeline_date
if timeline_date > end_date:
end_date = timeline_date
return start_date, end_date
def drop_database(db_name):
"""Remove a database.
:param db_name: The database
:type db_name: str
"""
dropdb_executable = which('dropdb')[0]
dropdb_command = '%s %s' % (dropdb_executable, db_name)
LOGGER.info(dropdb_command)
call(dropdb_command, shell=True)
def drop_database(db_name):
"""Remove a database.
:param db_name: The database
:type db_name: str
"""
dropdb_executable = which('dropdb')[0]
dropdb_command = '%s %s' % (dropdb_executable, db_name)
LOGGER.info(dropdb_command)
call(dropdb_command, shell=True)
def import_osm_file(db_name, feature_type, file_path):
"""Import the OSM xml file into a postgis database.
:param db_name: The database to use.
:type db_name: str
:param feature_type: The feature to import.
:type feature_type: str
:param file_path: Path to the OSM file.
:type file_path: str
"""
overpass_resource_path = overpass_resource_base_path(feature_type)
style_file = '%s.style' % overpass_resource_path
# Used to standarise types while data is in pg still
transform_path = '%s.sql' % overpass_resource_path
createdb_executable = which('createdb')[0]
createdb_command = '%s -T template_postgis %s' % (createdb_executable,
db_name)
osm2pgsql_executable = which('osm2pgsql')[0]
osm2pgsql_options = config.OSM2PGSQL_OPTIONS
osm2pgsql_command = '%s -S %s -d %s %s %s' % (osm2pgsql_executable,
style_file, db_name,
osm2pgsql_options, file_path)
psql_executable = which('psql')[0]
transform_command = '%s %s -f %s' % (psql_executable, db_name,
transform_path)
LOGGER.info(createdb_command)
call(createdb_command, shell=True)
LOGGER.info(osm2pgsql_command)
call(osm2pgsql_command, shell=True)
LOGGER.info(transform_command)
call(transform_command, shell=True)
def setUp(self):
"""Setup the selenium driver."""
self.app = self.create_app()
# First just do a basic test to see that the test server works
# independently of selenium tests.
# result = self.app.get(
# '/', data=dict(), follow_redirects=True)
# code = result.status_code
# self.assertEquals(code, 200)
# now setup selenium driver
# TODO add platform check and look for common browsers
# We can check firefox and chrome on all platforms...
try:
self.driver = webdriver.Chrome()
except Exception as e:
self.fail(
'Error setting up selenium driver for Chrome\n%s' % e.message)
try:
self.driver.get(self.get_server_url())
except Exception as e:
self.fail(
'Error getting server url for selenium tests\n%s' % e.message)
LOGGER.info('Preparing to run front end tests')
def setUp(self):
"""Setup the selenium driver."""
self.app = self.create_app()
# First just do a basic test to see that the test server works
# independently of selenium tests.
# result = self.app.get(
# '/', data=dict(), follow_redirects=True)
# code = result.status_code
# self.assertEquals(code, 200)
# now setup selenium driver
# TODO add platform check and look for common browsers
# We can check firefox and chrome on all platforms...
try:
self.driver = webdriver.Chrome()
except Exception as e:
self.fail('Error setting up selenium driver for Chrome\n%s' %
e.message)
try:
self.driver.get(self.get_server_url())
except Exception as e:
self.fail('Error getting server url for selenium tests\n%s' %
e.message)
LOGGER.info('Preparing to run front end tests')
def import_osm_file(db_name, feature_type, file_path):
"""Import the OSM xml file into a postgis database.
:param db_name: The database to use.
:type db_name: str
:param feature_type: The feature to import.
:type feature_type: str
:param file_path: Path to the OSM file.
:type file_path: str
"""
overpass_resource_path = overpass_resource_base_path(feature_type)
style_file = '%s.style' % overpass_resource_path
# Used to standarise types while data is in pg still
transform_path = '%s.sql' % overpass_resource_path
createdb_executable = which('createdb')[0]
createdb_command = '%s -T template_postgis %s' % (
createdb_executable, db_name)
osm2pgsql_executable = which('osm2pgsql')[0]
osm2pgsql_options = config.OSM2PGSQL_OPTIONS
osm2pgsql_command = '%s -S %s -d %s %s %s' % (
osm2pgsql_executable,
style_file,
db_name,
osm2pgsql_options,
file_path)
psql_executable = which('psql')[0]
transform_command = '%s %s -f %s' % (
psql_executable, db_name, transform_path)
LOGGER.info(createdb_command)
call(createdb_command, shell=True)
LOGGER.info(osm2pgsql_command)
call(osm2pgsql_command, shell=True)
LOGGER.info(transform_command)
call(transform_command, shell=True)
def extract_shapefile(
feature_type,
db_name,
directory_name,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Extract a database to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param db_name: The database to extract.
:type db_name: str
:param directory_name: The directory to use for the extract.
:type directory_name: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
# Extract
os.makedirs(directory_name)
shapefile_resource_path = shapefile_resource_base_path(feature_type)
shape_path = os.path.join(directory_name, '%s.shp' % output_prefix)
if qgis_version > 1:
qml_source_path = '%s-%s.qml' % (shapefile_resource_path, lang)
if not os.path.isfile(qml_source_path):
qml_source_path = '%s-en.qml' % shapefile_resource_path
else:
qml_source_path = '%s-qgis1.qml' % shapefile_resource_path
qml_dest_path = os.path.join(directory_name, '%s.qml' % output_prefix)
license_source_path = '%s.license' % generic_shapefile_base_path()
license_dest_path = os.path.join(
directory_name, '%s.license' % output_prefix)
prj_source_path = '%s.prj' % generic_shapefile_base_path()
prj_dest_path = os.path.join(
directory_name, '%s.prj' % output_prefix)
pgsql2shp_executable = which('pgsql2shp')[0]
pgsql2shp_command = '%s -f %s %s %s' % (
pgsql2shp_executable, shape_path, db_name, SQL_QUERY_MAP[feature_type])
# Now run the commands in sequence:
LOGGER.info(pgsql2shp_command)
call(pgsql2shp_command, shell=True)
copyfile(qml_source_path, qml_dest_path)
metadata = metadata_files(
inasafe_version, lang, feature_type, output_prefix)
for destination, source in metadata.items():
source_path = '%s%s' % (shapefile_resource_path, source)
destination_path = os.path.join(directory_name, destination)
copyfile(source_path, destination_path)
add_metadata_timestamp(destination_path)
# Generic files
copyfile(prj_source_path, prj_dest_path)
copyfile(license_source_path, license_dest_path)
# Now zip it up and return the path to the zip, removing the original shp
zipfile = zip_shp(
shape_path,
extra_ext=['.qml', '.keywords', '.license', '.xml'],
remove_file=True)
LOGGER.info('Shape written to {path}'.format(path=shape_path))
return zipfile
context['campaigns'] = Campaign.all()
context['categories'] = AbstractInsightsFunction.CATEGORIES
context['functions'] = get_selected_functions()
context['title'] = 'Edit Campaign'
return render_template(
'create_campaign.html', form=form, **context)
@campaign_manager.route('/land')
def landing_auth():
"""OSM auth landing page.
"""
return render_template('land.html')
@campaign_manager.route('/not-logged-in.html')
def not_logged_in():
"""Not logged in page.
"""
return render_template('not_authenticated.html')
if __name__ == '__main__':
if Config.DEBUG:
campaign_manager.debug = True
# set up flask to serve static content
campaign_manager.add_url_rule('/<path:path>', 'static_file', static_file)
else:
LOGGER.info('Running in production mode')
campaign_manager.run()
def extract_shapefile(feature_type,
db_name,
directory_name,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Extract a database to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param db_name: The database to extract.
:type db_name: str
:param directory_name: The directory to use for the extract.
:type directory_name: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
# Extract
os.makedirs(directory_name)
shapefile_resource_path = shapefile_resource_base_path(feature_type)
shape_path = os.path.join(directory_name, '%s.shp' % output_prefix)
if qgis_version > 1:
qml_source_path = '%s-%s.qml' % (shapefile_resource_path, lang)
if not os.path.isfile(qml_source_path):
qml_source_path = '%s-en.qml' % shapefile_resource_path
else:
qml_source_path = '%s-qgis1.qml' % shapefile_resource_path
qml_dest_path = os.path.join(directory_name, '%s.qml' % output_prefix)
license_source_path = '%s.license' % generic_shapefile_base_path()
license_dest_path = os.path.join(directory_name,
'%s.license' % output_prefix)
prj_source_path = '%s.prj' % generic_shapefile_base_path()
prj_dest_path = os.path.join(directory_name, '%s.prj' % output_prefix)
pgsql2shp_executable = which('pgsql2shp')[0]
pgsql2shp_command = '%s -f %s %s %s' % (
pgsql2shp_executable, shape_path, db_name, SQL_QUERY_MAP[feature_type])
# Now run the commands in sequence:
LOGGER.info(pgsql2shp_command)
call(pgsql2shp_command, shell=True)
copyfile(qml_source_path, qml_dest_path)
metadata = metadata_files(inasafe_version, lang, feature_type,
output_prefix)
for destination, source in metadata.items():
source_path = '%s%s' % (shapefile_resource_path, source)
destination_path = os.path.join(directory_name, destination)
copyfile(source_path, destination_path)
add_metadata_timestamp(destination_path)
# Generic files
copyfile(prj_source_path, prj_dest_path)
copyfile(license_source_path, license_dest_path)
# Now zip it up and return the path to the zip, removing the original shp
zipfile = zip_shp(shape_path,
extra_ext=['.qml', '.keywords', '.license', '.xml'],
remove_file=True)
LOGGER.info('Shape written to {path}'.format(path=shape_path))
return zipfile
def download_feature(feature_type):
"""Generic request to download OSM data.
:param feature_type The feature to extract.
:type feature_type str
:return A zip file
"""
if feature_type not in FEATURES:
abort(404)
bbox = request.args.get('bbox', config.BBOX)
# Get the QGIS version
# Currently 1, 2 are accepted, default to 2
# A different qml style file will be returned depending on the version
qgis_version = int(request.args.get('qgis_version', '2'))
# Optional parameter that allows the user to specify the filename.
output_prefix = request.args.get('output_prefix', feature_type)
# A different keywords file will be returned depending on the version.
inasafe_version = request.args.get('inasafe_version', None)
# Optional parameter that allows the user to specify the language for
# the legend in QGIS.
lang = request.args.get('lang', 'en')
# error = None
try:
coordinates = split_bbox(bbox)
except ValueError:
# error = "Invalid bbox"
# coordinates = split_bbox(config.BBOX)
abort(500)
else:
local_osm_file = abspath(
join(dirname(__file__), 'resources', 'pbf', 'data.pbf'))
if not exists(local_osm_file):
LOGGER.info('Going to download data from overpass.')
try:
file_handle = get_osm_file(coordinates, feature_type, 'body')
except OverpassTimeoutException:
abort(408)
except OverpassBadRequestException:
abort(500)
except OverpassConcurrentRequestException:
abort(509)
except URLError:
abort(500)
else:
LOGGER.info(
'Local PBF file detected. We will not use the Overpass API.')
file_handle = open(local_osm_file, 'rb')
# This is for logging requests so we can see what queries we received
date_time = datetime.datetime.now()
log_data = {
'feature_type': feature_type,
'qgis_version': qgis_version,
'inasafe_version': inasafe_version,
'year': date_time.year,
'month': date_time.month,
'day': date_time.day,
'hour': date_time.hour,
'minute': date_time.minute,
'second': date_time.second
}
# add keys for SW_lng, SW_lat, NE_lng, etc.
# to our log and write the log file out...
log_data.update(coordinates)
log_file_name = (
'{year}{month}{day}-{hour}{minute}{second}.geojson').format(**log_data)
log_path = os.path.join(config.LOG_DIR, log_file_name)
# Note that all the double {{ will be rendered as single below
# They need to be double so that python does not confuse them as
# string interpolators
log_message = """
{{
"type": "FeatureCollection",
"features": [
{{
"type": "Feature",
"properties": {{
"feature_type": "{feature_type}",
"qgis_version": "{qgis_version}",
"inasafe_version": "{inasafe_version}",
"year": {year},
"month": {month},
"day": {day},
"hour": {hour}
}},
"geometry": {{
"type": "Polygon",
"coordinates": [
[
[
{SW_lng},
{NE_lat}
],
[
{NE_lng},
{NE_lat}
],
[
{NE_lng},
{SW_lat}
],
[
{SW_lng},
{SW_lat}
],
[
{SW_lng},
{NE_lat}
]
]
]
}}
}}
]
}}""".format(**log_data)
log_file = open(log_path, "w")
log_file.write(log_message)
log_file.close()
try:
# noinspection PyUnboundLocalVariable
zip_file = import_and_extract_shapefile(
feature_type,
file_handle.name,
qgis_version,
output_prefix,
inasafe_version,
lang)
f = open(zip_file, 'rb')
except IOError:
abort(404)
return
return Response(f.read(), mimetype='application/zip')