def load_osm_document(file_path, url_path):
"""Load an osm document, refreshing it if the cached copy is stale.
To save bandwidth the file is not downloaded if it is less than 1 hour old.
:type file_path: basestring
:param file_path: The path on the filesystem to which the file should
be saved.
:param url_path: Path (relative to the ftp root) from which the file
should be retrieved.
:type url_path: str
:returns: A file object for the the downloaded file.
:rtype: file
Raises:
None
"""
elapsed_seconds = 0
if os.path.exists(file_path):
current_time = time.time() # in unix epoch
file_time = os.path.getmtime(file_path) # in unix epoch
elapsed_seconds = current_time - file_time
if elapsed_seconds > 3600:
os.remove(file_path)
if elapsed_seconds > 3600 or not os.path.exists(file_path):
fetch_osm(file_path, url_path)
message = ('fetched %s' % file_path)
LOGGER.info(message)
file_handle = open(file_path, 'rb')
return file_handle
def test_home(self):
"""Test the home page works."""
try:
return self.app.post('/', data=dict(), follow_redirects=True)
except Exception, e:
LOGGER.exception('Basic front page load failed.')
raise e
def date_range(timeline):
"""Given a timeline, determine the start and end dates.
The timeline may be sparse (containing fewer entries than all the dates
between the min and max dates) and since it is a dict,
the dates may be in any order.
:param timeline: A dictionary of non-sequential dates (in YYYY-MM-DD) as
keys and values (representing ways collected on that day).
:type timeline: dict
:returns: A tuple containing two dates:
* start_date - a date object representing the earliest date in the
time line.
* end_date - a date object representing the newest date in the time
line.
:rtype: (date, date)
"""
start_date = None
end_date = None
for next_date in timeline.keys():
year, month, day = next_date.split('-')
message = 'Date: %s' % next_date
LOGGER.info(message)
timeline_date = date(int(year), int(month), int(day))
if start_date is None:
start_date = timeline_date
if end_date is None:
end_date = timeline_date
if timeline_date < start_date:
start_date = timeline_date
if timeline_date > end_date:
end_date = timeline_date
return start_date, end_date
def load_osm_document(file_path, url_path):
"""Load an osm document, refreshing it if the cached copy is stale.
To save bandwidth the file is not downloaded if it is less than 1 hour old.
:type file_path: basestring
:param file_path: The path on the filesystem to which the file should
be saved.
:param url_path: Path (relative to the ftp root) from which the file
should be retrieved.
:type url_path: str
:returns: A file object for the the downloaded file.
:rtype: file
Raises:
None
"""
elapsed_seconds = 0
if os.path.exists(file_path):
current_time = time.time() # in unix epoch
file_time = os.path.getmtime(file_path) # in unix epoch
elapsed_seconds = current_time - file_time
if elapsed_seconds > 3600:
os.remove(file_path)
if elapsed_seconds > 3600 or not os.path.exists(file_path):
fetch_osm(file_path, url_path)
message = ('fetched %s' % file_path)
LOGGER.info(message)
file_handle = open(file_path, 'rb')
return file_handle
def which(name, flags=os.X_OK):
"""Search PATH for executable files with the given name.
..note:: This function was taken verbatim from the twisted framework,
licence available here:
http://twistedmatrix.com/trac/browser/tags/releases/twisted-8.2.0/LICENSE
On newer versions of MS-Windows, the PATHEXT environment variable will be
set to the list of file extensions for files considered executable. This
will normally include things like ".EXE". This fuction will also find files
with the given name ending with any of these extensions.
On MS-Windows the only flag that has any meaning is os.F_OK. Any other
flags will be ignored.
:type name: C{str}
:param name: The name for which to search.
:type flags: C{int}
:param flags: Arguments to L{os.access}.
:rtype: C{list}
:param: A list of the full paths to files found, in the
order in which they were found.
"""
if os.path.exists('/usr/bin/%s' % name):
return ['/usr/bin/%s' % name]
if os.path.exists('/usr/local/bin/%s' % name):
return ['/usr/local/bin/%s' % name]
result = []
# pylint: disable=W0141
extensions = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
# pylint: enable=W0141
path = os.environ.get('PATH', None)
# In c6c9b26 we removed this hard coding for issue #529 but I am
# adding it back here in case the user's path does not include the
# gdal binary dir on OSX but it is actually there. (TS)
if sys.platform == 'darwin': # Mac OS X
gdal_prefix = ('/Library/Frameworks/GDAL.framework/'
'Versions/1.10/Programs/')
path = '%s:%s' % (path, gdal_prefix)
message = 'Search path: %s' % path
LOGGER.debug(message)
if path is None:
return []
for p in path.split(os.pathsep):
p = os.path.join(p, name)
if os.access(p, flags):
result.append(p)
for e in extensions:
pext = p + e
if os.access(pext, flags):
result.append(pext)
return result
def date_range(timeline):
"""Given a timeline, determine the start and end dates.
The timeline may be sparse (containing fewer entries than all the dates
between the min and max dates) and since it is a dict,
the dates may be in any order.
:param timeline: A dictionary of non-sequential dates (in YYYY-MM-DD) as
keys and values (representing ways collected on that day).
:type timeline: dict
:returns: A tuple containing two dates:
* start_date - a date object representing the earliest date in the
time line.
* end_date - a date object representing the newest date in the time
line.
:rtype: (date, date)
"""
start_date = None
end_date = None
for next_date in timeline.keys():
year, month, day = next_date.split('-')
message = 'Date: %s' % next_date
LOGGER.info(message)
timeline_date = date(int(year), int(month), int(day))
if start_date is None:
start_date = timeline_date
if end_date is None:
end_date = timeline_date
if timeline_date < start_date:
start_date = timeline_date
if timeline_date > end_date:
end_date = timeline_date
return start_date, end_date
def import_and_extract_shapefile(feature_type,
file_path,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Convert the OSM xml file to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param file_path: Path to the OSM file name.
:type file_path: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
if not check_string(output_prefix):
error = 'Invalid output prefix: %s' % output_prefix
LOGGER.exception(error)
raise Exception(error)
output_prefix += feature_type
work_dir = temp_dir(sub_dir=feature_type)
directory_name = unique_filename(dir=work_dir)
db_name = os.path.basename(directory_name)
import_osm_file(db_name, feature_type, file_path)
zip_file = extract_shapefile(feature_type, db_name, directory_name,
qgis_version, output_prefix, inasafe_version,
lang)
drop_database(db_name)
return zip_file
def request_data(self, url_request):
web_request = Request(url_request, None, self.headers)
try:
url_handle = urlopen(web_request, timeout=60)
data = url_handle.read().decode('utf-8')
return data
except HTTPError as e:
LOGGER.exception('Error with request')
return e.msg
def test_home(self):
"""Test the home page works."""
try:
result = self.app.get('/', data=dict(), follow_redirects=True)
code = result.status_code
self.assertEquals(code, 200)
except Exception as e:
LOGGER.exception('Basic front page load failed.')
raise e
def drop_database(db_name):
"""Remove a database.
:param db_name: The database
:type db_name: str
"""
dropdb_executable = which('dropdb')[0]
dropdb_command = '%s %s' % (dropdb_executable, db_name)
LOGGER.info(dropdb_command)
call(dropdb_command, shell=True)
def drop_database(db_name):
"""Remove a database.
:param db_name: The database
:type db_name: str
"""
dropdb_executable = which('dropdb')[0]
dropdb_command = '%s %s' % (dropdb_executable, db_name)
LOGGER.info(dropdb_command)
call(dropdb_command, shell=True)
def test_home(self):
"""Test the home page works."""
try:
result = self.app.get(
'/', data=dict(), follow_redirects=True)
code = result.status_code
self.assertEquals(code, 200)
except Exception as e:
LOGGER.exception('Basic front page load failed.')
raise e
def fetch_osm_with_post(file_path, url_path, post_data, returns_format='json'):
"""Fetch an osm map and store locally.
:param url_path: The path (relative to the ftp root) from which the
file should be retrieved.
:type url_path: str
:param file_path: The path on the filesystem to which the file should
be saved.
:type file_path: str
:param post_data: Overpass data
:type post_data: str
:param returns_format: Format of the response, could be json or xml
:type returns_format: str
:returns: The path to the downloaded file.
"""
headers = {'User-Agent': 'HotOSM'}
try:
data = requests.post(url=url_path,
data={'data': post_data},
headers=headers)
if returns_format != 'xml':
regex = '<remark> runtime error:'
if re.search(regex, data.text):
raise OverpassTimeoutException
regex = '(elements|meta)'
if not re.search(regex, data.text):
raise OverpassDoesNotReturnData
if os.path.exists(file_path):
os.remove(file_path)
file_handle = open(file_path, 'wb')
file_handle.write(data.text.encode('utf-8'))
file_handle.close()
except HTTPError as e:
if e.code == 400:
LOGGER.exception('Bad request to Overpass')
raise OverpassBadRequestException
elif e.code == 419:
raise OverpassConcurrentRequestException
LOGGER.exception('Error with Overpass')
raise e
def import_osm_file(db_name, feature_type, file_path):
"""Import the OSM xml file into a postgis database.
:param db_name: The database to use.
:type db_name: str
:param feature_type: The feature to import.
:type feature_type: str
:param file_path: Path to the OSM file.
:type file_path: str
"""
overpass_resource_path = overpass_resource_base_path(feature_type)
style_file = '%s.style' % overpass_resource_path
# Used to standarise types while data is in pg still
transform_path = '%s.sql' % overpass_resource_path
createdb_executable = which('createdb')[0]
createdb_command = '%s -T template_postgis %s' % (createdb_executable,
db_name)
osm2pgsql_executable = which('osm2pgsql')[0]
osm2pgsql_options = config.OSM2PGSQL_OPTIONS
osm2pgsql_command = '%s -S %s -d %s %s %s' % (osm2pgsql_executable,
style_file, db_name,
osm2pgsql_options, file_path)
psql_executable = which('psql')[0]
transform_command = '%s %s -f %s' % (psql_executable, db_name,
transform_path)
LOGGER.info(createdb_command)
call(createdb_command, shell=True)
LOGGER.info(osm2pgsql_command)
call(osm2pgsql_command, shell=True)
LOGGER.info(transform_command)
call(transform_command, shell=True)
def user_status():
"""Get nodes for user as a json doc.
.. note:: User from reporter.js
To use e.g.:
http://localhost:5000/user?bbox=20.431909561157227,
-34.02849543118406,20.45207977294922,-34.02227106658948&
obj=building&username=timlinux
"""
username = request.args.get('username')
bbox = request.args.get('bbox')
try:
coordinates = split_bbox(bbox)
except ValueError:
error = "Invalid bbox"
coordinates = split_bbox(config.BBOX)
LOGGER.exception(error + str(coordinates))
else:
try:
file_handle = get_osm_file(coordinates)
except OverpassTimeoutException:
error = "Bad request. Maybe the bbox is too big!"
LOGGER.exception(error + str(coordinates))
except OverpassConcurrentRequestException:
error = 'Please try again later, another query is running.'
LOGGER.exception(error + str(coordinates))
except OverpassBadRequestException:
error = "Bad request."
LOGGER.exception(error + str(coordinates))
except URLError:
error = "Bad request."
LOGGER.exception(error + str(coordinates))
else:
node_data = osm_nodes_by_user(file_handle, username)
return jsonify(d=node_data)
def user_status():
"""Get nodes for user as a json doc.
.. note:: User from reporter.js
To use e.g.:
http://localhost:5000/user?bbox=20.431909561157227,
-34.02849543118406,20.45207977294922,-34.02227106658948&
obj=building&username=timlinux
"""
username = request.args.get('username')
bbox = request.args.get('bbox')
try:
coordinates = split_bbox(bbox)
except ValueError:
error = "Invalid bbox"
coordinates = split_bbox(config.BBOX)
LOGGER.exception(error + str(coordinates))
else:
try:
file_handle = get_osm_file(coordinates)
except OverpassTimeoutException:
error = "Bad request. Maybe the bbox is too big!"
LOGGER.exception(error + str(coordinates))
except OverpassConcurrentRequestException:
error = 'Please try again later, another query is running.'
LOGGER.exception(error + str(coordinates))
except OverpassBadRequestException:
error = "Bad request."
LOGGER.exception(error + str(coordinates))
except URLError:
error = "Bad request."
LOGGER.exception(error + str(coordinates))
else:
node_data = osm_nodes_by_user(file_handle, username)
return jsonify(d=node_data)
def setUp(self):
"""Setup the selenium driver."""
self.app = self.create_app()
# First just do a basic test to see that the test server works
# independently of selenium tests.
# result = self.app.get(
# '/', data=dict(), follow_redirects=True)
# code = result.status_code
# self.assertEquals(code, 200)
# now setup selenium driver
# TODO add platform check and look for common browsers
# We can check firefox and chrome on all platforms...
try:
self.driver = webdriver.Chrome()
except Exception as e:
self.fail('Error setting up selenium driver for Chrome\n%s' %
e.message)
try:
self.driver.get(self.get_server_url())
except Exception as e:
self.fail('Error getting server url for selenium tests\n%s' %
e.message)
LOGGER.info('Preparing to run front end tests')
def setUp(self):
"""Setup the selenium driver."""
self.app = self.create_app()
# First just do a basic test to see that the test server works
# independently of selenium tests.
# result = self.app.get(
# '/', data=dict(), follow_redirects=True)
# code = result.status_code
# self.assertEquals(code, 200)
# now setup selenium driver
# TODO add platform check and look for common browsers
# We can check firefox and chrome on all platforms...
try:
self.driver = webdriver.Chrome()
except Exception as e:
self.fail(
'Error setting up selenium driver for Chrome\n%s' % e.message)
try:
self.driver.get(self.get_server_url())
except Exception as e:
self.fail(
'Error getting server url for selenium tests\n%s' % e.message)
LOGGER.info('Preparing to run front end tests')
def fetch_osm(file_path, url_path):
"""Fetch an osm map and store locally.
:param url_path: The path (relative to the ftp root) from which the
file should be retrieved.
:type url_path: str
:param file_path: The path on the filesystem to which the file should
be saved.
:type file_path: str
:returns: The path to the downloaded file.
"""
LOGGER.debug('Getting URL: %s', url_path)
headers = {'User-Agent': 'InaSAFE'}
web_request = Request(url_path, None, headers)
try:
url_handle = urlopen(web_request, timeout=60)
data = url_handle.read().decode('utf-8')
regex = '<remark> runtime error:'
if re.search(regex, data):
raise OverpassTimeoutException
regex = '(elements|meta)'
if not re.search(regex, data):
raise OverpassDoesNotReturnData
if os.path.exists(file_path):
os.remove(file_path)
file_handle = open(file_path, 'wb')
file_handle.write(data.encode('utf-8'))
file_handle.close()
except HTTPError as e:
if e.code == 400:
LOGGER.exception('Bad request to Overpass')
raise OverpassBadRequestException
elif e.code == 419:
raise OverpassConcurrentRequestException
LOGGER.exception('Error with Overpass')
raise e
def import_osm_file(db_name, feature_type, file_path):
"""Import the OSM xml file into a postgis database.
:param db_name: The database to use.
:type db_name: str
:param feature_type: The feature to import.
:type feature_type: str
:param file_path: Path to the OSM file.
:type file_path: str
"""
overpass_resource_path = overpass_resource_base_path(feature_type)
style_file = '%s.style' % overpass_resource_path
# Used to standarise types while data is in pg still
transform_path = '%s.sql' % overpass_resource_path
createdb_executable = which('createdb')[0]
createdb_command = '%s -T template_postgis %s' % (
createdb_executable, db_name)
osm2pgsql_executable = which('osm2pgsql')[0]
osm2pgsql_options = config.OSM2PGSQL_OPTIONS
osm2pgsql_command = '%s -S %s -d %s %s %s' % (
osm2pgsql_executable,
style_file,
db_name,
osm2pgsql_options,
file_path)
psql_executable = which('psql')[0]
transform_command = '%s %s -f %s' % (
psql_executable, db_name, transform_path)
LOGGER.info(createdb_command)
call(createdb_command, shell=True)
LOGGER.info(osm2pgsql_command)
call(osm2pgsql_command, shell=True)
LOGGER.info(transform_command)
call(transform_command, shell=True)
def fetch_osm(file_path, url_path):
"""Fetch an osm map and store locally.
:param url_path: The path (relative to the ftp root) from which the
file should be retrieved.
:type url_path: str
:param file_path: The path on the filesystem to which the file should
be saved.
:type file_path: str
:returns: The path to the downloaded file.
"""
LOGGER.debug('Getting URL: %s', url_path)
headers = {'User-Agent': 'InaSAFE'}
web_request = Request(url_path, None, headers)
try:
url_handle = urlopen(web_request, timeout=60)
data = url_handle.read().decode('utf-8')
regex = '<remark> runtime error:'
if re.search(regex, data):
raise OverpassTimeoutException
file_handle = open(file_path, 'wb')
file_handle.write(data.encode('utf-8'))
file_handle.close()
except HTTPError as e:
if e.code == 400:
LOGGER.exception('Bad request to Overpass')
raise OverpassBadRequestException
elif e.code == 419:
raise OverpassConcurrentRequestException
LOGGER.exception('Error with Overpass')
raise e
def osm_object_contributions(osm_file, tag_name):
"""Compile a summary of user contributions for the selected osm data type.
:param osm_file: A file object reading from a .osm file.
:type osm_file: file, FileIO
:param tag_name: The tag name we want to filter on.
:type tag_name: str
:returns: A list of dicts where items in the list are sorted from highest
contributor (based on number of ways) down to lowest. Each element
in the list is a dict in the form: {
'user': <user>,
'ways': <way count>,
'nodes': <node count>,
'timeline': <timelinedict>,
'best': <most ways in a single day>,
'worst': <least ways in single day>,
'average': <average ways across active days>,
'crew': <bool> }
where crew is used to designate users who are part of an active
data gathering campaign.
The timeline dict will contain a collection of dates and
the total number of ways created on that date e.g.
{
u'2010-12-09': 10,
u'2012-07-10': 14
}
:rtype: list
"""
parser = OsmParser(tag_name=tag_name)
try:
xml.sax.parse(osm_file, parser)
except xml.sax.SAXParseException:
LOGGER.exception('Failed to parse OSM xml.')
raise
way_count_dict = parser.wayCountDict
node_count_dict = parser.nodeCountDict
timelines = parser.userDayCountDict
# Convert to a list of dicts so we can sort it.
crew_list = config.CREW
user_list = []
for key, value in way_count_dict.items():
crew_flag = False
if key in crew_list:
crew_flag = True
start_date, end_date = date_range(timelines[key])
start_date = time.strftime('%d-%m-%Y', start_date.timetuple())
end_date = time.strftime('%d-%m-%Y', end_date.timetuple())
user_timeline = timelines[key]
record = {
'name': key,
'ways': value,
'nodes': node_count_dict[key],
'timeline': interpolated_timeline(user_timeline),
'start': start_date,
'end': end_date,
'activeDays': len(user_timeline),
'best': best_active_day(user_timeline),
'worst': worst_active_day(user_timeline),
'average': average_for_active_days(user_timeline),
'crew': crew_flag
}
user_list.append(record)
# Sort it
sorted_user_list = sorted(
user_list, key=lambda d: (
-d['ways'],
d['nodes'],
d['name'],
d['timeline'],
d['start'],
d['end'],
d['activeDays'],
d['best'],
d['worst'],
d['average'],
d['crew']))
return sorted_user_list
def get_osm_file(coordinates,
feature='all',
overpass_verbosity='body',
date_from=None,
date_to=None):
"""Fetch an osm file given a bounding box using the overpass API.
:param coordinates: Coordinates as a list in the form:
[min lat, min lon, max lat, max lon]
:param feature: The type of feature to extract:
buildings, building-points, roads, potential-idp, boundary-[1,11]
:type feature: str
:param overpass_verbosity: Output verbosity in Overpass.
It can be body, skeleton, ids_only or meta.
:type overpass_verbosity: str
:param date_from: First date for date range.
:type date_from: str
:param date_to: Second date for date range.
:type date_to: str
:returns: A file which has been opened on the retrieved OSM dataset.
:rtype: file
Coordinates look like this:
{'NE_lng': 20.444537401199337,
'SW_lat': -34.0460012312071,
'SW_lng': 20.439494848251343,
'NE_lat': -34.044441058971394}
Example overpass API query for buildings (testable at
http://overpass-turbo.eu/)::
(
node
["building"]
["building"!="no"]
({{bbox}});
way
["building"]
["building"!="no"]
({{bbox}});
rel
["building"]
["building"!="no"]
({{bbox}});
<;);out+meta;
Equivalent url (http encoded)::
"""
server_url = 'http://overpass-api.de/api/interpreter?data='
parameters = coordinates
parameters['print_mode'] = overpass_verbosity
query = OVERPASS_QUERY_MAP[feature].format(**parameters)
if date_from and date_to:
try:
datetime_from = datetime.datetime.utcfromtimestamp(
float(date_from) / 1000.)
datetime_to = datetime.datetime.utcfromtimestamp(
float(date_to) / 1000.)
date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
diff_query = '[diff:"{date_from}", "{date_to}"];'.format(
date_from=datetime_from.strftime(date_format),
date_to=datetime_to.strftime(date_format))
query = diff_query + query
except ValueError as e:
LOGGER.debug(e)
encoded_query = quote(query)
url_path = '%s%s' % (server_url, encoded_query)
safe_name = hashlib.md5(query.encode('utf-8')).hexdigest() + '.osm'
file_path = os.path.join(config.CACHE_DIR, safe_name)
return load_osm_document(file_path, url_path)
def extract_shapefile(feature_type,
db_name,
directory_name,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Extract a database to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param db_name: The database to extract.
:type db_name: str
:param directory_name: The directory to use for the extract.
:type directory_name: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
# Extract
os.makedirs(directory_name)
shapefile_resource_path = shapefile_resource_base_path(feature_type)
shape_path = os.path.join(directory_name, '%s.shp' % output_prefix)
if qgis_version > 1:
qml_source_path = '%s-%s.qml' % (shapefile_resource_path, lang)
if not os.path.isfile(qml_source_path):
qml_source_path = '%s-en.qml' % shapefile_resource_path
else:
qml_source_path = '%s-qgis1.qml' % shapefile_resource_path
qml_dest_path = os.path.join(directory_name, '%s.qml' % output_prefix)
license_source_path = '%s.license' % generic_shapefile_base_path()
license_dest_path = os.path.join(directory_name,
'%s.license' % output_prefix)
prj_source_path = '%s.prj' % generic_shapefile_base_path()
prj_dest_path = os.path.join(directory_name, '%s.prj' % output_prefix)
pgsql2shp_executable = which('pgsql2shp')[0]
pgsql2shp_command = '%s -f %s %s %s' % (
pgsql2shp_executable, shape_path, db_name, SQL_QUERY_MAP[feature_type])
# Now run the commands in sequence:
LOGGER.info(pgsql2shp_command)
call(pgsql2shp_command, shell=True)
copyfile(qml_source_path, qml_dest_path)
metadata = metadata_files(inasafe_version, lang, feature_type,
output_prefix)
for destination, source in metadata.items():
source_path = '%s%s' % (shapefile_resource_path, source)
destination_path = os.path.join(directory_name, destination)
copyfile(source_path, destination_path)
add_metadata_timestamp(destination_path)
# Generic files
copyfile(prj_source_path, prj_dest_path)
copyfile(license_source_path, license_dest_path)
# Now zip it up and return the path to the zip, removing the original shp
zipfile = zip_shp(shape_path,
extra_ext=['.qml', '.keywords', '.license', '.xml'],
remove_file=True)
LOGGER.info('Shape written to {path}'.format(path=shape_path))
return zipfile
def osm_object_contributions(osm_file,
tag_name,
date_start=None,
date_end=None):
"""Compile a summary of user contributions for the selected osm data type.
:param osm_file: A file object reading from a .osm file.
:type osm_file: file, FileIO
:param tag_name: The tag name we want to filter on.
:type tag_name: str
:param date_start: The start date we want to filter
:type date_start: float
:param date_end: The end date we want to filter
:type date_end: float
:returns: A list of dicts where items in the list are sorted from highest
contributor (based on number of ways) down to lowest. Each element
in the list is a dict in the form: {
'user': <user>,
'ways': <way count>,
'nodes': <node count>,
'timeline': <timelinedict>,
'best': <most ways in a single day>,
'worst': <least ways in single day>,
'average': <average ways across active days>,
'crew': <bool> }
where crew is used to designate users who are part of an active
data gathering campaign.
The timeline dict will contain a collection of dates and
the total number of ways created on that date e.g.
{
u'2010-12-09': 10,
u'2012-07-10': 14
}
:rtype: list
"""
parser = OsmParser(start_date=date_start, end_date=date_end)
try:
xml.sax.parse(osm_file, parser)
except xml.sax.SAXParseException:
LOGGER.exception('Failed to parse OSM xml.')
raise
way_count_dict = parser.wayCountDict
node_count_dict = parser.nodeCountDict
timelines = parser.userDayCountDict
# Convert to a list of dicts so we can sort it.
crew_list = config.CREW
user_list = []
for key, value in way_count_dict.items():
start_date, end_date = date_range(timelines[key])
start_date = time.strftime('%d-%m-%Y', start_date.timetuple())
end_date = time.strftime('%d-%m-%Y', end_date.timetuple())
user_timeline = timelines[key]
node_count = 0
if key in node_count_dict:
node_count = node_count_dict[key]
record = {
'name': key,
'ways': value,
'nodes': node_count,
'timeline': interpolated_timeline(user_timeline),
'start': start_date,
'end': end_date,
'activeDays': len(user_timeline),
'best': best_active_day(user_timeline),
'worst': worst_active_day(user_timeline),
'average': average_for_active_days(user_timeline)
}
user_list.append(record)
for key, value in node_count_dict.items():
start_date, end_date = date_range(timelines[key])
start_date = time.strftime('%d-%m-%Y', start_date.timetuple())
end_date = time.strftime('%d-%m-%Y', end_date.timetuple())
user_timeline = timelines[key]
record = {
'name': key,
'ways': 0,
'nodes': value,
'timeline': interpolated_timeline(user_timeline),
'start': start_date,
'end': end_date,
'activeDays': len(user_timeline),
'best': best_active_day(user_timeline),
'worst': worst_active_day(user_timeline),
'average': average_for_active_days(user_timeline)
}
user_list.append(record)
# Sort it
sorted_user_list = sorted(
user_list,
key=lambda d:
(-d['ways'], d['nodes'], d['name'], d['timeline'], d['start'], d[
'end'], d['activeDays'], d['best'], d['worst'], d['average']))
return sorted_user_list
def get_osm_file(
coordinates,
feature='all',
overpass_verbosity='body',
date_from=None,
date_to=None):
"""Fetch an osm file given a bounding box using the overpass API.
:param coordinates: Coordinates as a list in the form:
[min lat, min lon, max lat, max lon]
:param feature: The type of feature to extract:
buildings, building-points, roads, potential-idp, boundary-[1,11]
:type feature: str
:param overpass_verbosity: Output verbosity in Overpass.
It can be body, skeleton, ids_only or meta.
:type overpass_verbosity: str
:param date_from: First date for date range.
:type date_from: str
:param date_to: Second date for date range.
:type date_to: str
:returns: A file which has been opened on the retrieved OSM dataset.
:rtype: file
Coordinates look like this:
{'NE_lng': 20.444537401199337,
'SW_lat': -34.0460012312071,
'SW_lng': 20.439494848251343,
'NE_lat': -34.044441058971394}
Example overpass API query for buildings (testable at
http://overpass-turbo.eu/)::
(
node
["building"]
["building"!="no"]
({{bbox}});
way
["building"]
["building"!="no"]
({{bbox}});
rel
["building"]
["building"!="no"]
({{bbox}});
<;);out+meta;
Equivalent url (http encoded)::
"""
server_url = 'http://overpass-api.de/api/interpreter?data='
parameters = coordinates
parameters['print_mode'] = overpass_verbosity
query = OVERPASS_QUERY_MAP[feature].format(**parameters)
if date_from and date_to:
try:
datetime_from = datetime.datetime.utcfromtimestamp(
float(date_from) / 1000.)
datetime_to = datetime.datetime.utcfromtimestamp(
float(date_to) / 1000.)
date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
diff_query = '[diff:"{date_from}", "{date_to}"];'.format(
date_from=datetime_from.strftime(date_format),
date_to=datetime_to.strftime(date_format)
)
query = diff_query + query
except ValueError as e:
LOGGER.debug(e)
encoded_query = quote(query)
url_path = '%s%s' % (server_url, encoded_query)
safe_name = hashlib.md5(query.encode('utf-8')).hexdigest() + '.osm'
file_path = os.path.join(config.CACHE_DIR, safe_name)
return load_osm_document(file_path, url_path)
def extract_shapefile(
feature_type,
db_name,
directory_name,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Extract a database to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param db_name: The database to extract.
:type db_name: str
:param directory_name: The directory to use for the extract.
:type directory_name: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
# Extract
os.makedirs(directory_name)
shapefile_resource_path = shapefile_resource_base_path(feature_type)
shape_path = os.path.join(directory_name, '%s.shp' % output_prefix)
if qgis_version > 1:
qml_source_path = '%s-%s.qml' % (shapefile_resource_path, lang)
if not os.path.isfile(qml_source_path):
qml_source_path = '%s-en.qml' % shapefile_resource_path
else:
qml_source_path = '%s-qgis1.qml' % shapefile_resource_path
qml_dest_path = os.path.join(directory_name, '%s.qml' % output_prefix)
license_source_path = '%s.license' % generic_shapefile_base_path()
license_dest_path = os.path.join(
directory_name, '%s.license' % output_prefix)
prj_source_path = '%s.prj' % generic_shapefile_base_path()
prj_dest_path = os.path.join(
directory_name, '%s.prj' % output_prefix)
pgsql2shp_executable = which('pgsql2shp')[0]
pgsql2shp_command = '%s -f %s %s %s' % (
pgsql2shp_executable, shape_path, db_name, SQL_QUERY_MAP[feature_type])
# Now run the commands in sequence:
LOGGER.info(pgsql2shp_command)
call(pgsql2shp_command, shell=True)
copyfile(qml_source_path, qml_dest_path)
metadata = metadata_files(
inasafe_version, lang, feature_type, output_prefix)
for destination, source in metadata.items():
source_path = '%s%s' % (shapefile_resource_path, source)
destination_path = os.path.join(directory_name, destination)
copyfile(source_path, destination_path)
add_metadata_timestamp(destination_path)
# Generic files
copyfile(prj_source_path, prj_dest_path)
copyfile(license_source_path, license_dest_path)
# Now zip it up and return the path to the zip, removing the original shp
zipfile = zip_shp(
shape_path,
extra_ext=['.qml', '.keywords', '.license', '.xml'],
remove_file=True)
LOGGER.info('Shape written to {path}'.format(path=shape_path))
return zipfile
context['campaigns'] = Campaign.all()
context['categories'] = AbstractInsightsFunction.CATEGORIES
context['functions'] = get_selected_functions()
context['title'] = 'Edit Campaign'
return render_template(
'create_campaign.html', form=form, **context)
@campaign_manager.route('/land')
def landing_auth():
"""OSM auth landing page.
"""
return render_template('land.html')
@campaign_manager.route('/not-logged-in.html')
def not_logged_in():
"""Not logged in page.
"""
return render_template('not_authenticated.html')
if __name__ == '__main__':
if Config.DEBUG:
campaign_manager.debug = True
# set up flask to serve static content
campaign_manager.add_url_rule('/<path:path>', 'static_file', static_file)
else:
LOGGER.info('Running in production mode')
campaign_manager.run()
def import_and_extract_shapefile(
feature_type,
file_path,
qgis_version=2,
output_prefix='',
inasafe_version=None,
lang='en'):
"""Convert the OSM xml file to a shapefile.
This is a multi-step process:
* Create a temporary postgis database
* Load the osm dataset into POSTGIS with osm2pgsql and our custom
style file.
* Save the data out again to a shapefile
* Zip the shapefile ready for user to download
:param feature_type: The feature to extract.
:type feature_type: str
:param file_path: Path to the OSM file name.
:type file_path: str
:param qgis_version: Get the QGIS version. Currently 1,
2 are accepted, default to 2. A different qml style file will be
returned depending on the version
:type qgis_version: int
:param output_prefix: Base name for the shape file. Defaults to ''
which will result in an output file of feature_type + '.shp'. Adding a
prefix of e.g. 'test-' would result in a downloaded file name of
'test-buildings.shp'. Allowed characters are [a-zA-Z-_0-9].
:type output_prefix: str
:param inasafe_version: The InaSAFE version, to get correct metadata.
:type inasafe_version: str
:param lang: The language desired for the labels in the legend.
Example : 'en', 'fr', etc. Default is 'en'.
:type lang: str
:returns: Path to zipfile that was created.
:rtype: str
"""
if not check_string(output_prefix):
error = 'Invalid output prefix: %s' % output_prefix
LOGGER.exception(error)
raise Exception(error)
output_prefix += feature_type
work_dir = temp_dir(sub_dir=feature_type)
directory_name = unique_filename(dir=work_dir)
db_name = os.path.basename(directory_name)
import_osm_file(db_name, feature_type, file_path)
zip_file = extract_shapefile(
feature_type,
db_name,
directory_name,
qgis_version,
output_prefix,
inasafe_version,
lang)
drop_database(db_name)
return zip_file
def which(name, flags=os.X_OK):
"""Search PATH for executable files with the given name.
..note:: This function was taken verbatim from the twisted framework,
licence available here:
http://twistedmatrix.com/trac/browser/tags/releases/twisted-8.2.0/LICENSE
On newer versions of MS-Windows, the PATHEXT environment variable will be
set to the list of file extensions for files considered executable. This
will normally include things like ".EXE". This fuction will also find files
with the given name ending with any of these extensions.
On MS-Windows the only flag that has any meaning is os.F_OK. Any other
flags will be ignored.
:type name: C{str}
:param name: The name for which to search.
:type flags: C{int}
:param flags: Arguments to L{os.access}.
:rtype: C{list}
:param: A list of the full paths for files found, in the
order in which they were found.
"""
if os.path.exists('/usr/bin/%s' % name):
return ['/usr/bin/%s' % name]
if os.path.exists('/usr/local/bin/%s' % name):
return ['/usr/local/bin/%s' % name]
result = []
# pylint: disable=W0141
extensions = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
# pylint: enable=W0141
path = os.environ.get('PATH', None)
# In c6c9b26 we removed this hard coding for issue #529 but I am
# adding it back here in case the user's path does not include the
# gdal binary dir on OSX but it is actually there. (TS)
if sys.platform == 'darwin': # Mac OS X
postgis_prefix = (
'/Applications/Postgres.app/Contents/Versions/9.4/bin/')
path = '%s:%s' % (path, postgis_prefix)
gdal_prefix = (
'/Library/Frameworks/GDAL.framework/'
'Versions/1.10/Programs/')
path = '%s:%s' % (path, gdal_prefix)
message = 'Search path: %s' % path
LOGGER.debug(message)
if path is None:
return []
for p in path.split(os.pathsep):
p = os.path.join(p, name)
if os.access(p, flags):
result.append(p)
for e in extensions:
pext = p + e
if os.access(pext, flags):
result.append(pext)
return result
def download_feature(feature_type):
"""Generic request to download OSM data.
:param feature_type The feature to extract.
:type feature_type str
:return A zip file
"""
if feature_type not in FEATURES:
abort(404)
bbox = request.args.get('bbox', config.BBOX)
# Get the QGIS version
# Currently 1, 2 are accepted, default to 2
# A different qml style file will be returned depending on the version
qgis_version = int(request.args.get('qgis_version', '2'))
# Optional parameter that allows the user to specify the filename.
output_prefix = request.args.get('output_prefix', feature_type)
# A different keywords file will be returned depending on the version.
inasafe_version = request.args.get('inasafe_version', None)
# Optional parameter that allows the user to specify the language for
# the legend in QGIS.
lang = request.args.get('lang', 'en')
# error = None
try:
coordinates = split_bbox(bbox)
except ValueError:
# error = "Invalid bbox"
# coordinates = split_bbox(config.BBOX)
abort(500)
else:
local_osm_file = abspath(
join(dirname(__file__), 'resources', 'pbf', 'data.pbf'))
if not exists(local_osm_file):
LOGGER.info('Going to download data from overpass.')
try:
file_handle = get_osm_file(coordinates, feature_type, 'body')
except OverpassTimeoutException:
abort(408)
except OverpassBadRequestException:
abort(500)
except OverpassConcurrentRequestException:
abort(509)
except URLError:
abort(500)
else:
LOGGER.info(
'Local PBF file detected. We will not use the Overpass API.')
file_handle = open(local_osm_file, 'rb')
# This is for logging requests so we can see what queries we received
date_time = datetime.datetime.now()
log_data = {
'feature_type': feature_type,
'qgis_version': qgis_version,
'inasafe_version': inasafe_version,
'year': date_time.year,
'month': date_time.month,
'day': date_time.day,
'hour': date_time.hour,
'minute': date_time.minute,
'second': date_time.second
}
# add keys for SW_lng, SW_lat, NE_lng, etc.
# to our log and write the log file out...
log_data.update(coordinates)
log_file_name = (
'{year}{month}{day}-{hour}{minute}{second}.geojson').format(**log_data)
log_path = os.path.join(config.LOG_DIR, log_file_name)
# Note that all the double {{ will be rendered as single below
# They need to be double so that python does not confuse them as
# string interpolators
log_message = """
{{
"type": "FeatureCollection",
"features": [
{{
"type": "Feature",
"properties": {{
"feature_type": "{feature_type}",
"qgis_version": "{qgis_version}",
"inasafe_version": "{inasafe_version}",
"year": {year},
"month": {month},
"day": {day},
"hour": {hour}
}},
"geometry": {{
"type": "Polygon",
"coordinates": [
[
[
{SW_lng},
{NE_lat}
],
[
{NE_lng},
{NE_lat}
],
[
{NE_lng},
{SW_lat}
],
[
{SW_lng},
{SW_lat}
],
[
{SW_lng},
{NE_lat}
]
]
]
}}
}}
]
}}""".format(**log_data)
log_file = open(log_path, "w")
log_file.write(log_message)
log_file.close()
try:
# noinspection PyUnboundLocalVariable
zip_file = import_and_extract_shapefile(
feature_type,
file_handle.name,
qgis_version,
output_prefix,
inasafe_version,
lang)
f = open(zip_file, 'rb')
except IOError:
abort(404)
return
return Response(f.read(), mimetype='application/zip')
