def test_load_osm_document(self):
"""Check that we can fetch an osm doc and that it caches properly."""
#
# NOTE - INTERNET CONNECTION NEEDED FOR THIS TEST
#
url = (
'http://overpass-api.de/api/interpreter?data='
'(node(-34.03112731086964,20.44997155666351,'
'-34.029571310785315,20.45501410961151);<;);out+meta;')
file_path = '/tmp/test_load_osm_document.osm'
if os.path.exists(file_path):
os.remove(file_path)
# We test twice - once to ensure its fetched from the overpass api
# and once to ensure the cached file is used on second access
# Note: There is a small chance the second test could fail if it
# exactly straddles the cache expiry time.
try:
file_handle = load_osm_document(file_path, url)
except:
message = 'load_osm_document from overpass failed %s' % url
LOGGER.exception(message)
raise
string = file_handle.read()
message = 'load_osm_document from overpass content check failed.'
assert 'Jacoline' in string, message
#file_handle = load_osm_document(myFilePath, url)
file_time = os.path.getmtime(file_path)
#
# This one should be cached now....
#
load_osm_document(file_path, url)
file_time2 = os.path.getmtime(file_path)
message = 'load_osm_document cache test failed.'
self.assertEqual(file_time, file_time2, message)
def test_get_osm_file_with_date_range(self):
"""Check that we can get osm file with date range as query"""
#
# NOTE - INTERNET CONNECTION NEEDED FOR THIS TEST
#
url = ('http://overpass-api.de/api/interpreter?data='
'[diff:"2012-09-14T15:00:00Z","2015-09-21T15:00:00Z"];'
'(node(-34.03112731086964,20.44997155666351,'
'-34.029571310785315,20.45501410961151);<;);out+meta;')
file_path = '/tmp/test_load_osm_document_with_Date.osm'
if os.path.exists(file_path):
os.remove(file_path)
# We test twice - once to ensure its fetched from the overpass api
# and once to ensure the cached file is used on second access
# Note: There is a small chance the second test could fail if it
# exactly straddles the cache expiry time.
try:
file_handle = load_osm_document(file_path, url)
except:
message = 'load_osm_document from overpass failed %s' % url
LOGGER.exception(message)
raise
string = file_handle.read().decode('utf-8')
LOGGER.info('Checking that Jacoline is in the returned document...')
self.assertIn('Jacoline', string)
file_time = os.path.getmtime(file_path)
#
# This one should be cached now....
#
load_osm_document(file_path, url)
file_time2 = os.path.getmtime(file_path)
message = 'load_osm_document cache test failed.'
LOGGER.info('Checking that downloaded file has a recent timestamp...')
self.assertEqual(file_time, file_time2, message)
LOGGER.info('....OK')
def test_load_osm_document(self):
"""Check that we can fetch an osm doc and that it caches properly."""
#
# NOTE - INTERNET CONNECTION NEEDED FOR THIS TEST
#
url = ('http://overpass-api.de/api/interpreter?data='
'(node(-34.03112731086964,20.44997155666351,'
'-34.029571310785315,20.45501410961151);<;);out+meta;')
file_path = '/tmp/test_load_osm_document.osm'
if os.path.exists(file_path):
os.remove(file_path)
# We test twice - once to ensure its fetched from the overpass api
# and once to ensure the cached file is used on second access
# Note: There is a small chance the second test could fail if it
# exactly straddles the cache expiry time.
try:
file_handle = load_osm_document(file_path, url)
except:
message = 'load_osm_document from overpass failed %s' % url
LOGGER.exception(message)
raise
string = file_handle.read()
message = 'load_osm_document from overpass content check failed.'
assert 'Jacoline' in string, message
#file_handle = load_osm_document(myFilePath, url)
file_time = os.path.getmtime(file_path)
#
# This one should be cached now....
#
load_osm_document(file_path, url)
file_time2 = os.path.getmtime(file_path)
message = 'load_osm_document cache test failed.'
self.assertEqual(file_time, file_time2, message)
def test_load_osm_document(self, mock_open_file, mock_fetch_osm,
mock_get_mtime):
"""Check that we can fetch an osm doc and that it caches properly."""
url = ('http://overpass-api.de/api/interpreter?data='
'(node(-34.03112731086964,20.44997155666351,'
'-34.029571310785315,20.45501410961151);<;);out+meta;')
file_path = '/tmp/test_load_osm_document.osm'
LOGGER.info('url: %s' % url)
LOGGER.info('file_path: %s' % file_path)
if os.path.exists(file_path):
os.remove(file_path)
# We test twice - once to ensure its fetched from the overpass api
# and once to ensure the cached file is used on second access
# Note: There is a small chance the second test could fail if it
# exactly straddles the cache expiry time.
try:
file_handle = load_osm_document(file_path, url)
except:
message = 'load_osm_document from overpass failed %s' % url
LOGGER.exception(message)
raise
# string = file_handle.read().decode('utf-8')
LOGGER.info('Checking that Jacoline is in the returned document...')
# self.assertIn('Jacoline', file_handle)
self.assertIsNotNone(file_handle)
self.assertEquals(file_handle, 'success')
LOGGER.info('....OK')
# file_handle = load_osm_document(myFilePath, url)
file_time = os.path.getmtime(file_path)
#
# This one should be cached now....
#
load_osm_document(file_path, url)
file_time2 = os.path.getmtime(file_path)
message = 'load_osm_document cache test failed.'
LOGGER.info('Checking that downloaded file has a recent timestamp...')
self.assertEqual(file_time, file_time2, message)
LOGGER.info('....OK')
def get_osm_data(feature, polygon):
"""Get osm data.
:param feature: The type of feature to extract:
buildings, building-points, roads, potential-idp, boundary-[1,11]
:type feature: str
:param polygon: list of array describing polygon area e.g.
'[[28.01513671875,-25.77516058680343],[28.855590820312504,-25.567220388070023],
[29.168701171875004,-26.34265280938059]]
:type polygon: list
:raises: OverpassTimeoutException
:returns: A dict from retrieved OSM dataset.
:rtype: dict
"""
server_url = 'http://overpass-api.de/api/interpreter?data='
tag_name = feature
overpass_verbosity = 'body'
try:
polygon_string = split_polygon(polygon)
except ValueError:
error = "Invalid area"
polygon_string = config.POLYGON
feature_type = TAG_MAPPING[tag_name]
parameters = dict()
parameters['print_mode'] = overpass_verbosity
parameters['polygon'] = polygon_string
query = OVERPASS_QUERY_MAP_POLYGON[feature_type].format(**parameters)
# Query to returns json string
query = '[out:json];' + query
encoded_query = quote(query)
url_path = '%s%s' % (server_url, encoded_query)
safe_name = hashlib.md5(query.encode('utf-8')).hexdigest() + '.osm'
file_path = os.path.join(config.CACHE_DIR, safe_name)
osm_document = load_osm_document(file_path, url_path)
osm_data = json.loads(osm_document.read())
regex = 'runtime error:'
if 'remark' in osm_data:
if re.search(regex, osm_data['remark']):
raise OverpassTimeoutException
return osm_data
def test_get_osm_file_with_date_range(self):
"""Check that we can get osm file with date range as query"""
#
# NOTE - INTERNET CONNECTION NEEDED FOR THIS TEST
#
url = (
'http://overpass-api.de/api/interpreter?data='
'[diff:"2012-09-14T15:00:00Z","2015-09-21T15:00:00Z"];'
'(node(-34.03112731086964,20.44997155666351,'
'-34.029571310785315,20.45501410961151);<;);out+meta;')
file_path = '/tmp/test_load_osm_document_with_Date.osm'
if os.path.exists(file_path):
os.remove(file_path)
# We test twice - once to ensure its fetched from the overpass api
# and once to ensure the cached file is used on second access
# Note: There is a small chance the second test could fail if it
# exactly straddles the cache expiry time.
try:
file_handle = load_osm_document(file_path, url)
except:
message = 'load_osm_document from overpass failed %s' % url
LOGGER.exception(message)
raise
string = file_handle.read().decode('utf-8')
LOGGER.info('Checking that Jacoline is in the returned document...')
self.assertIn('Jacoline', string)
file_time = os.path.getmtime(file_path)
#
# This one should be cached now....
#
load_osm_document(file_path, url)
file_time2 = os.path.getmtime(file_path)
message = 'load_osm_document cache test failed.'
LOGGER.info('Checking that downloaded file has a recent timestamp...')
self.assertEqual(file_time, file_time2, message)
LOGGER.info('....OK')