def which(name, flags=os.X_OK):
"""Search PATH for executable files with the given name.
..note:: This function was taken verbatim from the twisted framework,
licence available here:
http://twistedmatrix.com/trac/browser/tags/releases/twisted-8.2.0/LICENSE
On newer versions of MS-Windows, the PATHEXT environment variable will be
set to the list of file extensions for files considered executable. This
will normally include things like ".EXE". This fuction will also find files
with the given name ending with any of these extensions.
On MS-Windows the only flag that has any meaning is os.F_OK. Any other
flags will be ignored.
:type name: C{str}
:param name: The name for which to search.
:type flags: C{int}
:param flags: Arguments to L{os.access}.
:rtype: C{list}
:param: A list of the full paths to files found, in the
order in which they were found.
"""
if os.path.exists('/usr/bin/%s' % name):
return ['/usr/bin/%s' % name]
if os.path.exists('/usr/local/bin/%s' % name):
return ['/usr/local/bin/%s' % name]
result = []
# pylint: disable=W0141
extensions = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
# pylint: enable=W0141
path = os.environ.get('PATH', None)
# In c6c9b26 we removed this hard coding for issue #529 but I am
# adding it back here in case the user's path does not include the
# gdal binary dir on OSX but it is actually there. (TS)
if sys.platform == 'darwin': # Mac OS X
gdal_prefix = ('/Library/Frameworks/GDAL.framework/'
'Versions/1.10/Programs/')
path = '%s:%s' % (path, gdal_prefix)
message = 'Search path: %s' % path
LOGGER.debug(message)
if path is None:
return []
for p in path.split(os.pathsep):
p = os.path.join(p, name)
if os.access(p, flags):
result.append(p)
for e in extensions:
pext = p + e
if os.access(pext, flags):
result.append(pext)
return result
def fetch_osm(file_path, url_path):
"""Fetch an osm map and store locally.
:param url_path: The path (relative to the ftp root) from which the
file should be retrieved.
:type url_path: str
:param file_path: The path on the filesystem to which the file should
be saved.
:type file_path: str
:returns: The path to the downloaded file.
"""
LOGGER.debug('Getting URL: %s', url_path)
headers = {'User-Agent': 'InaSAFE'}
web_request = Request(url_path, None, headers)
try:
url_handle = urlopen(web_request, timeout=60)
data = url_handle.read().decode('utf-8')
regex = '<remark> runtime error:'
if re.search(regex, data):
raise OverpassTimeoutException
regex = '(elements|meta)'
if not re.search(regex, data):
raise OverpassDoesNotReturnData
if os.path.exists(file_path):
os.remove(file_path)
file_handle = open(file_path, 'wb')
file_handle.write(data.encode('utf-8'))
file_handle.close()
except HTTPError as e:
if e.code == 400:
LOGGER.exception('Bad request to Overpass')
raise OverpassBadRequestException
elif e.code == 419:
raise OverpassConcurrentRequestException
LOGGER.exception('Error with Overpass')
raise e
def fetch_osm(file_path, url_path):
"""Fetch an osm map and store locally.
:param url_path: The path (relative to the ftp root) from which the
file should be retrieved.
:type url_path: str
:param file_path: The path on the filesystem to which the file should
be saved.
:type file_path: str
:returns: The path to the downloaded file.
"""
LOGGER.debug('Getting URL: %s', url_path)
headers = {'User-Agent': 'InaSAFE'}
web_request = Request(url_path, None, headers)
try:
url_handle = urlopen(web_request, timeout=60)
data = url_handle.read().decode('utf-8')
regex = '<remark> runtime error:'
if re.search(regex, data):
raise OverpassTimeoutException
file_handle = open(file_path, 'wb')
file_handle.write(data.encode('utf-8'))
file_handle.close()
except HTTPError as e:
if e.code == 400:
LOGGER.exception('Bad request to Overpass')
raise OverpassBadRequestException
elif e.code == 419:
raise OverpassConcurrentRequestException
LOGGER.exception('Error with Overpass')
raise e
def which(name, flags=os.X_OK):
"""Search PATH for executable files with the given name.
..note:: This function was taken verbatim from the twisted framework,
licence available here:
http://twistedmatrix.com/trac/browser/tags/releases/twisted-8.2.0/LICENSE
On newer versions of MS-Windows, the PATHEXT environment variable will be
set to the list of file extensions for files considered executable. This
will normally include things like ".EXE". This fuction will also find files
with the given name ending with any of these extensions.
On MS-Windows the only flag that has any meaning is os.F_OK. Any other
flags will be ignored.
:type name: C{str}
:param name: The name for which to search.
:type flags: C{int}
:param flags: Arguments to L{os.access}.
:rtype: C{list}
:param: A list of the full paths for files found, in the
order in which they were found.
"""
if os.path.exists('/usr/bin/%s' % name):
return ['/usr/bin/%s' % name]
if os.path.exists('/usr/local/bin/%s' % name):
return ['/usr/local/bin/%s' % name]
result = []
# pylint: disable=W0141
extensions = filter(None, os.environ.get('PATHEXT', '').split(os.pathsep))
# pylint: enable=W0141
path = os.environ.get('PATH', None)
# In c6c9b26 we removed this hard coding for issue #529 but I am
# adding it back here in case the user's path does not include the
# gdal binary dir on OSX but it is actually there. (TS)
if sys.platform == 'darwin': # Mac OS X
postgis_prefix = (
'/Applications/Postgres.app/Contents/Versions/9.4/bin/')
path = '%s:%s' % (path, postgis_prefix)
gdal_prefix = (
'/Library/Frameworks/GDAL.framework/'
'Versions/1.10/Programs/')
path = '%s:%s' % (path, gdal_prefix)
message = 'Search path: %s' % path
LOGGER.debug(message)
if path is None:
return []
for p in path.split(os.pathsep):
p = os.path.join(p, name)
if os.access(p, flags):
result.append(p)
for e in extensions:
pext = p + e
if os.access(pext, flags):
result.append(pext)
return result
def get_osm_file(
coordinates,
feature='all',
overpass_verbosity='body',
date_from=None,
date_to=None):
"""Fetch an osm file given a bounding box using the overpass API.
:param coordinates: Coordinates as a list in the form:
[min lat, min lon, max lat, max lon]
:param feature: The type of feature to extract:
buildings, building-points, roads, potential-idp, boundary-[1,11]
:type feature: str
:param overpass_verbosity: Output verbosity in Overpass.
It can be body, skeleton, ids_only or meta.
:type overpass_verbosity: str
:param date_from: First date for date range.
:type date_from: str
:param date_to: Second date for date range.
:type date_to: str
:returns: A file which has been opened on the retrieved OSM dataset.
:rtype: file
Coordinates look like this:
{'NE_lng': 20.444537401199337,
'SW_lat': -34.0460012312071,
'SW_lng': 20.439494848251343,
'NE_lat': -34.044441058971394}
Example overpass API query for buildings (testable at
http://overpass-turbo.eu/)::
(
node
["building"]
["building"!="no"]
({{bbox}});
way
["building"]
["building"!="no"]
({{bbox}});
rel
["building"]
["building"!="no"]
({{bbox}});
<;);out+meta;
Equivalent url (http encoded)::
"""
server_url = 'http://overpass-api.de/api/interpreter?data='
parameters = coordinates
parameters['print_mode'] = overpass_verbosity
query = OVERPASS_QUERY_MAP[feature].format(**parameters)
if date_from and date_to:
try:
datetime_from = datetime.datetime.utcfromtimestamp(
float(date_from) / 1000.)
datetime_to = datetime.datetime.utcfromtimestamp(
float(date_to) / 1000.)
date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
diff_query = '[diff:"{date_from}", "{date_to}"];'.format(
date_from=datetime_from.strftime(date_format),
date_to=datetime_to.strftime(date_format)
)
query = diff_query + query
except ValueError as e:
LOGGER.debug(e)
encoded_query = quote(query)
url_path = '%s%s' % (server_url, encoded_query)
safe_name = hashlib.md5(query.encode('utf-8')).hexdigest() + '.osm'
file_path = os.path.join(config.CACHE_DIR, safe_name)
return load_osm_document(file_path, url_path)
def get_osm_file(coordinates,
feature='all',
overpass_verbosity='body',
date_from=None,
date_to=None):
"""Fetch an osm file given a bounding box using the overpass API.
:param coordinates: Coordinates as a list in the form:
[min lat, min lon, max lat, max lon]
:param feature: The type of feature to extract:
buildings, building-points, roads, potential-idp, boundary-[1,11]
:type feature: str
:param overpass_verbosity: Output verbosity in Overpass.
It can be body, skeleton, ids_only or meta.
:type overpass_verbosity: str
:param date_from: First date for date range.
:type date_from: str
:param date_to: Second date for date range.
:type date_to: str
:returns: A file which has been opened on the retrieved OSM dataset.
:rtype: file
Coordinates look like this:
{'NE_lng': 20.444537401199337,
'SW_lat': -34.0460012312071,
'SW_lng': 20.439494848251343,
'NE_lat': -34.044441058971394}
Example overpass API query for buildings (testable at
http://overpass-turbo.eu/)::
(
node
["building"]
["building"!="no"]
({{bbox}});
way
["building"]
["building"!="no"]
({{bbox}});
rel
["building"]
["building"!="no"]
({{bbox}});
<;);out+meta;
Equivalent url (http encoded)::
"""
server_url = 'http://overpass-api.de/api/interpreter?data='
parameters = coordinates
parameters['print_mode'] = overpass_verbosity
query = OVERPASS_QUERY_MAP[feature].format(**parameters)
if date_from and date_to:
try:
datetime_from = datetime.datetime.utcfromtimestamp(
float(date_from) / 1000.)
datetime_to = datetime.datetime.utcfromtimestamp(
float(date_to) / 1000.)
date_format = "%Y-%m-%dT%H:%M:%S.%fZ"
diff_query = '[diff:"{date_from}", "{date_to}"];'.format(
date_from=datetime_from.strftime(date_format),
date_to=datetime_to.strftime(date_format))
query = diff_query + query
except ValueError as e:
LOGGER.debug(e)
encoded_query = quote(query)
url_path = '%s%s' % (server_url, encoded_query)
safe_name = hashlib.md5(query.encode('utf-8')).hexdigest() + '.osm'
file_path = os.path.join(config.CACHE_DIR, safe_name)
return load_osm_document(file_path, url_path)
