def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get('service')
params = self.config.get('params')
if not service:
raise Exception('A service key needs to be defined to include the scale of source in meters')
coverages = service.get('coverages', params.get('COVERAGE'))
coverages = str(coverages).split(',')
if not coverages:
logger.error('No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.')
raise Exception("Data source incorrectly configured.")
logger.info("Getting Dimensions...")
width, height = get_dimensions(self.bbox, float(service.get('scale')))
logger.info("{}, {}".format(width, height))
params['width'] = str(width)
params['height'] = str(height)
params['service'] = 'WCS'
params['bbox'] = ','.join(map(str, self.bbox))
geotiffs = []
for idx, coverage in enumerate(coverages):
params['COVERAGE'] = coverage
file_path, ext = os.path.splitext(self.out)
outfile = '{0}-{1}{2}'.format(file_path, idx, ext)
try:
os.remove(outfile)
except OSError:
pass
try:
req = auth_requests.get(self.service_url, params=params, slug=self.slug, stream=True,
verify=getattr(settings, 'SSL_VERIFICATION', True))
logger.info("Getting the coverage: {0}".format(req.url))
try:
size = int(req.headers.get('content-length'))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception("Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile, 'wb', user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = merge_geotiffs(geotiffs, self.out, task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
def get_check_response(self):
"""
Sends a GET request to provider URL and returns its response if status code is ok
"""
try:
if not self.service_url:
self.result = CheckResults.NO_URL
return None
response = auth_requests.get(self.service_url,
slug=self.slug,
params=self.query,
timeout=self.timeout)
self.token_dict['status'] = response.status_code
if response.status_code in [401, 403]:
self.result = CheckResults.UNAUTHORIZED
return None
if response.status_code == 404:
self.result = CheckResults.NOT_FOUND
return None
if not response.ok:
self.result = CheckResults.UNAVAILABLE
return None
except requests.exceptions.ConnectTimeout as ex:
logger.error("Provider check timed out for URL {}".format(
self.service_url))
self.result = CheckResults.TIMEOUT
return None
except requests.exceptions.SSLError as ex:
logger.error("SSL connection failed for URL {}: {}".format(
self.service_url, ex.message))
self.result = CheckResults.SSL_EXCEPTION
return None
except requests.exceptions.ConnectionError as ex:
logger.error("Provider check failed for URL {}: {}".format(
self.service_url, ex.message))
self.result = CheckResults.CONNECTION
return None
return response
def get_osm_last_update(url, slug=None):
"""
:param url: A path to the overpass api.
:param slug: Optionally a slug if credentials are needed
:return: The default timestamp as a string (2018-06-18T13:09:59Z)
"""
try:
timestamp_url = "{0}timestamp".format(url.rstrip('/').rstrip('interpreter'))
response = auth_requests.get(timestamp_url, slug=slug)
if response:
return response.content.decode()
raise Exception("Get OSM last update failed with {0}: {1}".format(response.status_code, response.content))
except Exception as e:
logger.warning(e)
logger.warning("Could not get the timestamp from the overpass url.")
return None
def get_osm_last_update(url, slug=None):
"""
:param url: A path to the overpass api.
:param slug: Optionally a slug if credentials are needed
:return: The default timestamp as a string (2018-06-18T13:09:59Z)
"""
try:
timestamp_url = "{0}timestamp".format(
url.rstrip('/').rstrip('interpreter'))
response = auth_requests.get(timestamp_url, slug=slug)
if response:
return response.content.decode()
raise Exception("Get OSM last update failed with {0}: {1}".format(
response.status_code, response.content))
except Exception as e:
logger.warning(e)
logger.warning("Could not get the timestamp from the overpass url.")
return None
def get_check_response(self):
"""
Sends a GET request to provider URL and returns its response if status code is ok
"""
try:
if not self.service_url:
self.result = CheckResults.NO_URL
return None
response = auth_requests.get(self.service_url, slug=self.slug, params=self.query, timeout=self.timeout,
verify=self.verify)
self.token_dict['status'] = response.status_code
if response.status_code in [401, 403]:
self.result = CheckResults.UNAUTHORIZED
return None
if response.status_code == 404:
self.result = CheckResults.NOT_FOUND
return None
if not response.ok:
self.result = CheckResults.UNAVAILABLE
return None
except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout) as ex:
logger.error("Provider check timed out for URL {}".format(self.service_url))
self.result = CheckResults.TIMEOUT
return None
except requests.exceptions.SSLError as ex:
logger.error("SSL connection failed for URL {}: {}".format(self.service_url, str(ex)))
self.result = CheckResults.SSL_EXCEPTION
return None
except requests.exceptions.ConnectionError as ex:
logger.error("Provider check failed for URL {}: {}".format(self.service_url, str(ex)))
self.result = CheckResults.CONNECTION
return None
return response
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get('service')
params = self.config.get('params')
if not service:
raise Exception(
'A service key needs to be defined to include the scale of source in meters'
)
coverages = service.get('coverages', params.get('COVERAGE'))
coverages = str(coverages).split(',')
if not coverages:
logger.error(
'No coverages were specified for this provider, please specify `coverages` under service or `COVERAGE` under params.'
)
raise Exception("Data source incorrectly configured.")
logger.info("Getting Dimensions...")
width, height = get_dimensions(self.bbox, float(service.get('scale')))
logger.info("{}, {}".format(width, height))
params['width'] = str(width)
params['height'] = str(height)
params['service'] = 'WCS'
params['bbox'] = ','.join(map(str, self.bbox))
geotiffs = []
for idx, coverage in enumerate(coverages):
params['COVERAGE'] = coverage
file_path, ext = os.path.splitext(self.out)
outfile = '{0}-{1}{2}'.format(file_path, idx, ext)
try:
os.remove(outfile)
except OSError:
pass
try:
req = auth_requests.get(self.service_url,
params=params,
slug=self.slug,
stream=True,
verify=getattr(settings,
'SSL_VERIFICATION',
True))
logger.info("Getting the coverage: {0}".format(req.url))
try:
size = int(req.headers.get('content-length'))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
'wb',
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
self.out = merge_geotiffs(geotiffs,
self.out,
task_uid=self.task_uid)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")