def get_cached_response(url, payload):
cookies = get_session_cookies()
headers = get_auth_headers()
session = get_or_update_session(headers=headers)
response = session.get(url, params=payload, cookies=cookies)
if response.ok and check_data(response):
return response
def fetch_user_from_token(token):
"""
:param token: Uses OAuth token to retrieve user data from the resource server.
:return: User object.
"""
logger.debug("Sending request: access_token=%s", token)
try:
session = get_or_update_session(token=token)
logger.debug("Session headers: %s", session.headers)
response = session.get(getattr(settings, "OAUTH_PROFILE_URL"))
logger.debug("Received response: %s", response.text)
response.raise_for_status()
except requests.ConnectionError as err:
logger.error("Could not reach OAuth Resource Server: %s", err)
raise OAuthServerUnreachable()
except requests.HTTPError as err:
status_code = err.response.status_code
if status_code == 401:
logger.error("OAuth Resource Server rejected access token: %s",
err.response.text)
raise Unauthorized("OAuth Resource Server rejected access token")
logger.error("OAuth Resource Server returned HTTP %s %s", status_code,
err.response.text)
raise OAuthError(status_code)
orig_data = response.json()
logger.debug(f"OAUTH PROFILE DATA: {orig_data}")
user_data = get_user_data_from_schema(orig_data)
return get_user(user_data, orig_data)
def update_geom(self):
from eventkit_cloud.tasks.helpers import download_data
from eventkit_cloud.ui.helpers import file_to_geojson
geometry = None
if self.config != self.__config:
orig_extent_url = load_provider_config(
self.__config).get("extent_url")
config = load_provider_config(self.config)
extent_url = config.get("extent_url")
if extent_url and extent_url != orig_extent_url:
random_uuid = uuid.uuid4()
session = get_or_update_session(**config)
if not extent_url:
return
output_file = download_data(task_uid=str(random_uuid),
input_url=extent_url,
session=session)
geojson = file_to_geojson(output_file)
geojson_geometry = geojson.get("geometry") or geojson.get(
"features", [{}])[0].get("geometry")
geometry = GEOSGeometry(json.dumps(geojson_geometry),
srid=4326)
elif (self.url != self.__url) or (self.layer != self.__layer):
try:
client = self.get_service_client()
geometry = client.download_geometry()
except AttributeError as e:
# TODO: This fails in tests. How to handle failure to update geometry?
logger.info(e, exc_info=True)
if geometry:
self.the_geom = convert_polygon(geometry)
def add_bbox(self, update_url, data):
# the different gid levels that should be checked for a bbox
ids = [
"neighbourhood_gid",
"locality_gid",
"county_gid",
"region_gid",
"country_gid",
]
search_id = ""
for id in ids:
gid = data.get(id, data.get("properties", None).get(id, None))
# use the gid if it exists and its not gid of the data in question
# (if it is the gid of the data then we should already have a bbox if its available at that level)
if gid and gid != data.get("gid"):
search_id = gid
break
if search_id:
session = get_or_update_session(cert_info=get_geocode_cert_info())
response = session.get(update_url, params={"ids": search_id}).json()
features = response.get("features", [])
if len(features):
feature = features[0]
bbox = feature.get("bbox", None)
if bbox:
data["bbox"] = bbox
data["properties"]["bbox"] = bbox
return data
def __init__(self,
service_url,
layer,
aoi_geojson=None,
slug=None,
max_area=0,
config: dict = None):
"""
Initialize this ProviderCheck object with a service URL and layer.
:param service_url: URL of provider, if applicable. Query string parameters are ignored.
:param layer: Layer or coverage to check for
:param aoi_geojson: (Optional) AOI to check for layer intersection
:param slug: (Optional) A provider slug to use for getting credentials.
:param max_area: The upper limit for this datasource.
"""
self.service_url = service_url
self.query = None
self.layer = layer
self.slug = slug
self.max_area = max_area
self.timeout = 10
self.config = config or dict()
self.session = get_or_update_session(session=None, **self.config)
self.set_aoi(aoi_geojson)
def __init__(self, url, config, session_token, task_id, *args, **kwargs):
self.base_url = url.rstrip("/\\")
self.base_url += "/"
self.config = config
self.task_id = task_id
self.job_url = None
self.session = get_or_update_session(*args, **kwargs)
def get_access_tokens(data):
try:
session = get_or_update_session()
response = session.post(
getattr(settings, "OAUTH_TOKEN_URL"),
auth=(getattr(settings, "OAUTH_CLIENT_ID"),
getattr(settings, "OAUTH_CLIENT_SECRET")),
data=data,
)
logger.debug(f"Received response: {response.text}")
response.raise_for_status()
except requests.ConnectionError as err:
logger.error(f"Could not reach Token Server: {err}")
raise OAuthServerUnreachable()
except requests.HTTPError as err:
status_code = err.response.status_code
if status_code == 401:
logger.error(
f"OAuth server rejected user refresh token: {err.response.text}"
)
raise Unauthorized("OAuth server rejected refresh token.")
logger.error(f"OAuth server returned HTTP {status_code}",
err.response.text)
raise OAuthError(status_code)
oauth_token_key = getattr(settings, "OAUTH_TOKEN_KEY")
access = response.json()
access_token = access.get(oauth_token_key)
refresh_token = access.get(getattr(settings, "OAUTH_REFRESH_KEY"))
if not access_token:
logger.error(
f"OAuth server response missing `{oauth_token_key}`. Response Text:\n{1}"
)
raise InvalidOauthResponse(f"missing `{oauth_token_key}`",
response.text)
return access_token, refresh_token
def get_session(request, provider):
config = load_provider_config(provider.config)
session = get_or_update_session(**config)
session_token = request.session.get("access_token")
valid_token = has_valid_access_token(session_token)
if valid_token:
session.headers.update({"Authorization": f"Bearer: {session_token}"})
return session
def __init__(self, url, config, session_token, task_id, *args, **kwargs):
self.base_url = url.rstrip("/\\")
self.base_url += "/"
self.config = config
self.task_id = task_id
self.job_url = None
logger.info(f"Session: {session_token}, {args}, {kwargs}")
valid_token = has_valid_access_token(session_token)
if not valid_token:
raise Exception("Invalid access token.")
self.session = get_or_update_session(*args, **kwargs)
def test_get_or_update_session(self, mock_get_cred):
expected_headers = {"test": "value"}
cert_info = {"cert_path": "path/to/file", "cert_pass_var": "CERT_PATH"}
expected_user = "******"
expected_pass = "******"
mock_get_cred.return_value = [expected_user, expected_pass]
session = get_or_update_session(headers=expected_headers,
cert_info=cert_info,
slug="abc")
self.assertEqual(session.auth, (expected_user, expected_pass))
self.assertEqual(len(session.adapters), 2)
self.assertTrue(
expected_headers.items() <= dict(session.headers).items())
self.assertEqual(session.verify, 10)
def get_response(self, payload):
error_message = (
"The Geocoding service received an error. Please try again or contact an Eventkit administrator."
)
if os.getenv("GEOCODING_AUTH_CERT"):
response = get_cached_response(self.url, payload)
if not response:
response = get_auth_response(self.url, payload)
if response:
return response
else:
raise Exception(error_message)
else:
session = get_or_update_session()
response = session.get(self.url, params=payload)
if not response.ok:
raise Exception(error_message)
return response
def update_geom(self):
from eventkit_cloud.tasks.helpers import download_data
from eventkit_cloud.ui.helpers import file_to_geojson
if self.config != self.__config:
orig_extent_url = load_provider_config(self.__config).get("extent_url")
config = load_provider_config(self.config)
extent_url = config.get("extent_url")
if extent_url and extent_url != orig_extent_url:
random_uuid = uuid.uuid4()
session = get_or_update_session(**config)
if not extent_url:
return
output_file = download_data(task_uid=str(random_uuid), input_url=extent_url, session=session)
geojson = file_to_geojson(output_file)
geometry = geojson.get("geometry") or geojson.get("features", [{}])[0].get("geometry")
if geometry:
self.the_geom = convert_polygon(GEOSGeometry(json.dumps(geometry), srid=4326))
def authenticate():
auth_response = None
try:
url = getattr(settings, "GEOCODING_AUTH_URL")
if url:
logger.info("Receiving new authentication token for geocoder.")
session = get_or_update_session(cert_info=get_geocode_cert_info())
auth_response = session.get(getattr(settings, "GEOCODING_AUTH_URL")).json()
token = auth_response.get("token")
# if not token set the token key as something so we know not to check this everytime.
cache.set(CACHE_TOKEN_KEY, token or url, CACHE_TOKEN_TIMEOUT)
return token
return None
except requests.exceptions.RequestException:
logger.error("FAILED TO AUTHENTICATE.")
if auth_response:
logger.error(auth_response.content)
cache.delete(CACHE_TOKEN_KEY)
return None
def __init__(self, service_url, layer, aoi_geojson=None, slug=None, max_area=0, config: dict = None):
"""
Initialize this ProviderCheck object with a service URL and layer.
:param service_url: URL of provider, if applicable. Query string parameters are ignored.
:param layer: Layer or coverage to check for
:param aoi_geojson: (Optional) AOI to check for layer intersection
:param slug: (Optional) A provider slug to use for getting credentials.
:param max_area: The upper limit for this datasource.
"""
self.service_url = service_url
self.query = None
self.layer = layer
self.slug = slug
self.max_area = max_area
self.timeout = 10
self.config = config or dict()
self.session = get_or_update_session(session=None, **self.config)
if aoi_geojson is not None and aoi_geojson != "":
if isinstance(aoi_geojson, str):
aoi_geojson = json.loads(aoi_geojson)
geoms = tuple(
[
GEOSGeometry(json.dumps(feature.get("geometry")), srid=4326)
for feature in aoi_geojson.get("features")
]
)
geom_collection = GeometryCollection(geoms, srid=4326)
logger.debug("AOI: {}".format(json.dumps(aoi_geojson)))
self.aoi = geom_collection
else:
self.aoi = None
logger.debug("AOI was not given")
def get_coverage_with_requests(self):
logger.info("Using admin configuration for the WCS request.")
service = self.config.get("service")
params = self.config.get("params")
if not service:
raise Exception(
"A service key needs to be defined to include the scale of source in meters"
)
coverages = service.get("coverages", params.get("COVERAGE"))
coverages = str(coverages).split(",")
if not coverages:
logger.error(
"No coverages were specified for this provider, "
"please specify `coverages` under service or `COVERAGE` under params."
# NOQA
)
raise Exception("Data source incorrectly configured.")
scale = float(service.get("scale"))
params["service"] = "WCS"
width, height = get_dimensions(self.bbox, scale)
tile_bboxes = get_chunked_bbox(self.bbox, (width, height))
geotiffs = []
session = get_or_update_session(slug=self.slug, **self.config)
for idx, coverage in enumerate(coverages):
params["COVERAGE"] = coverage
file_path, ext = os.path.splitext(self.out)
try:
for (
_bbox_idx,
_tile_bbox,
) in enumerate(tile_bboxes):
outfile = "{0}-{1}-{2}{3}".format(file_path, idx,
_bbox_idx, ext)
try:
os.remove(outfile)
except OSError:
pass
# Setting this to arbitrarily high values improves the computed
# resolution but makes the requests slow down.
# If it is set in the config, use that value, otherwise compute approximate res based on scale
if self.config.get("tile_size", None) is None:
tile_x, tile_y = get_dimensions(_tile_bbox, scale)
params["width"] = tile_x
params["height"] = tile_y
else:
params["width"] = self.config.get("tile_size")
params["height"] = self.config.get("tile_size")
params["bbox"] = ",".join(map(str, _tile_bbox))
req = session.get(self.service_url,
params=params,
stream=True)
try:
size = int(req.headers.get("content-length"))
except (ValueError, TypeError):
if req.content:
size = len(req.content)
else:
raise Exception(
"Overpass Query failed to return any data")
if not req:
logger.error(req.content)
raise Exception("WCS request for {0} failed.".format(
self.name))
CHUNK = 1024 * 1024 * 2 # 2MB chunks
from audit_logging.file_logging import logging_open
with logging_open(outfile,
"wb",
user_details=self.user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
size += CHUNK
geotiffs += [outfile]
except Exception as e:
logger.error(e)
raise Exception("There was an error writing the file to disk.")
if len(geotiffs) > 1:
task_process = TaskProcess(self.task_uid)
self.out = merge_geotiffs(geotiffs,
self.out,
executor=task_process.start_process)
else:
shutil.copy(geotiffs[0], self.out)
if not os.path.isfile(self.out):
raise Exception("Nothing was returned from the WCS service.")
if not get_meta(self.out).get("is_raster"):
with open(self.out, "r") as output_file:
logger.error("Content of failed WCS request")
logger.error(output_file.read())
raise Exception("The service failed to return a proper response")
def run_query(self, user_details=None, subtask_percentage=100, subtask_start=0, eta=None):
"""
Run the overpass query.
subtask_percentage is the percentage of the task referenced by self.task_uid this method takes up.
Used to update progress.
Return:
the path to the overpass extract
"""
from eventkit_cloud.tasks.helpers import update_progress
from audit_logging.file_logging import logging_open
# This is just to make it easier to trace when user_details haven't been sent
if user_details is None:
user_details = {"username": "******"}
req = None
query = self.get_query()
logger.debug(query)
logger.debug(f"Query started at: {datetime.now()}")
try:
update_progress(
self.task_uid,
progress=0,
subtask_percentage=subtask_percentage,
subtask_start=subtask_start,
eta=eta,
msg="Querying provider data",
)
conf: dict = yaml.safe_load(self.config) or dict()
session = get_or_update_session(slug=self.slug, **conf)
req = session.post(self.url, data=query, stream=True)
if not req.ok:
# Workaround for https://bugs.python.org/issue27777
query = {"data": query}
req = session.post(self.url, data=query, stream=True)
req.raise_for_status()
try:
total_size = int(req.headers.get("content-length"))
except (ValueError, TypeError):
if req.content:
total_size = len(req.content)
else:
raise Exception("Overpass Query failed to return any data")
# Since the request takes a while, jump progress to a very high percent...
query_percent = 85.0
download_percent = 100.0 - query_percent
update_progress(
self.task_uid,
progress=query_percent,
subtask_percentage=subtask_percentage,
subtask_start=subtask_start,
eta=eta,
msg="Downloading data from provider: 0 of {:.2f} MB(s)".format(total_size / float(1e6)),
)
CHUNK = 1024 * 1024 * 2 # 2MB chunks
update_interval = 1024 * 1024 * 250 # Every 250 MB
written_size = 0
last_update = 0
with logging_open(self.raw_osm, "wb", user_details=user_details) as fd:
for chunk in req.iter_content(CHUNK):
fd.write(chunk)
written_size += CHUNK
# Limit the number of calls to update_progress because every time update_progress is called,
# the ExportTask model is updated, causing django_audit_logging to update the audit way to much
# (via the post_save hook). In the future, we might try still using update progress just as much
# but update the model less to make the audit log less spammed, or making audit_logging only log
# certain model changes rather than logging absolutely everything.
last_update += CHUNK
if last_update > update_interval:
last_update = 0
progress = query_percent + (float(written_size) / float(total_size) * download_percent)
update_progress(
self.task_uid,
progress=progress,
subtask_percentage=subtask_percentage,
subtask_start=subtask_start,
eta=eta,
msg="Downloading data from provider: {:.2f} of {:.2f} MB(s)".format(
written_size / float(1e6), total_size / float(1e6)
),
)
# Done w/ this subtask
update_progress(
self.task_uid,
progress=100,
subtask_percentage=subtask_percentage,
subtask_start=subtask_start,
eta=eta,
msg="Completed downloading data from provider",
)
except exceptions.RequestException as e:
logger.error("Overpass query threw: {0}".format(e))
raise exceptions.RequestException(e)
finally:
if req:
req.close()
logger.debug(f"Query finished at {datetime.now()}")
logger.debug(f"Wrote overpass query results to: {self.raw_osm}")
return self.raw_osm
def get_auth_response(url, payload):
session = get_or_update_session(cert_info=get_geocode_cert_info())
response = session.get(url, params=payload)
if response.ok and check_data(response):
update_session_cookies(session.cookies)
return response
