def get_cached_response(url, payload): cookies = get_session_cookies() headers = get_auth_headers() session = get_or_update_session(headers=headers) response = session.get(url, params=payload, cookies=cookies) if response.ok and check_data(response): return response
def fetch_user_from_token(token): """ :param token: Uses OAuth token to retrieve user data from the resource server. :return: User object. """ logger.debug("Sending request: access_token=%s", token) try: session = get_or_update_session(token=token) logger.debug("Session headers: %s", session.headers) response = session.get(getattr(settings, "OAUTH_PROFILE_URL")) logger.debug("Received response: %s", response.text) response.raise_for_status() except requests.ConnectionError as err: logger.error("Could not reach OAuth Resource Server: %s", err) raise OAuthServerUnreachable() except requests.HTTPError as err: status_code = err.response.status_code if status_code == 401: logger.error("OAuth Resource Server rejected access token: %s", err.response.text) raise Unauthorized("OAuth Resource Server rejected access token") logger.error("OAuth Resource Server returned HTTP %s %s", status_code, err.response.text) raise OAuthError(status_code) orig_data = response.json() logger.debug(f"OAUTH PROFILE DATA: {orig_data}") user_data = get_user_data_from_schema(orig_data) return get_user(user_data, orig_data)
def update_geom(self): from eventkit_cloud.tasks.helpers import download_data from eventkit_cloud.ui.helpers import file_to_geojson geometry = None if self.config != self.__config: orig_extent_url = load_provider_config( self.__config).get("extent_url") config = load_provider_config(self.config) extent_url = config.get("extent_url") if extent_url and extent_url != orig_extent_url: random_uuid = uuid.uuid4() session = get_or_update_session(**config) if not extent_url: return output_file = download_data(task_uid=str(random_uuid), input_url=extent_url, session=session) geojson = file_to_geojson(output_file) geojson_geometry = geojson.get("geometry") or geojson.get( "features", [{}])[0].get("geometry") geometry = GEOSGeometry(json.dumps(geojson_geometry), srid=4326) elif (self.url != self.__url) or (self.layer != self.__layer): try: client = self.get_service_client() geometry = client.download_geometry() except AttributeError as e: # TODO: This fails in tests. How to handle failure to update geometry? logger.info(e, exc_info=True) if geometry: self.the_geom = convert_polygon(geometry)
def add_bbox(self, update_url, data): # the different gid levels that should be checked for a bbox ids = [ "neighbourhood_gid", "locality_gid", "county_gid", "region_gid", "country_gid", ] search_id = "" for id in ids: gid = data.get(id, data.get("properties", None).get(id, None)) # use the gid if it exists and its not gid of the data in question # (if it is the gid of the data then we should already have a bbox if its available at that level) if gid and gid != data.get("gid"): search_id = gid break if search_id: session = get_or_update_session(cert_info=get_geocode_cert_info()) response = session.get(update_url, params={"ids": search_id}).json() features = response.get("features", []) if len(features): feature = features[0] bbox = feature.get("bbox", None) if bbox: data["bbox"] = bbox data["properties"]["bbox"] = bbox return data
def __init__(self, service_url, layer, aoi_geojson=None, slug=None, max_area=0, config: dict = None): """ Initialize this ProviderCheck object with a service URL and layer. :param service_url: URL of provider, if applicable. Query string parameters are ignored. :param layer: Layer or coverage to check for :param aoi_geojson: (Optional) AOI to check for layer intersection :param slug: (Optional) A provider slug to use for getting credentials. :param max_area: The upper limit for this datasource. """ self.service_url = service_url self.query = None self.layer = layer self.slug = slug self.max_area = max_area self.timeout = 10 self.config = config or dict() self.session = get_or_update_session(session=None, **self.config) self.set_aoi(aoi_geojson)
def __init__(self, url, config, session_token, task_id, *args, **kwargs): self.base_url = url.rstrip("/\\") self.base_url += "/" self.config = config self.task_id = task_id self.job_url = None self.session = get_or_update_session(*args, **kwargs)
def get_access_tokens(data): try: session = get_or_update_session() response = session.post( getattr(settings, "OAUTH_TOKEN_URL"), auth=(getattr(settings, "OAUTH_CLIENT_ID"), getattr(settings, "OAUTH_CLIENT_SECRET")), data=data, ) logger.debug(f"Received response: {response.text}") response.raise_for_status() except requests.ConnectionError as err: logger.error(f"Could not reach Token Server: {err}") raise OAuthServerUnreachable() except requests.HTTPError as err: status_code = err.response.status_code if status_code == 401: logger.error( f"OAuth server rejected user refresh token: {err.response.text}" ) raise Unauthorized("OAuth server rejected refresh token.") logger.error(f"OAuth server returned HTTP {status_code}", err.response.text) raise OAuthError(status_code) oauth_token_key = getattr(settings, "OAUTH_TOKEN_KEY") access = response.json() access_token = access.get(oauth_token_key) refresh_token = access.get(getattr(settings, "OAUTH_REFRESH_KEY")) if not access_token: logger.error( f"OAuth server response missing `{oauth_token_key}`. Response Text:\n{1}" ) raise InvalidOauthResponse(f"missing `{oauth_token_key}`", response.text) return access_token, refresh_token
def get_session(request, provider): config = load_provider_config(provider.config) session = get_or_update_session(**config) session_token = request.session.get("access_token") valid_token = has_valid_access_token(session_token) if valid_token: session.headers.update({"Authorization": f"Bearer: {session_token}"}) return session
def __init__(self, url, config, session_token, task_id, *args, **kwargs): self.base_url = url.rstrip("/\\") self.base_url += "/" self.config = config self.task_id = task_id self.job_url = None logger.info(f"Session: {session_token}, {args}, {kwargs}") valid_token = has_valid_access_token(session_token) if not valid_token: raise Exception("Invalid access token.") self.session = get_or_update_session(*args, **kwargs)
def test_get_or_update_session(self, mock_get_cred): expected_headers = {"test": "value"} cert_info = {"cert_path": "path/to/file", "cert_pass_var": "CERT_PATH"} expected_user = "******" expected_pass = "******" mock_get_cred.return_value = [expected_user, expected_pass] session = get_or_update_session(headers=expected_headers, cert_info=cert_info, slug="abc") self.assertEqual(session.auth, (expected_user, expected_pass)) self.assertEqual(len(session.adapters), 2) self.assertTrue( expected_headers.items() <= dict(session.headers).items()) self.assertEqual(session.verify, 10)
def get_response(self, payload): error_message = ( "The Geocoding service received an error. Please try again or contact an Eventkit administrator." ) if os.getenv("GEOCODING_AUTH_CERT"): response = get_cached_response(self.url, payload) if not response: response = get_auth_response(self.url, payload) if response: return response else: raise Exception(error_message) else: session = get_or_update_session() response = session.get(self.url, params=payload) if not response.ok: raise Exception(error_message) return response
def update_geom(self): from eventkit_cloud.tasks.helpers import download_data from eventkit_cloud.ui.helpers import file_to_geojson if self.config != self.__config: orig_extent_url = load_provider_config(self.__config).get("extent_url") config = load_provider_config(self.config) extent_url = config.get("extent_url") if extent_url and extent_url != orig_extent_url: random_uuid = uuid.uuid4() session = get_or_update_session(**config) if not extent_url: return output_file = download_data(task_uid=str(random_uuid), input_url=extent_url, session=session) geojson = file_to_geojson(output_file) geometry = geojson.get("geometry") or geojson.get("features", [{}])[0].get("geometry") if geometry: self.the_geom = convert_polygon(GEOSGeometry(json.dumps(geometry), srid=4326))
def authenticate(): auth_response = None try: url = getattr(settings, "GEOCODING_AUTH_URL") if url: logger.info("Receiving new authentication token for geocoder.") session = get_or_update_session(cert_info=get_geocode_cert_info()) auth_response = session.get(getattr(settings, "GEOCODING_AUTH_URL")).json() token = auth_response.get("token") # if not token set the token key as something so we know not to check this everytime. cache.set(CACHE_TOKEN_KEY, token or url, CACHE_TOKEN_TIMEOUT) return token return None except requests.exceptions.RequestException: logger.error("FAILED TO AUTHENTICATE.") if auth_response: logger.error(auth_response.content) cache.delete(CACHE_TOKEN_KEY) return None
def __init__(self, service_url, layer, aoi_geojson=None, slug=None, max_area=0, config: dict = None): """ Initialize this ProviderCheck object with a service URL and layer. :param service_url: URL of provider, if applicable. Query string parameters are ignored. :param layer: Layer or coverage to check for :param aoi_geojson: (Optional) AOI to check for layer intersection :param slug: (Optional) A provider slug to use for getting credentials. :param max_area: The upper limit for this datasource. """ self.service_url = service_url self.query = None self.layer = layer self.slug = slug self.max_area = max_area self.timeout = 10 self.config = config or dict() self.session = get_or_update_session(session=None, **self.config) if aoi_geojson is not None and aoi_geojson != "": if isinstance(aoi_geojson, str): aoi_geojson = json.loads(aoi_geojson) geoms = tuple( [ GEOSGeometry(json.dumps(feature.get("geometry")), srid=4326) for feature in aoi_geojson.get("features") ] ) geom_collection = GeometryCollection(geoms, srid=4326) logger.debug("AOI: {}".format(json.dumps(aoi_geojson))) self.aoi = geom_collection else: self.aoi = None logger.debug("AOI was not given")
def get_coverage_with_requests(self): logger.info("Using admin configuration for the WCS request.") service = self.config.get("service") params = self.config.get("params") if not service: raise Exception( "A service key needs to be defined to include the scale of source in meters" ) coverages = service.get("coverages", params.get("COVERAGE")) coverages = str(coverages).split(",") if not coverages: logger.error( "No coverages were specified for this provider, " "please specify `coverages` under service or `COVERAGE` under params." # NOQA ) raise Exception("Data source incorrectly configured.") scale = float(service.get("scale")) params["service"] = "WCS" width, height = get_dimensions(self.bbox, scale) tile_bboxes = get_chunked_bbox(self.bbox, (width, height)) geotiffs = [] session = get_or_update_session(slug=self.slug, **self.config) for idx, coverage in enumerate(coverages): params["COVERAGE"] = coverage file_path, ext = os.path.splitext(self.out) try: for ( _bbox_idx, _tile_bbox, ) in enumerate(tile_bboxes): outfile = "{0}-{1}-{2}{3}".format(file_path, idx, _bbox_idx, ext) try: os.remove(outfile) except OSError: pass # Setting this to arbitrarily high values improves the computed # resolution but makes the requests slow down. # If it is set in the config, use that value, otherwise compute approximate res based on scale if self.config.get("tile_size", None) is None: tile_x, tile_y = get_dimensions(_tile_bbox, scale) params["width"] = tile_x params["height"] = tile_y else: params["width"] = self.config.get("tile_size") params["height"] = self.config.get("tile_size") params["bbox"] = ",".join(map(str, _tile_bbox)) req = session.get(self.service_url, params=params, stream=True) try: size = int(req.headers.get("content-length")) except (ValueError, TypeError): if req.content: size = len(req.content) else: raise Exception( "Overpass Query failed to return any data") if not req: logger.error(req.content) raise Exception("WCS request for {0} failed.".format( self.name)) CHUNK = 1024 * 1024 * 2 # 2MB chunks from audit_logging.file_logging import logging_open with logging_open(outfile, "wb", user_details=self.user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) size += CHUNK geotiffs += [outfile] except Exception as e: logger.error(e) raise Exception("There was an error writing the file to disk.") if len(geotiffs) > 1: task_process = TaskProcess(self.task_uid) self.out = merge_geotiffs(geotiffs, self.out, executor=task_process.start_process) else: shutil.copy(geotiffs[0], self.out) if not os.path.isfile(self.out): raise Exception("Nothing was returned from the WCS service.") if not get_meta(self.out).get("is_raster"): with open(self.out, "r") as output_file: logger.error("Content of failed WCS request") logger.error(output_file.read()) raise Exception("The service failed to return a proper response")
def run_query(self, user_details=None, subtask_percentage=100, subtask_start=0, eta=None): """ Run the overpass query. subtask_percentage is the percentage of the task referenced by self.task_uid this method takes up. Used to update progress. Return: the path to the overpass extract """ from eventkit_cloud.tasks.helpers import update_progress from audit_logging.file_logging import logging_open # This is just to make it easier to trace when user_details haven't been sent if user_details is None: user_details = {"username": "******"} req = None query = self.get_query() logger.debug(query) logger.debug(f"Query started at: {datetime.now()}") try: update_progress( self.task_uid, progress=0, subtask_percentage=subtask_percentage, subtask_start=subtask_start, eta=eta, msg="Querying provider data", ) conf: dict = yaml.safe_load(self.config) or dict() session = get_or_update_session(slug=self.slug, **conf) req = session.post(self.url, data=query, stream=True) if not req.ok: # Workaround for https://bugs.python.org/issue27777 query = {"data": query} req = session.post(self.url, data=query, stream=True) req.raise_for_status() try: total_size = int(req.headers.get("content-length")) except (ValueError, TypeError): if req.content: total_size = len(req.content) else: raise Exception("Overpass Query failed to return any data") # Since the request takes a while, jump progress to a very high percent... query_percent = 85.0 download_percent = 100.0 - query_percent update_progress( self.task_uid, progress=query_percent, subtask_percentage=subtask_percentage, subtask_start=subtask_start, eta=eta, msg="Downloading data from provider: 0 of {:.2f} MB(s)".format(total_size / float(1e6)), ) CHUNK = 1024 * 1024 * 2 # 2MB chunks update_interval = 1024 * 1024 * 250 # Every 250 MB written_size = 0 last_update = 0 with logging_open(self.raw_osm, "wb", user_details=user_details) as fd: for chunk in req.iter_content(CHUNK): fd.write(chunk) written_size += CHUNK # Limit the number of calls to update_progress because every time update_progress is called, # the ExportTask model is updated, causing django_audit_logging to update the audit way to much # (via the post_save hook). In the future, we might try still using update progress just as much # but update the model less to make the audit log less spammed, or making audit_logging only log # certain model changes rather than logging absolutely everything. last_update += CHUNK if last_update > update_interval: last_update = 0 progress = query_percent + (float(written_size) / float(total_size) * download_percent) update_progress( self.task_uid, progress=progress, subtask_percentage=subtask_percentage, subtask_start=subtask_start, eta=eta, msg="Downloading data from provider: {:.2f} of {:.2f} MB(s)".format( written_size / float(1e6), total_size / float(1e6) ), ) # Done w/ this subtask update_progress( self.task_uid, progress=100, subtask_percentage=subtask_percentage, subtask_start=subtask_start, eta=eta, msg="Completed downloading data from provider", ) except exceptions.RequestException as e: logger.error("Overpass query threw: {0}".format(e)) raise exceptions.RequestException(e) finally: if req: req.close() logger.debug(f"Query finished at {datetime.now()}") logger.debug(f"Wrote overpass query results to: {self.raw_osm}") return self.raw_osm
def get_auth_response(url, payload): session = get_or_update_session(cert_info=get_geocode_cert_info()) response = session.get(url, params=payload) if response.ok and check_data(response): update_session_cookies(session.cookies) return response