def test_patch_https(self, getenv, create_ssl_sslcontext):
# NB: HTTPSConnection is never mocked here; the monkey-patch applies to the actual httplib library.
# If other tests in the future have any issues with httplib (they shouldn't, the patch is transparent,
# and the original initializer is restored in the finally block), this may be why.
orig_init = auth_requests._ORIG_HTTPSCONNECTION_INIT
try:
new_orig_init = MagicMock()
auth_requests._ORIG_HTTPSCONNECTION_INIT = new_orig_init
# Confirm that the patch is applied
getenv.return_value = "key and cert contents"
auth_requests.patch_https(dict(cert_path="fake/path/to", cert_pass_var="fakepassvar"))
self.assertNotEqual(auth_requests._ORIG_HTTPSCONNECTION_INIT, http.client.HTTPSConnection.__init__)
self.assertEqual(
"_new_init", http.client.HTTPSConnection.__init__.__name__
) # complicated because decorator
with patch("builtins.open", mock_open(read_data="data")):
# Confirm that a base HTTPSConnection picks up the passed ssl context object
create_ssl_sslcontext.return_value = "sslcontext"
http.client.HTTPSConnection()
new_orig_init.assert_called_with(ANY, context="sslcontext")
VerifiedHTTPSConnection()
new_orig_init.assert_called_with(ANY, context="sslcontext")
# Test removing the patch
auth_requests.unpatch_https()
self.assertEqual(http.client.HTTPSConnection.__init__, new_orig_init)
finally:
auth_requests._ORIG_HTTPSCONNECTION_INIT = orig_init
auth_requests.unpatch_https()
def convert(self,):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True, task_uid=self.task_uid, progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True, target=seeder.seed,
kwargs={"tasks": seed_configuration.seeds(['seed']),
"concurrency": get_concurrency(conf_dict),
"progress_logger": progress_logger})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception("The Raster Service failed to complete, please contact an administrator.")
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(self):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get("max_repeat", 5)), task_uid=self.task_uid
)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
conf = yaml.safe_load(self.config) or dict()
cert_info = conf.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=self.name, cred_var=cred_var)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(
task_uid=self.task_uid,
progress_store=progress_store,
verbose=log_settings.get("verbose"),
silent=log_settings.get("silent"),
)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
lambda: seeder.seed(
tasks=seed_configuration.seeds(["seed"]),
concurrency=get_concurrency(conf_dict),
progress_logger=progress_logger,
)
)
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
except CancelException:
raise
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
logger.error(e)
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(self, ):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config(
)
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid,
progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
get_concurrency(conf_dict),
"progress_logger":
progress_logger
})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception(
"The Raster Service failed to complete, please contact an administrator."
)
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def test_patch_https(self, getenv):
# NB: HTTPSConnection is never mocked here; the monkey-patch applies to the actual httplib library.
# If other tests in the future have any issues with httplib (they shouldn't, the patch is transparent,
# and the original initializer is restored in the finally block), this may be why.
orig_init = auth_requests._ORIG_HTTPSCONNECTION_INIT
try:
new_orig_init = MagicMock()
auth_requests._ORIG_HTTPSCONNECTION_INIT = new_orig_init
# Confirm that the patch is applied
getenv.return_value = "key and cert contents"
auth_requests.patch_https("test-provider-slug")
self.assertNotEqual(auth_requests._ORIG_HTTPSCONNECTION_INIT, http.client.HTTPSConnection.__init__)
self.assertEqual("_new_init", http.client.HTTPSConnection.__init__
.__closure__[1].cell_contents.__name__) # complicated because decorator
named_tempfile = MagicMock()
cert_tempfile = MagicMock()
cert_tempfile.name = "temp filename"
named_tempfile.__enter__ = MagicMock(return_value=cert_tempfile)
cert_tempfile.write = MagicMock()
cert_tempfile.flush = MagicMock()
with patch('eventkit_cloud.utils.auth_requests.NamedTemporaryFile', return_value=named_tempfile,
create=True):
# Confirm that a base HTTPSConnection picks up key and cert files
conn = http.client.HTTPSConnection()
getenv.assert_called_with("test_provider_slug_CERT")
new_orig_init.assert_called_with(ANY, key_file="temp filename", cert_file="temp filename")
cert_tempfile.write.assert_called_once_with("key and cert contents".encode())
# Confirm that a MapProxy VerifiedHTTPSConnection picks up key and cert files
cert_tempfile.write.reset_mock()
conn = VerifiedHTTPSConnection()
new_orig_init.assert_called_with(ANY, key_file="temp filename", cert_file="temp filename")
cert_tempfile.write.assert_called_once_with("key and cert contents".encode())
# Test removing the patch
auth_requests.unpatch_https()
self.assertEqual(http.client.HTTPSConnection.__init__, new_orig_init)
finally:
auth_requests._ORIG_HTTPSCONNECTION_INIT = orig_init
auth_requests.unpatch_https()
def create_mapproxy_app(slug: str, user: User = None) -> TestApp:
mapproxy_config_key = get_mapproxy_config_template(slug, user=user)
mapproxy_config = cache.get(mapproxy_config_key)
conf_dict = cache.get_or_set(f"base-config-{slug}", lambda: get_conf_dict(slug), 360)
if not mapproxy_config:
# TODO: place this somewhere else consolidate settings.
base_config: Dict[str, Any] = {
"services": {
"demo": None,
"tms": None,
"wmts": {
"featureinfo_formats": [
{"mimetype": "application/json", "suffix": "json"},
{"mimetype": "application/gml+xml; version=3.1", "suffix": "gml"},
]
},
},
# Cache based on slug so that the caches don't overwrite each other.
"caches": {slug: {"cache": {"type": "file"}, "sources": ["default"], "grids": ["default"]}},
"layers": [{"name": slug, "title": slug, "sources": [slug]}],
"globals": {"cache": {"base_dir": getattr(settings, "TILE_CACHE_DIR")}},
}
if conf_dict["sources"].get("info"):
base_config["caches"][slug]["sources"] += ["info"]
if conf_dict["sources"].get("footprint"):
base_config["caches"][get_footprint_layer_name(slug)] = {
"cache": {"type": "file"},
"sources": ["footprint"],
"grids": ["default"],
}
base_config["layers"] += [
{
"name": get_footprint_layer_name(slug),
"title": get_footprint_layer_name(slug),
"sources": [get_footprint_layer_name(slug)],
}
]
base_config, conf_dict = add_restricted_regions_to_config(base_config, conf_dict, slug, None)
try:
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=base_config)
load_config(mapproxy_config, config_dict=conf_dict)
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
if settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS:
regional_justification_timeout = settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS * 86400
else:
regional_justification_timeout = None
mapproxy_configs_set = cache.get_or_set(mapproxy_config_keys_index, set())
mapproxy_configs_set.add(mapproxy_config_key)
cache.set(mapproxy_config_keys_index, mapproxy_configs_set)
cache.set(mapproxy_config_key, mapproxy_config, regional_justification_timeout)
except ConfigurationError as e:
logger.error(e)
raise
else:
try:
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
except ConfigurationError as e:
logger.error(e)
raise
cert_info = conf_dict.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf_dict.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=slug, cred_var=cred_var)
app = MapProxyApp(mapproxy_configuration.configured_services(), mapproxy_config)
return TestApp(app)