def convert(self,):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True, task_uid=self.task_uid, progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True, target=seeder.seed,
kwargs={"tasks": seed_configuration.seeds(['seed']),
"concurrency": get_concurrency(conf_dict),
"progress_logger": progress_logger})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception("The Raster Service failed to complete, please contact an administrator.")
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(self):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config()
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get("max_repeat", 5)), task_uid=self.task_uid
)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
conf = yaml.safe_load(self.config) or dict()
cert_info = conf.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=self.name, cred_var=cred_var)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(
task_uid=self.task_uid,
progress_store=progress_store,
verbose=log_settings.get("verbose"),
silent=log_settings.get("silent"),
)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
lambda: seeder.seed(
tasks=seed_configuration.seeds(["seed"]),
concurrency=get_concurrency(conf_dict),
progress_logger=progress_logger,
)
)
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
except CancelException:
raise
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
logger.error(e)
raise
finally:
connections.close_all()
return self.gpkgfile
def convert(self, ):
"""
Convert external service to gpkg.
"""
from eventkit_cloud.tasks.task_process import TaskProcess
conf_dict, seed_configuration, mapproxy_configuration = self.get_check_config(
)
# Customizations...
mapproxy.seed.seeder.exp_backoff = get_custom_exp_backoff(
max_repeat=int(conf_dict.get('max_repeat', 5)))
mapproxy.cache.geopackage.GeopackageCache.load_tile_metadata = load_tile_metadata
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
try:
auth_requests.patch_https(self.name)
auth_requests.patch_mapproxy_opener_cache(slug=self.name)
progress_store = get_progress_store(self.gpkgfile)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid,
progress_store=progress_store)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
get_concurrency(conf_dict),
"progress_logger":
progress_logger
})
check_zoom_levels(self.gpkgfile, mapproxy_configuration)
remove_empty_zoom_levels(self.gpkgfile)
set_gpkg_contents_bounds(self.gpkgfile, self.layer, self.bbox)
if task_process.exitcode != 0:
raise Exception(
"The Raster Service failed to complete, please contact an administrator."
)
except Exception:
logger.error("Export failed for url {}.".format(self.service_url))
raise
finally:
connections.close_all()
return self.gpkgfile
def test_mapproxy_opener_patch(self):
orig_call = auth_requests._ORIG_URLOPENERCACHE_CALL
try:
new_orig_call = MagicMock()
auth_requests._ORIG_URLOPENERCACHE_CALL = new_orig_call
# Confirm that the patch is applied
auth_requests.patch_mapproxy_opener_cache()
self.assertEqual("_new_call", _URLOpenerCache.__call__.__name__)
create_url_opener = _URLOpenerCache()
opener = create_url_opener(None, "example.com", "test_user", "test_password")
self.assertTrue(any([isinstance(h, urllib.request.HTTPCookieProcessor) for h in opener.handlers]))
# Test removing the patch
auth_requests.unpatch_mapproxy_opener_cache()
self.assertEqual(new_orig_call, _URLOpenerCache.__call__)
finally:
auth_requests._ORIG_URLOPENERCACHE_CALL = orig_call
auth_requests.unpatch_mapproxy_opener_cache()
def create_mapproxy_app(slug: str, user: User = None) -> TestApp:
mapproxy_config_key = get_mapproxy_config_template(slug, user=user)
mapproxy_config = cache.get(mapproxy_config_key)
conf_dict = cache.get_or_set(f"base-config-{slug}", lambda: get_conf_dict(slug), 360)
if not mapproxy_config:
# TODO: place this somewhere else consolidate settings.
base_config: Dict[str, Any] = {
"services": {
"demo": None,
"tms": None,
"wmts": {
"featureinfo_formats": [
{"mimetype": "application/json", "suffix": "json"},
{"mimetype": "application/gml+xml; version=3.1", "suffix": "gml"},
]
},
},
# Cache based on slug so that the caches don't overwrite each other.
"caches": {slug: {"cache": {"type": "file"}, "sources": ["default"], "grids": ["default"]}},
"layers": [{"name": slug, "title": slug, "sources": [slug]}],
"globals": {"cache": {"base_dir": getattr(settings, "TILE_CACHE_DIR")}},
}
if conf_dict["sources"].get("info"):
base_config["caches"][slug]["sources"] += ["info"]
if conf_dict["sources"].get("footprint"):
base_config["caches"][get_footprint_layer_name(slug)] = {
"cache": {"type": "file"},
"sources": ["footprint"],
"grids": ["default"],
}
base_config["layers"] += [
{
"name": get_footprint_layer_name(slug),
"title": get_footprint_layer_name(slug),
"sources": [get_footprint_layer_name(slug)],
}
]
base_config, conf_dict = add_restricted_regions_to_config(base_config, conf_dict, slug, None)
try:
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=base_config)
load_config(mapproxy_config, config_dict=conf_dict)
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
if settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS:
regional_justification_timeout = settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS * 86400
else:
regional_justification_timeout = None
mapproxy_configs_set = cache.get_or_set(mapproxy_config_keys_index, set())
mapproxy_configs_set.add(mapproxy_config_key)
cache.set(mapproxy_config_keys_index, mapproxy_configs_set)
cache.set(mapproxy_config_key, mapproxy_config, regional_justification_timeout)
except ConfigurationError as e:
logger.error(e)
raise
else:
try:
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
except ConfigurationError as e:
logger.error(e)
raise
cert_info = conf_dict.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf_dict.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=slug, cred_var=cred_var)
app = MapProxyApp(mapproxy_configuration.configured_services(), mapproxy_config)
return TestApp(app)