def generate_confs(tileset, layer, title, ignore_warnings=True, renderd=False):
"""
Default para productos SATMO
Takes a Tileset object and returns mapproxy and seed config files
"""
# Start with a sane configuration using MapProxy's defaults
mapproxy_config = load_default_config()
tileset_conf_json = get_mapproxy_conf(tileset, layer, title)
tileset_conf = yaml.safe_load(tileset_conf_json)
# print tileset_conf_json
# merge our config
load_config(mapproxy_config, config_dict=tileset_conf)
seed_conf_json = get_seed_conf(tileset)
seed_conf = yaml.safe_load(seed_conf_json)
errors, informal_only = validate_options(mapproxy_config)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(
', '.join(errors)))
mapproxy_cf = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(
', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf)
return mapproxy_cf, seed_cf
def generate_confs(tileset, ignore_warnings=True, renderd=False):
"""
Takes a Tileset object and returns mapproxy and seed config files
"""
# Start with a sane configuration using MapProxy's defaults
mapproxy_config = load_default_config()
tileset_conf_json = get_mapproxy_conf(tileset)
tileset_conf = yaml.safe_load(tileset_conf_json)
# merge our config
load_config(mapproxy_config, config_dict=tileset_conf)
seed_conf_json = get_seed_conf(tileset)
seed_conf = yaml.safe_load(seed_conf_json)
errors, informal_only = validate_options(mapproxy_config)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors)))
mapproxy_cf = ProxyConfiguration(mapproxy_config, seed=seed, renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf)
return mapproxy_cf, seed_cf
def configure_mapproxy(extra_config,
seed=False,
ignore_warnings=True,
renderd=False):
"""Create an validate mapproxy configuration based on a dict.
"""
# Start with a sane configuration using MapProxy's defaults
conf_options = load_default_config()
# Merge both
load_config(conf_options, config_dict=extra_config)
# Make sure the config is valid.
errors, informal_only = validate_options(conf_options)
for error in errors:
LOGGER.warn(error)
if errors and not ignore_warnings:
raise ConfigurationError('invalid configuration: %s' %
', '.join(errors))
errors = validate_references(conf_options)
for error in errors:
LOGGER.warn(error)
if errors and not ignore_warnings:
raise ConfigurationError('invalid references: %s' % ', '.join(errors))
conf = ProxyConfiguration(conf_options, seed=seed, renderd=renderd)
return conf
def create_mapproxy_image(yaml_file, img_file):
with open(yaml_file, 'rb') as f:
yaml_text = yaml.load(f)
captured = []
# Inline function that accesses captured. Do not refactor out of this function.
def start_response(status, headers, exc_info=None):
captured[:] = [status, headers, exc_info]
return output.append
output = []
bbox_req, lay_name = get_path_info_params(yaml_text)
path_info = (
'/service?LAYERS={0}&FORMAT=image%2Fpng&SRS=EPSG%3A4326'
'&EXCEPTIONS=application%2Fvnd.ogc.se_inimage&TRANSPARENT=TRUE&SERVICE=WMS&VERSION=1.1.1&'
'REQUEST=GetMap&STYLES=&BBOX={1}&WIDTH=200&HEIGHT=150').format(
lay_name, bbox_req)
conf_options = load_default_config()
# Merge both
load_config(conf_options, config_dict=yaml_text)
conf = ProxyConfiguration(conf_options, seed=False, renderd=False)
# Create a MapProxy App
app = MapProxyApp(conf.configured_services(), conf.base_config)
# Get a response from MapProxyAppy as if it was running standalone.
environ = environ_from_url(path_info)
app_iter = None
try:
app_iter = app(environ, start_response)
except:
return 1
if app_iter is None:
return 1
try:
with open(img_file, 'wb') as img:
img.write(app_iter.next())
except:
return 1
content = 'error'
with open(img_file, 'rb') as img:
content = img.read()
if 'error' in content:
os.remove(img_file)
return 1
return 0
def test_convert(self, seed_template, cache_template, load_config, seeder, seeding_config, connections,
remove_zoom_levels, mock_check_zoom_levels, mock_set_gpkg_contents_bounds,
patch_https, mock_retry):
gpkgfile = '/var/lib/eventkit/test.gpkg'
config = "layers:\r\n - name: imagery\r\n title: imagery\r\n sources: [cache]\r\n\r\nsources:\r\n imagery:\r\n type: tile\r\n grid: webmercator\r\n url: http://a.tile.openstreetmap.fr/hot/%(z)s/%(x)s/%(y)s.png\r\n\r\ngrids:\r\n webmercator:\r\n srs: EPSG:3857\r\n tile_size: [256, 256]\r\n origin: nw"
json_config = real_yaml.load(config)
mapproxy_config = load_default_config()
bbox = [-2, -2, 2, 2]
cache_template.return_value = {'sources': ['imagery'], 'cache': {'type': 'geopackage', 'filename': '/var/lib/eventkit/test.gpkg'}, 'grids': ['webmercator']}
seed_template.return_value = {'coverages': {'geom': {'srs': 'EPSG:4326', 'bbox': [-2, -2, 2, 2]}}, 'seeds': {'seed': {'coverages': ['geom'], 'refresh_before': {'minutes': 0}, 'levels': {'to': 10, 'from': 0}, 'caches': ['cache']}}}
self.task_process.return_value = Mock(exitcode=0)
w2g = MapproxyGeopackage(config=config,
gpkgfile=gpkgfile,
bbox=bbox,
service_url='http://generic.server/WMTS?SERVICE=WMTS&REQUEST=GetTile&TILEMATRIXSET=default028mm&TILEMATRIX=%(z)s&TILEROW=%(y)s&TILECOL=%(x)s&FORMAT=image%%2Fpng',
layer='imagery',
debug=True,
name='imagery',
level_from=0,
level_to=10,
service_type='wmts',
task_uid=self.task_uid)
result = w2g.convert()
mock_check_zoom_levels.assert_called_once()
connections.close_all.assert_called_once()
self.assertEqual(result, gpkgfile)
cache_template.assert_called_once_with(["imagery"], [grids for grids in json_config.get('grids')], gpkgfile, table_name='imagery')
json_config['caches'] = {'cache': {'sources': ['imagery'], 'cache': {'type': 'geopackage', 'filename': '/var/lib/eventkit/test.gpkg'}, 'grids': ['webmercator']}}
json_config['services'] = ['demo']
patch_https.assert_called_once_with('imagery')
load_config.assert_called_once_with(mapproxy_config, config_dict=json_config)
remove_zoom_levels.assert_called_once_with(gpkgfile)
mock_set_gpkg_contents_bounds.assert_called_once_with(gpkgfile, 'imagery', bbox)
seed_template.assert_called_once_with(bbox=bbox, coverage_file=None, level_from=0, level_to=10)
self.task_process.side_effect = Exception()
with self.assertRaises(Exception):
w2g.convert()
def test_convert(self, seed_template, cache_template, load_config, seeder, seeding_config, connections,
remove_zoom_levels, mock_check_zoom_levels, mock_set_gpkg_contents_bounds,
patch_https, mock_retry):
gpkgfile = '/var/lib/eventkit/test.gpkg'
config = "layers:\r\n - name: imagery\r\n title: imagery\r\n sources: [cache]\r\n\r\nsources:\r\n imagery:\r\n type: tile\r\n grid: webmercator\r\n url: http://a.tile.openstreetmap.fr/hot/%(z)s/%(x)s/%(y)s.png\r\n\r\ngrids:\r\n webmercator:\r\n srs: EPSG:3857\r\n tile_size: [256, 256]\r\n origin: nw"
json_config = real_yaml.load(config)
mapproxy_config = load_default_config()
bbox = [-2, -2, 2, 2]
cache_template.return_value = {'sources': ['imagery'], 'cache': {'type': 'geopackage', 'filename': '/var/lib/eventkit/test.gpkg'}, 'grids': ['webmercator']}
seed_template.return_value = {'coverages': {'geom': {'srs': 'EPSG:4326', 'bbox': [-2, -2, 2, 2]}}, 'seeds': {'seed': {'coverages': ['geom'], 'refresh_before': {'minutes': 0}, 'levels': {'to': 10, 'from': 0}, 'caches': ['cache']}}}
self.task_process.return_value = Mock(exitcode=0)
w2g = MapproxyGeopackage(config=config,
gpkgfile=gpkgfile,
bbox=bbox,
service_url='http://generic.server/WMTS?SERVICE=WMTS&REQUEST=GetTile&TILEMATRIXSET=default028mm&TILEMATRIX=%(z)s&TILEROW=%(y)s&TILECOL=%(x)s&FORMAT=image%%2Fpng',
layer='imagery',
debug=True,
name='imagery',
level_from=0,
level_to=10,
service_type='wmts',
task_uid=self.task_uid)
result = w2g.convert()
mock_check_zoom_levels.assert_called_once()
connections.close_all.assert_called_once()
self.assertEqual(result, gpkgfile)
cache_template.assert_called_once_with(["imagery"], [grids for grids in json_config.get('grids')], gpkgfile, table_name='imagery')
json_config['caches'] = {'cache': {'sources': ['imagery'], 'cache': {'type': 'geopackage', 'filename': '/var/lib/eventkit/test.gpkg'}, 'grids': ['webmercator']}}
json_config['services'] = ['demo']
patch_https.assert_called_once_with('imagery')
load_config.assert_called_once_with(mapproxy_config, config_dict=json_config)
remove_zoom_levels.assert_called_once_with(gpkgfile)
mock_set_gpkg_contents_bounds.assert_called_once_with(gpkgfile, 'imagery', bbox)
seed_template.assert_called_once_with(bbox=bbox, coverage_file=None, level_from=0, level_to=10)
self.task_process.side_effect = Exception()
with self.assertRaises(Exception):
w2g.convert()
def create_mapproxy_app(slug: str, user: User = None) -> TestApp:
mapproxy_config_key = get_mapproxy_config_template(slug, user=user)
mapproxy_config = cache.get(mapproxy_config_key)
conf_dict = cache.get_or_set(f"base-config-{slug}", lambda: get_conf_dict(slug), 360)
if not mapproxy_config:
# TODO: place this somewhere else consolidate settings.
base_config: Dict[str, Any] = {
"services": {
"demo": None,
"tms": None,
"wmts": {
"featureinfo_formats": [
{"mimetype": "application/json", "suffix": "json"},
{"mimetype": "application/gml+xml; version=3.1", "suffix": "gml"},
]
},
},
# Cache based on slug so that the caches don't overwrite each other.
"caches": {slug: {"cache": {"type": "file"}, "sources": ["default"], "grids": ["default"]}},
"layers": [{"name": slug, "title": slug, "sources": [slug]}],
"globals": {"cache": {"base_dir": getattr(settings, "TILE_CACHE_DIR")}},
}
if conf_dict["sources"].get("info"):
base_config["caches"][slug]["sources"] += ["info"]
if conf_dict["sources"].get("footprint"):
base_config["caches"][get_footprint_layer_name(slug)] = {
"cache": {"type": "file"},
"sources": ["footprint"],
"grids": ["default"],
}
base_config["layers"] += [
{
"name": get_footprint_layer_name(slug),
"title": get_footprint_layer_name(slug),
"sources": [get_footprint_layer_name(slug)],
}
]
base_config, conf_dict = add_restricted_regions_to_config(base_config, conf_dict, slug, None)
try:
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=base_config)
load_config(mapproxy_config, config_dict=conf_dict)
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
if settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS:
regional_justification_timeout = settings.REGIONAL_JUSTIFICATION_TIMEOUT_DAYS * 86400
else:
regional_justification_timeout = None
mapproxy_configs_set = cache.get_or_set(mapproxy_config_keys_index, set())
mapproxy_configs_set.add(mapproxy_config_key)
cache.set(mapproxy_config_keys_index, mapproxy_configs_set)
cache.set(mapproxy_config_key, mapproxy_config, regional_justification_timeout)
except ConfigurationError as e:
logger.error(e)
raise
else:
try:
mapproxy_configuration = ProxyConfiguration(mapproxy_config)
except ConfigurationError as e:
logger.error(e)
raise
cert_info = conf_dict.get("cert_info")
auth_requests.patch_https(cert_info=cert_info)
cred_var = conf_dict.get("cred_var")
auth_requests.patch_mapproxy_opener_cache(slug=slug, cred_var=cred_var)
app = MapProxyApp(mapproxy_configuration.configured_services(), mapproxy_config)
return TestApp(app)
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config or self.projection:
conf_dict = yaml.safe_load(self.config) or dict()
else:
raise ConfigurationError("MapProxy configuration is required for raster data providers")
if not conf_dict.get("grids"):
conf_dict["grids"] = {
"default": {"srs": "EPSG:4326", "tile_size": [256, 256], "origin": "nw"},
"webmercator": {"srs": "EPSG:3857", "tile_size": [256, 256], "origin": "nw"},
}
elif self.projection:
conf_dict["grids"].update(
{str(self.projection): {"srs": f"EPSG:{self.projection}", "tile_size": [256, 256], "origin": "nw"}}
)
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict["caches"] = conf_dict.get("caches", {})
try:
conf_dict["caches"]["default"]["cache"]["filename"] = self.gpkgfile
except KeyError:
conf_dict["caches"]["default"] = get_cache_template(
["{0}".format(self.layer)],
[grids for grids in conf_dict.get("grids")],
self.gpkgfile,
table_name=self.layer,
)
if self.projection:
conf_dict["caches"]["repro_cache"] = copy.deepcopy(conf_dict["caches"]["default"])
conf_dict["caches"]["repro_cache"]["cache"]["filename"] = self.input_gpkg
conf_dict["caches"]["repro_cache"]["sources"] = []
conf_dict["caches"]["default"]["meta_size"] = [4, 4]
conf_dict["caches"]["default"]["bulk_meta_tiles"] = True
conf_dict["caches"]["default"] = get_cache_template(
["repro_cache"], [str(self.projection)], self.gpkgfile, "default"
)
# Need something listed as a service to pass the mapproxy validation.
conf_dict["services"] = ["demo"]
# disable SSL cert checks
ssl_verify = getattr(settings, "SSL_VERIFICATION", True)
if isinstance(ssl_verify, bool):
if not ssl_verify:
conf_dict["globals"] = {"http": {"ssl_no_cert_checks": ssl_verify}}
else:
conf_dict["globals"] = {"http": {"ssl_ca_certs": ssl_verify}}
# Add autoconfiguration to base_config
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seeder.seed, renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(self.bbox[1], self.bbox[3], rel_tol=0.001):
logger.warning("Using bbox instead of selection, because the area is too small")
self.selection = None
seed_dict = get_seed_template(
bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection,
projection=self.projection,
)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError("MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def get_mapproxy(layer, seed=False, ignore_warnings=True, renderd=False):
"""Creates a mapproxy config for a given layer-like object.
Compatible with django-registry and GeoNode.
"""
bbox = list(wkt2geom(layer.wkt_geometry))
# TODO: Check for correct url
url = 'http://test.registry.org'
# url = str(layer.service.url)
layer_name = str(layer.title)
srs = 'EPSG:4326'
bbox_srs = 'EPSG:4326'
grid_srs = 'EPSG:3857'
default_source = {
'type': 'wms',
'coverage': {
'bbox': bbox,
'srs': srs,
'bbox_srs': bbox_srs,
'supported_srs': ['EPSG:4326', 'EPSG:900913', 'EPSG:3857'],
},
'req': {
'layers': str(layer.title),
'url': url,
'transparent': True,
},
}
if layer.type == 'ESRI:ArcGIS:MapServer' or layer.type == 'ESRI:ArcGIS:ImageServer':
# blindly replace it with /arcgis/
url = url.replace("/ArcGIS/rest/", "/arcgis/")
# same for uppercase
url = url.replace("/arcgis/rest/", "/arcgis/")
# and for old versions
url = url.replace("ArcX/rest/services", "arcx/services")
# in uppercase or lowercase
url = url.replace("arcx/rest/services", "arcx/services")
srs = 'EPSG:3857'
bbox_srs = 'EPSG:3857'
default_source = {
'type': 'arcgis',
'req': {
'url': url,
'grid': 'default_grid',
'transparent': True,
},
}
# A source is the WMS config
sources = {'default_source': default_source}
# A grid is where it will be projects (Mercator in our case)
grids = {
'default_grid': {
'tile_size': [256, 256],
'srs': grid_srs,
'origin': 'nw',
}
}
# A cache that does not store for now. It needs a grid and a source.
caches = {
'default_cache': {
'disable_storage': True,
'grids': ['default_grid'],
'sources': ['default_source']
},
}
# The layer is connected to the cache
layers = [
{
'name': layer_name,
'sources': ['default_cache'],
'title': str(layer.title),
},
]
# Services expose all layers.
# WMS is used for reprojecting
# TMS is used for easy tiles
# Demo is used to test our installation, may be disabled in final version
services = {
'wms': {
'image_formats': ['image/png'],
'md': {
'abstract': 'This is the Harvard HyperMap Proxy.',
'title': 'Harvard HyperMap Proxy'
},
'srs': ['EPSG:4326', 'EPSG:3857'],
'versions': ['1.1.1']
},
'wmts': {
'restful':
True,
'restful_template':
'/{Layer}/{TileMatrixSet}/{TileMatrix}/{TileCol}/{TileRow}.png',
},
'tms': {
'origin': 'nw',
},
'demo': None,
}
global_config = {
'http': {
'ssl_no_cert_checks': True
},
}
# Start with a sane configuration using MapProxy's defaults
conf_options = load_default_config()
# Populate a dictionary with custom config changes
extra_config = {
'caches': caches,
'grids': grids,
'layers': layers,
'services': services,
'sources': sources,
'globals': global_config,
}
yaml_config = yaml.dump(extra_config, default_flow_style=False)
# If you want to test the resulting configuration. Turn on the next
# line and use that to generate a yaml config.
# assert False
# Merge both
load_config(conf_options, config_dict=extra_config)
# TODO: Make sure the config is valid.
errors, informal_only = validate_options(conf_options)
for error in errors:
LOGGER.warn(error)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration')
errors = validate_references(conf_options)
for error in errors:
LOGGER.warn(error)
conf = ProxyConfiguration(conf_options, seed=seed, renderd=renderd)
# Create a MapProxy App
app = MapProxyApp(conf.configured_services(), conf.base_config)
return app, yaml_config
def get_mapproxy(layer, seed=False, ignore_warnings=True, renderd=False):
"""Creates a mapproxy config for a given layers
"""
bbox = [float(layer.bbox_x0), float(layer.bbox_y0), float(layer.bbox_x1), float(layer.bbox_y1)]
url = str(layer.service.url)
layer_name = simple_name(layer.name)
srs = 'EPSG:4326'
bbox_srs = 'EPSG:4326'
grid_srs = 'EPSG:3857'
if layer.type == 'ESRI:ArcGIS:MapServer' or layer.type == 'ESRI:ArcGIS:ImageServer':
url = str(layer.service.url).split('?')[0] + 'WMSServer?'
# blindly replace it with /arcgis/
url = url.replace("/ArcGIS/rest/", "/arcgis/")
# same for uppercase
url = url.replace("/arcgis/rest/", "/arcgis/")
# and for old versions
url = url.replace("ArcX/rest/services", "arcx/services")
# in uppercase or lowercase
url = url.replace("arcx/rest/services", "arcx/services")
srs = 'EPSG:3857'
bbox_srs = 'EPSG:3857'
if layer.type == 'Hypermap:WARPER':
url = str(layer.url.replace("maps//wms", "maps/wms"))
grid_srs = 'EPSG:900913'
if layer.type == 'Hypermap:WorldMap':
url = str(layer.url.replace("maps//wms", "maps/wms"))
default_source = {
'type': 'wms',
'coverage': {
'bbox': bbox,
'srs': srs,
'bbox_srs': bbox_srs,
'supported_srs': ['EPSG:4326', 'EPSG:900913', 'EPSG:3857'],
},
'req': {
'layers': layer_name,
'url': url,
'transparent': True,
},
}
if layer.type == 'ESRI:ArcGIS:MapServer':
default_source = {
'type': 'tile',
'url': str(layer.service.url).split('?')[0] + 'tile/%(z)s/%(y)s/%(x)s',
'grid': 'default_grid',
'transparent': True,
}
# A source is the WMS config
sources = {
'default_source': default_source
}
# A grid is where it will be projects (Mercator in our case)
grids = {
'default_grid': {
'tile_size': [256, 256],
'srs': grid_srs,
'origin': 'nw',
}
}
# A cache that does not store for now. It needs a grid and a source.
caches = {'default_cache':
{
'cache':
{
'type': 'file',
'directory_layout': 'tms',
'directory': os.path.join(tempfile.gettempdir(),
'mapproxy',
'layer',
'%s' % layer.id,
'map',
'wmts',
layer_name,
'default_grid',
),
},
'grids': ['default_grid'],
'sources': ['default_source']},
}
# The layer is connected to the cache
layers = [
{'name': layer_name,
'sources': ['default_cache'],
'title': str(layer.title),
},
]
# Services expose all layers.
# WMS is used for reprojecting
# TMS is used for easy tiles
# Demo is used to test our installation, may be disabled in final version
services = {
'wms': {'image_formats': ['image/png'],
'md': {'abstract': 'This is the Harvard HyperMap Proxy.',
'title': 'Harvard HyperMap Proxy'},
'srs': ['EPSG:4326', 'EPSG:3857'],
'versions': ['1.1.1']},
'wmts': {
'restful': True,
'restful_template':
'/{Layer}/{TileMatrixSet}/{TileMatrix}/{TileCol}/{TileRow}.png',
},
'tms': {
'origin': 'nw',
},
'demo': None,
}
global_config = {
'http': {'ssl_no_cert_checks': True},
}
# Start with a sane configuration using MapProxy's defaults
conf_options = load_default_config()
# Populate a dictionary with custom config changes
extra_config = {
'caches': caches,
'grids': grids,
'layers': layers,
'services': services,
'sources': sources,
'globals': global_config,
}
yaml_config = yaml.dump(extra_config, default_flow_style=False)
# If you want to test the resulting configuration. Turn on the next
# line and use that to generate a yaml config.
# assert False
# Merge both
load_config(conf_options, config_dict=extra_config)
# Make sure the config is valid.
errors, informal_only = validate_options(conf_options)
for error in errors:
log.warn(error)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration')
errors = validate_references(conf_options)
for error in errors:
log.warn(error)
conf = ProxyConfiguration(conf_options, seed=seed, renderd=renderd)
# Create a MapProxy App
app = MapProxyApp(conf.configured_services(), conf.base_config)
# Wrap it in an object that allows to get requests by path as a string.
return TestApp(app), yaml_config
def convert(self, ):
"""
Convert external service to gpkg.
"""
from ..tasks.task_process import TaskProcess
from .geopackage import remove_empty_zoom_levels
if self.config:
conf_dict = yaml.load(self.config)
else:
conf_dict = create_conf_from_url(self.service_url)
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}_{1}".format(self.layer, self.service_type)],
[grids for grids in conf_dict.get('grids')], self.gpkgfile)
# Prevent the service from failing if source has missing tiles.
for source in conf_dict.get('sources'):
if 'wmts' in source:
conf_dict['sources'][source]['transparent'] = True
conf_dict['sources'][source]['on_error'] = {
"other": {
"response": "transparent",
"cache": False
}
}
# disable SSL cert checks
if getattr(settings, "DISABLE_SSL_VERIFICATION", False):
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.01) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.01):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
logger.error(mapproxy_config)
try:
check_service(conf_dict)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
int(getattr(settings, 'MAPPROXY_CONCURRENCY', 1)),
"progress_logger":
progress_logger
})
remove_empty_zoom_levels(self.gpkgfile)
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
errors, informal_only = validate_options(mapproxy_config)
if not informal_only:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(mapproxy_config)
errors, informal_only = validate_seed_conf(seed_dict)
if not informal_only:
logger.error("Mapproxy Seed failed.")
logger.error("Using Seed Configuration:")
logger.error(seed_dict)
raise SeedConfigurationError(
'MapProxy seed configuration error - {}'.format(
', '.join(errors)))
raise e
finally:
connections.close_all()
return self.gpkgfile
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config:
conf_dict = yaml.load(self.config)
else:
raise ConfigurationError("MapProxy configuration is required for raster data providers")
if not conf_dict.get('grids'):
conf_dict['grids'] = {'geodetic': {'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'},
'webmercator': {'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'}}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(["{0}".format(self.layer)],
[grids for grids in conf_dict.get('grids')],
self.gpkgfile, table_name=self.layer)
# Need something listed as a service to pass the mapproxy validation.
conf_dict['services'] = ['demo']
# disable SSL cert checks
ssl_verify = getattr(settings, "SSL_VERIFICATION", True)
if not ssl_verify:
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seeder.seed, renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(self.bbox[0], self.bbox[2], rel_tol=0.001):
logger.warning('Using bbox instead of selection, because the area is too small')
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox, level_from=self.level_from, level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError("MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def get_mapproxy(layer, seed=False, ignore_warnings=True, renderd=False):
"""Creates a mapproxy config for a given layers
"""
bbox = [float(layer.bbox_x0), float(layer.bbox_y0), float(layer.bbox_x1), float(layer.bbox_y1)]
url = str(layer.service.url)
layer_name = simple_name(layer.name)
srs = "EPSG:4326"
bbox_srs = "EPSG:4326"
grid_srs = "EPSG:3857"
if layer.service.type == "ESRI_MapServer" or layer.service.type == "ESRI_ImageServer":
url = str(layer.service.url).split("?")[0] + "WMSServer?"
# blindly replace it with /arcgis/
url = url.replace("/ArcGIS/rest/", "/arcgis/")
# same for uppercase
url = url.replace("/arcgis/rest/", "/arcgis/")
# and for old versions
url = url.replace("ArcX/rest/services", "arcx/services")
# in uppercase or lowercase
url = url.replace("arcx/rest/services", "arcx/services")
srs = "EPSG:3857"
bbox_srs = "EPSG:3857"
if layer.service.type == "WARPER":
url = str(layer.url.replace("maps//wms", "maps/wms"))
grid_srs = "EPSG:900913"
if layer.service.type == "WM":
url = str(layer.url.replace("maps//wms", "maps/wms"))
default_source = {
"type": "wms",
"coverage": {
"bbox": bbox,
"srs": srs,
"bbox_srs": bbox_srs,
"supported_srs": ["EPSG:4326", "EPSG:900913", "EPSG:3857"],
},
"req": {"layers": layer_name, "url": url, "transparent": True},
}
if layer.service.type == "ESRI_MapServer":
default_source = {
"type": "tile",
"url": str(layer.service.url).split("?")[0] + "tile/%(z)s/%(y)s/%(x)s",
"grid": "default_grid",
"transparent": True,
}
# A source is the WMS config
sources = {"default_source": default_source}
# A grid is where it will be projects (Mercator in our case)
grids = {"default_grid": {"tile_size": [256, 256], "srs": grid_srs, "origin": "nw"}}
# A cache that does not store for now. It needs a grid and a source.
caches = {
"default_cache": {
"cache": {
"type": "file",
"directory_layout": "tms",
"directory": os.path.join(
tempfile.gettempdir(),
"mapproxy",
"layer",
"%s" % layer.id,
"map",
"wmts",
layer_name,
"default_grid",
),
},
"grids": ["default_grid"],
"sources": ["default_source"],
}
}
# The layer is connected to the cache
layers = [{"name": layer_name, "sources": ["default_cache"], "title": str(layer.title)}]
# Services expose all layers.
# WMS is used for reprojecting
# TMS is used for easy tiles
# Demo is used to test our installation, may be disabled in final version
services = {
"wms": {
"image_formats": ["image/png"],
"md": {"abstract": "This is the Harvard HyperMap Proxy.", "title": "Harvard HyperMap Proxy"},
"srs": ["EPSG:4326", "EPSG:3857"],
"versions": ["1.1.1"],
},
"wmts": {"restful": True, "restful_template": "/{Layer}/{TileMatrixSet}/{TileMatrix}/{TileCol}/{TileRow}.png"},
"tms": {"origin": "nw"},
"demo": None,
}
# Start with a sane configuration using MapProxy's defaults
conf_options = load_default_config()
# Populate a dictionary with custom config changes
extra_config = {"caches": caches, "grids": grids, "layers": layers, "services": services, "sources": sources}
yaml_config = yaml.dump(extra_config, default_flow_style=False)
# If you want to test the resulting configuration. Turn on the next
# line and use that to generate a yaml config.
# assert False
# Merge both
load_config(conf_options, config_dict=extra_config)
# Make sure the config is valid.
errors, informal_only = validate_options(conf_options)
for error in errors:
log.warn(error)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError("invalid configuration")
errors = validate_references(conf_options)
for error in errors:
log.warn(error)
conf = ProxyConfiguration(conf_options, seed=seed, renderd=renderd)
# Create a MapProxy App
app = MapProxyApp(conf.configured_services(), conf.base_config)
# Wrap it in an object that allows to get requests by path as a string.
return TestApp(app), yaml_config
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config:
conf_dict = yaml.load(self.config)
else:
raise ConfigurationError(
"MapProxy configuration is required for raster data providers")
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}".format(self.layer)],
[grids for grids in conf_dict.get('grids')],
self.gpkgfile,
table_name=self.layer)
# Need something listed as a service to pass the mapproxy validation.
conf_dict['services'] = ['demo']
# disable SSL cert checks
ssl_verify = getattr(settings, "SSL_VERIFICATION", True)
if not ssl_verify:
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.001):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError(
"MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def get_mapproxy(layer, seed=False, ignore_warnings=True, renderd=False):
"""Creates a mapproxy config for a given layers
"""
bbox = [
float(layer.bbox_x0),
float(layer.bbox_y0),
float(layer.bbox_x1),
float(layer.bbox_y1)
]
url = str(layer.service.url)
layer_name = simple_name(layer.name)
srs = 'EPSG:4326'
bbox_srs = 'EPSG:4326'
grid_srs = 'EPSG:3857'
if layer.type == 'ESRI:ArcGIS:MapServer' or layer.type == 'ESRI:ArcGIS:ImageServer':
url = str(layer.service.url).split('?')[0] + 'WMSServer?'
# blindly replace it with /arcgis/
url = url.replace("/ArcGIS/rest/", "/arcgis/")
# same for uppercase
url = url.replace("/arcgis/rest/", "/arcgis/")
# and for old versions
url = url.replace("ArcX/rest/services", "arcx/services")
# in uppercase or lowercase
url = url.replace("arcx/rest/services", "arcx/services")
srs = 'EPSG:3857'
bbox_srs = 'EPSG:3857'
if layer.type == 'Hypermap:WARPER':
url = str(layer.url.replace("maps//wms", "maps/wms"))
grid_srs = 'EPSG:900913'
if layer.type == 'Hypermap:WorldMap':
url = str(layer.url.replace("maps//wms", "maps/wms"))
default_source = {
'type': 'wms',
'coverage': {
'bbox': bbox,
'srs': srs,
'bbox_srs': bbox_srs,
'supported_srs': ['EPSG:4326', 'EPSG:900913', 'EPSG:3857'],
},
'req': {
'layers': layer_name,
'url': url,
'transparent': True,
},
}
if layer.type == 'ESRI:ArcGIS:MapServer':
default_source = {
'type': 'tile',
'url':
str(layer.service.url).split('?')[0] + 'tile/%(z)s/%(y)s/%(x)s',
'grid': 'default_grid',
'transparent': True,
}
# A source is the WMS config
sources = {'default_source': default_source}
# A grid is where it will be projects (Mercator in our case)
grids = {
'default_grid': {
'tile_size': [256, 256],
'srs': grid_srs,
'origin': 'nw',
}
}
# A cache that does not store for now. It needs a grid and a source.
caches = {
'default_cache': {
'cache': {
'type':
'file',
'directory_layout':
'tms',
'directory':
os.path.join(
tempfile.gettempdir(),
'mapproxy',
'layer',
'%s' % layer.id,
'map',
'wmts',
layer_name,
'default_grid',
),
},
'grids': ['default_grid'],
'sources': ['default_source']
},
}
# The layer is connected to the cache
layers = [
{
'name': layer_name,
'sources': ['default_cache'],
'title': str(layer.title),
},
]
# Services expose all layers.
# WMS is used for reprojecting
# TMS is used for easy tiles
# Demo is used to test our installation, may be disabled in final version
services = {
'wms': {
'image_formats': ['image/png'],
'md': {
'abstract': 'This is the Harvard HyperMap Proxy.',
'title': 'Harvard HyperMap Proxy'
},
'srs': ['EPSG:4326', 'EPSG:3857'],
'versions': ['1.1.1']
},
'wmts': {
'restful':
True,
'restful_template':
'/{Layer}/{TileMatrixSet}/{TileMatrix}/{TileCol}/{TileRow}.png',
},
'tms': {
'origin': 'nw',
},
'demo': None,
}
global_config = {
'http': {
'ssl_no_cert_checks': True
},
}
# Start with a sane configuration using MapProxy's defaults
conf_options = load_default_config()
# Populate a dictionary with custom config changes
extra_config = {
'caches': caches,
'grids': grids,
'layers': layers,
'services': services,
'sources': sources,
'globals': global_config,
}
yaml_config = yaml.dump(extra_config, default_flow_style=False)
# If you want to test the resulting configuration. Turn on the next
# line and use that to generate a yaml config.
# assert False
# Merge both
load_config(conf_options, config_dict=extra_config)
# Make sure the config is valid.
errors, informal_only = validate_options(conf_options)
for error in errors:
log.warn(error)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration')
errors = validate_references(conf_options)
for error in errors:
log.warn(error)
conf = ProxyConfiguration(conf_options, seed=seed, renderd=renderd)
# Create a MapProxy App
app = MapProxyApp(conf.configured_services(), conf.base_config)
# Wrap it in an object that allows to get requests by path as a string.
return TestApp(app), yaml_config
def test_convert(
self,
seed_template,
cache_template,
load_config,
seeder,
seeding_config,
connections,
remove_zoom_levels,
mock_check_zoom_levels,
mock_set_gpkg_contents_bounds,
patch_https,
):
with self.settings(SSL_VERIFICATION=True):
gpkgfile = "/var/lib/eventkit/test.gpkg"
config = (
"layers:\r\n - name: default\r\n title: imagery\r\n sources: [default]\r\n\r\nsources:\r\n "
"default:\r\n type: tile\r\n grid: default\r\n "
"url: http://a.tile.openstreetmap.fr/hot/%(z)s/%(x)s/%(y)s.png\r\n\r\ngrids:\r\n default:\r\n "
"srs: WGS84:3857\r\n tile_size: [256, 256]\r\n origin: nw"
)
json_config = real_yaml.safe_load(config)
mapproxy_config = load_default_config()
bbox = [-2, -2, 2, 2]
cache_template.return_value = {
"sources": ["default"],
"cache": {
"type": "geopackage",
"filename": "/var/lib/eventkit/test.gpkg"
},
"grids": ["default"],
}
seed_template.return_value = {
"coverages": {
"geom": {
"srs": "EPSG:4326",
"bbox": [-2, -2, 2, 2]
}
},
"seeds": {
"seed": {
"coverages": ["geom"],
"refresh_before": {
"minutes": 0
},
"levels": {
"to": 10,
"from": 0
},
"caches": ["default"],
}
},
}
self.task_process.return_value = Mock(exitcode=0)
w2g = MapproxyGeopackage(
config=config,
gpkgfile=gpkgfile,
bbox=bbox,
service_url=
"http://generic.server/WMTS?SERVICE=WMTS&REQUEST=GetTile&TILEMATRIXSET=default028mm&"
"TILEMATRIX=%(z)s&TILEROW=%(y)s&TILECOL=%(x)s&FORMAT=image%%2Fpng",
layer="imagery",
debug=True,
name="imagery",
level_from=0,
level_to=10,
service_type="wmts",
task_uid=self.task_uid,
)
result = w2g.convert()
mock_check_zoom_levels.assert_called_once()
connections.close_all.assert_called_once()
self.assertEqual(result, gpkgfile)
cache_template.assert_called_once_with(
["imagery"], [grids for grids in json_config.get("grids")],
gpkgfile,
table_name="imagery")
json_config["caches"] = {
"default": {
"sources": ["default"],
"cache": {
"type": "geopackage",
"filename": "/var/lib/eventkit/test.gpkg"
},
"grids": ["default"],
}
}
json_config["services"] = ["demo"]
patch_https.assert_called_once_with(cert_info=None)
load_config.assert_called_once_with(mapproxy_config,
config_dict=json_config)
remove_zoom_levels.assert_called_once_with(gpkgfile)
mock_set_gpkg_contents_bounds.assert_called_once_with(
gpkgfile, "imagery", bbox)
seed_template.assert_called_once_with(bbox=bbox,
coverage_file=None,
level_from=0,
level_to=10,
projection=None)
self.task_process.side_effect = Exception()
with self.assertRaises(Exception):
w2g.convert()
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config:
conf_dict = yaml.load(self.config)
else:
conf_dict = create_conf_from_url(self.service_url)
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}_{1}".format(self.layer, self.service_type)],
[grids for grids in conf_dict.get('grids')],
self.gpkgfile,
table_name=self.layer)
conf_dict['services'] = ['demo']
# Prevent the service from failing if source has missing tiles.
for source in conf_dict.get('sources') or []:
if 'wmts' in source:
conf_dict['sources'][source]['transparent'] = True
# You can set any number of error codes here, and mapproxy will ignore them any time they appear and
# just skip the tile instead (normally it retries the tile for a very long time before finally erroring
# out and quitting the job). Putting the string "other" as an additional error code will cause mapproxy
# to skip tiles with ANY retrieval error. For now, we want to have mapproxy skip 404 tiles, and retry
# everything else.
conf_dict['sources'][source]['on_error'] = {
404: {
"response": "transparent",
"cache": False
}
}
# disable SSL cert checks
if getattr(settings, "DISABLE_SSL_VERIFICATION", False):
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.001):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError(
"MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration