def generate_confs(tileset, layer, title, ignore_warnings=True, renderd=False):
"""
Default para productos SATMO
Takes a Tileset object and returns mapproxy and seed config files
"""
# Start with a sane configuration using MapProxy's defaults
mapproxy_config = load_default_config()
tileset_conf_json = get_mapproxy_conf(tileset, layer, title)
tileset_conf = yaml.safe_load(tileset_conf_json)
# print tileset_conf_json
# merge our config
load_config(mapproxy_config, config_dict=tileset_conf)
seed_conf_json = get_seed_conf(tileset)
seed_conf = yaml.safe_load(seed_conf_json)
errors, informal_only = validate_options(mapproxy_config)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(
', '.join(errors)))
mapproxy_cf = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(
', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf)
return mapproxy_cf, seed_cf
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config or self.projection:
conf_dict = yaml.safe_load(self.config) or dict()
else:
raise ConfigurationError("MapProxy configuration is required for raster data providers")
if not conf_dict.get("grids"):
conf_dict["grids"] = {
"default": {"srs": "EPSG:4326", "tile_size": [256, 256], "origin": "nw"},
"webmercator": {"srs": "EPSG:3857", "tile_size": [256, 256], "origin": "nw"},
}
elif self.projection:
conf_dict["grids"].update(
{str(self.projection): {"srs": f"EPSG:{self.projection}", "tile_size": [256, 256], "origin": "nw"}}
)
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict["caches"] = conf_dict.get("caches", {})
try:
conf_dict["caches"]["default"]["cache"]["filename"] = self.gpkgfile
except KeyError:
conf_dict["caches"]["default"] = get_cache_template(
["{0}".format(self.layer)],
[grids for grids in conf_dict.get("grids")],
self.gpkgfile,
table_name=self.layer,
)
if self.projection:
conf_dict["caches"]["repro_cache"] = copy.deepcopy(conf_dict["caches"]["default"])
conf_dict["caches"]["repro_cache"]["cache"]["filename"] = self.input_gpkg
conf_dict["caches"]["repro_cache"]["sources"] = []
conf_dict["caches"]["default"]["meta_size"] = [4, 4]
conf_dict["caches"]["default"]["bulk_meta_tiles"] = True
conf_dict["caches"]["default"] = get_cache_template(
["repro_cache"], [str(self.projection)], self.gpkgfile, "default"
)
# Need something listed as a service to pass the mapproxy validation.
conf_dict["services"] = ["demo"]
# disable SSL cert checks
ssl_verify = getattr(settings, "SSL_VERIFICATION", True)
if isinstance(ssl_verify, bool):
if not ssl_verify:
conf_dict["globals"] = {"http": {"ssl_no_cert_checks": ssl_verify}}
else:
conf_dict["globals"] = {"http": {"ssl_ca_certs": ssl_verify}}
# Add autoconfiguration to base_config
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seeder.seed, renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(self.bbox[1], self.bbox[3], rel_tol=0.001):
logger.warning("Using bbox instead of selection, because the area is too small")
self.selection = None
seed_dict = get_seed_template(
bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection,
projection=self.projection,
)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError("MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config:
conf_dict = yaml.load(self.config)
else:
conf_dict = create_conf_from_url(self.service_url)
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}_{1}".format(self.layer, self.service_type)],
[grids for grids in conf_dict.get('grids')],
self.gpkgfile,
table_name=self.layer)
conf_dict['services'] = ['demo']
# Prevent the service from failing if source has missing tiles.
for source in conf_dict.get('sources') or []:
if 'wmts' in source:
conf_dict['sources'][source]['transparent'] = True
# You can set any number of error codes here, and mapproxy will ignore them any time they appear and
# just skip the tile instead (normally it retries the tile for a very long time before finally erroring
# out and quitting the job). Putting the string "other" as an additional error code will cause mapproxy
# to skip tiles with ANY retrieval error. For now, we want to have mapproxy skip 404 tiles, and retry
# everything else.
conf_dict['sources'][source]['on_error'] = {
404: {
"response": "transparent",
"cache": False
}
}
# disable SSL cert checks
if getattr(settings, "DISABLE_SSL_VERIFICATION", False):
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.001):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError(
"MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def generate_confs(tileset, ignore_warnings=True, renderd=False):
"""
Takes a Tileset object and returns mapproxy and seed config files
"""
mapproxy_conf_json = """
{
"services":{
"wms":{
"on_source_errors":"raise",
"image_formats": ["image/png"]
}
},
"layers":[
{
"name":"",
"title":"",
"sources":[
"tileset_cache"
]
}
],
"caches":{
"tileset_cache":{
"grids":[
"webmercator"
],
"sources":[
"tileset_source"
],
"cache":{
"type":"mbtiles",
"filename": "/provide/valid/path/to/file.mbtiles"
}
}
},
"sources":{
"tileset_source":{
}
},
"grids":{
"webmercator":{
"base":"GLOBAL_MERCATOR"
}
},
"globals": {
"image": {
"paletted": false
}
}
}
"""
seed_conf_json = """
{
"coverages": {
"tileset_geom": {
"bbox": [-77.47, 38.72, -76.72, 39.08],
"datasource": "path/to/geom/file.xxx",
"srs": "EPSG:4326"
}
},
"seeds": {
"tileset_seed": {
"refresh_before": {
"minutes": 0
},
"caches": [
"tileset_cache"
],
"levels": {
"from": 0,
"to": 2
},
"coverages": ["tileset_geom"]
}
}
}
"""
seed_conf = yaml.safe_load(seed_conf_json)
mapproxy_conf = yaml.safe_load(mapproxy_conf_json)
print '---- mbtiles file to generate: {}'.format(
get_tileset_filename(tileset.name))
mapproxy_conf['sources']['tileset_source']['type'] = u_to_str(
tileset.server_service_type)
if u_to_str(tileset.server_service_type) == 'wms':
"""
"req":{
"url":"http://*****:*****@192.168.99.100/geoserver/wms?",
"layers":"geonode:ne_50m_admin_0_countries"
},
"""
mapproxy_conf['sources']['tileset_source']['req'] = {}
mapproxy_conf['sources']['tileset_source']['req']['url'] = u_to_str(
tileset.server_url)
mapproxy_conf['sources']['tileset_source']['req']['layers'] = u_to_str(
tileset.layer_name)
elif u_to_str(tileset.server_service_type) == 'tile':
"""
"url": "http://a.tile.openstreetmap.org/%(z)s/%(x)s/%(y)s.png",
"""
mapproxy_conf['sources']['tileset_source']['url'] = u_to_str(
tileset.server_url)
mapproxy_conf['layers'][0]['name'] = u_to_str(tileset.layer_name)
mapproxy_conf['layers'][0]['title'] = u_to_str(tileset.layer_name)
mapproxy_conf['caches']['tileset_cache']['cache'][
'filename'] = get_tileset_filename(tileset.name, 'generating')
if tileset.layer_zoom_start > tileset.layer_zoom_stop:
raise ConfigurationError(
'invalid configuration - zoom start is greater than zoom stop')
seed_conf['seeds']['tileset_seed']['levels'][
'from'] = tileset.layer_zoom_start
seed_conf['seeds']['tileset_seed']['levels'][
'to'] = tileset.layer_zoom_stop
# any specified refresh before for mbtiles will result in regeneration of the tile set
seed_conf['seeds']['tileset_seed']['refresh_before']['minutes'] = 0
if tileset.geom:
geom_type = 'other'
if tileset.geom.startswith('{"'):
geom_type = 'geojson'
elif tileset.geom.lower().startswith(
'polygon') or tileset.geom.lower().startswith('multipolygon'):
geom_type = 'txt'
elif tileset.geom.startswith('['):
geom_type = 'bbox'
if geom_type in ['geojson', 'txt']:
geom_dir = '{}/geoms'.format(get_tileset_dir())
if not os.path.exists(geom_dir):
os.makedirs(geom_dir)
# TODO: remove geom files when done or pair them up with the actual tileset files?
geom_filename = '{}/geoms/{}.{}'.format(get_tileset_dir(),
tileset.name, geom_type)
with open(geom_filename, 'w+') as geom_file:
geom_file.write(tileset.geom)
seed_conf['coverages']['tileset_geom'][
'datasource'] = geom_filename
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
elif geom_type is 'bbox':
seed_conf['coverages']['tileset_geom']['bbox'] = yaml.safe_load(
tileset.geom)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
else:
# if not bbox or file, just set it as is to the datasource since mapproxy can handle other datastores
# and they should work as is
seed_conf['coverages']['tileset_geom'][
'datasource'] = yaml.safe_load(tileset.geom)
print '---- tileset geom_type: {}, geom: {}'.format(
geom_type, tileset.geom)
else:
# if a geom is not specified, remove the coverages key from tileset_seed
seed_conf['seeds']['tileset_seed'].pop('coverages', None)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
print '--[ mapproxy_conf: '
print yaml.dump(mapproxy_conf)
print '--[ seed_conf: '
print yaml.dump(seed_conf)
if tileset.server_username and tileset.server_password:
"""
"http":{
"headers":{
"Authorization":"Basic YWRtaW46Z2Vvc2VydmVy"
}
}
"""
encoded = base64.b64encode('{}:{}'.format(tileset.server_username,
tileset.server_password))
mapproxy_conf['sources']['tileset_source']['http'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers'][
'Authorization'] = 'Basic {}'.format(encoded)
errors, informal_only = validate_mapproxy_conf(mapproxy_conf)
for error in errors:
print error
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(
', '.join(errors)))
cf = ProxyConfiguration(mapproxy_conf,
conf_base_dir=get_tileset_dir(),
seed=seed,
renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
for error in errors:
print error
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(
', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=cf)
return cf, seed_cf
def get_check_config(self):
"""
Create a MapProxy configuration object and verifies its validity
"""
if self.config:
conf_dict = yaml.load(self.config)
else:
raise ConfigurationError(
"MapProxy configuration is required for raster data providers")
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}".format(self.layer)],
[grids for grids in conf_dict.get('grids')],
self.gpkgfile,
table_name=self.layer)
# Need something listed as a service to pass the mapproxy validation.
conf_dict['services'] = ['demo']
# disable SSL cert checks
ssl_verify = getattr(settings, "SSL_VERIFICATION", True)
if not ssl_verify:
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if self.bbox:
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.001) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.001):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
errors = validate_references(conf_dict)
if errors:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(conf_dict)
raise ConfigurationError(
"MapProxy returned the error - {0}".format(", ".join(errors)))
return conf_dict, seed_configuration, mapproxy_configuration
def convert(self, ):
"""
Convert external service to gpkg.
"""
from ..tasks.task_process import TaskProcess
from .geopackage import remove_empty_zoom_levels
if self.config:
conf_dict = yaml.load(self.config)
else:
conf_dict = create_conf_from_url(self.service_url)
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}_{1}".format(self.layer, self.service_type)],
[grids for grids in conf_dict.get('grids')], self.gpkgfile)
# Prevent the service from failing if source has missing tiles.
for source in conf_dict.get('sources'):
if 'wmts' in source:
conf_dict['sources'][source]['transparent'] = True
conf_dict['sources'][source]['on_error'] = {
"other": {
"response": "transparent",
"cache": False
}
}
# disable SSL cert checks
if getattr(settings, "DISABLE_SSL_VERIFICATION", False):
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.01) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.01):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
logger.error(mapproxy_config)
try:
check_service(conf_dict)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
int(getattr(settings, 'MAPPROXY_CONCURRENCY', 1)),
"progress_logger":
progress_logger
})
remove_empty_zoom_levels(self.gpkgfile)
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
errors, informal_only = validate_options(mapproxy_config)
if not informal_only:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(mapproxy_config)
errors, informal_only = validate_seed_conf(seed_dict)
if not informal_only:
logger.error("Mapproxy Seed failed.")
logger.error("Using Seed Configuration:")
logger.error(seed_dict)
raise SeedConfigurationError(
'MapProxy seed configuration error - {}'.format(
', '.join(errors)))
raise e
finally:
connections.close_all()
return self.gpkgfile