def generate_confs(tileset, layer, title, ignore_warnings=True, renderd=False):
"""
Default para productos SATMO
Takes a Tileset object and returns mapproxy and seed config files
"""
# Start with a sane configuration using MapProxy's defaults
mapproxy_config = load_default_config()
tileset_conf_json = get_mapproxy_conf(tileset, layer, title)
tileset_conf = yaml.safe_load(tileset_conf_json)
# print tileset_conf_json
# merge our config
load_config(mapproxy_config, config_dict=tileset_conf)
seed_conf_json = get_seed_conf(tileset)
seed_conf = yaml.safe_load(seed_conf_json)
errors, informal_only = validate_options(mapproxy_config)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(
', '.join(errors)))
mapproxy_cf = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(
', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf)
return mapproxy_cf, seed_cf
def generate_confs(tileset, ignore_warnings=True, renderd=False):
"""
Takes a Tileset object and returns mapproxy and seed config files
"""
# Start with a sane configuration using MapProxy's defaults
mapproxy_config = load_default_config()
tileset_conf_json = get_mapproxy_conf(tileset)
tileset_conf = yaml.safe_load(tileset_conf_json)
# merge our config
load_config(mapproxy_config, config_dict=tileset_conf)
seed_conf_json = get_seed_conf(tileset)
seed_conf = yaml.safe_load(seed_conf_json)
errors, informal_only = validate_options(mapproxy_config)
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors)))
mapproxy_cf = ProxyConfiguration(mapproxy_config, seed=seed, renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf)
return mapproxy_cf, seed_cf
def test_union(self):
conf = {
'coverages': {
'covname': {
'union': [
{'bbox': [0, 0, 10, 10], 'srs': 'EPSG:4326'},
{'bbox': [10, 0, 20, 10], 'srs': 'EPSG:4326', 'unknown': True},
],
},
},
}
errors, informal_only = validate_seed_conf(conf)
assert informal_only
assert len(errors) == 1
assert errors[0] == "unknown 'unknown' in coverages.covname.union[1]"
def test_union(self):
conf = {
'coverages': {
'covname': {
'union': [
{'bbox': [0, 0, 10, 10], 'srs': 'EPSG:4326'},
{'bbox': [10, 0, 20, 10], 'srs': 'EPSG:4326', 'unknown': True},
],
},
},
}
errors, informal_only = validate_seed_conf(conf)
assert informal_only
assert len(errors) == 1
eq_(errors[0], "unknown 'unknown' in coverages.covname.union[1]")
def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
try:
conf = load_yaml_file(seed_conf_filename)
except YAMLError as ex:
raise SeedConfigurationError(ex)
if 'views' in conf:
# TODO: deprecate old config
seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
else:
errors, informal_only = validate_seed_conf(conf)
for error in errors:
log.warn(error)
if not informal_only:
raise SeedConfigurationError('invalid configuration')
seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
return seed_conf
def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
try:
conf = load_yaml_file(seed_conf_filename)
except YAMLError as ex:
raise SeedConfigurationError(ex)
if 'views' in conf:
# TODO: deprecate old config
seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
else:
errors, informal_only = validate_seed_conf(conf)
for error in errors:
log.warn(error)
if not informal_only:
raise SeedConfigurationError('invalid configuration')
seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
return seed_conf
import logging
log = logging.getLogger('mapproxy.seed.config')
def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
try:
conf = load_yaml_file(seed_conf_filename)
except YAMLError, ex:
raise SeedConfigurationError(ex)
if 'views' in conf:
# TODO: deprecate old config
seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
else:
errors, informal_only = validate_seed_conf(conf)
for error in errors:
log.warn(error)
if not informal_only:
raise SeedConfigurationError('invalid configuration')
seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
return seed_conf
class LegacySeedingConfiguration(object):
"""
Read old seed.yaml configuration (with seed and views).
"""
def __init__(self, seed_conf, mapproxy_conf):
self.conf = seed_conf
self.mapproxy_conf = mapproxy_conf
self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())
import logging
log = logging.getLogger('mapproxy.seed.config')
def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf):
try:
conf = load_yaml_file(seed_conf_filename)
except YAMLError, ex:
raise SeedConfigurationError(ex)
if 'views' in conf:
# TODO: deprecate old config
seed_conf = LegacySeedingConfiguration(conf,
mapproxy_conf=mapproxy_conf)
else:
errors, informal_only = validate_seed_conf(conf)
for error in errors:
log.warn(error)
if not informal_only:
raise SeedConfigurationError('invalid configuration')
seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf)
return seed_conf
class LegacySeedingConfiguration(object):
"""
Read old seed.yaml configuration (with seed and views).
"""
def __init__(self, seed_conf, mapproxy_conf):
self.conf = seed_conf
self.mapproxy_conf = mapproxy_conf
def generate_confs(tileset, ignore_warnings=True, renderd=False):
"""
Takes a Tileset object and returns mapproxy and seed config files
"""
mapproxy_conf_json = """
{
"services":{
"wms":{
"on_source_errors":"raise",
"image_formats": ["image/png"]
}
},
"layers":[
{
"name":"",
"title":"",
"sources":[
"tileset_cache"
]
}
],
"caches":{
"tileset_cache":{
"grids":[
"webmercator"
],
"sources":[
"tileset_source"
],
"cache":{
"type":"mbtiles",
"filename": "/provide/valid/path/to/file.mbtiles"
}
}
},
"sources":{
"tileset_source":{
}
},
"grids":{
"webmercator":{
"base":"GLOBAL_MERCATOR"
}
},
"globals": {
"image": {
"paletted": false
}
}
}
"""
seed_conf_json = """
{
"coverages": {
"tileset_geom": {
"bbox": [-77.47, 38.72, -76.72, 39.08],
"datasource": "path/to/geom/file.xxx",
"srs": "EPSG:4326"
}
},
"seeds": {
"tileset_seed": {
"refresh_before": {
"minutes": 0
},
"caches": [
"tileset_cache"
],
"levels": {
"from": 0,
"to": 2
},
"coverages": ["tileset_geom"]
}
}
}
"""
seed_conf = yaml.safe_load(seed_conf_json)
mapproxy_conf = yaml.safe_load(mapproxy_conf_json)
print '---- mbtiles file to generate: {}'.format(get_tileset_filename(tileset.name))
mapproxy_conf['sources']['tileset_source']['type'] = u_to_str(tileset.server_service_type)
if u_to_str(tileset.server_service_type) == 'wms':
"""
"req":{
"url":"http://*****:*****@192.168.99.100/geoserver/wms?",
"layers":"geonode:ne_50m_admin_0_countries"
},
"""
mapproxy_conf['sources']['tileset_source']['req'] = {}
mapproxy_conf['sources']['tileset_source']['req']['url'] = u_to_str(tileset.server_url)
mapproxy_conf['sources']['tileset_source']['req']['layers'] = u_to_str(tileset.layer_name)
elif u_to_str(tileset.server_service_type) == 'tile':
"""
"url": "http://a.tile.openstreetmap.org/%(z)s/%(x)s/%(y)s.png",
"""
mapproxy_conf['sources']['tileset_source']['url'] = u_to_str(tileset.server_url)
mapproxy_conf['layers'][0]['name'] = u_to_str(tileset.layer_name)
mapproxy_conf['layers'][0]['title'] = u_to_str(tileset.layer_name)
mapproxy_conf['caches']['tileset_cache']['cache']['filename'] = get_tileset_filename(tileset.name, 'generating')
if tileset.layer_zoom_start > tileset.layer_zoom_stop:
raise ConfigurationError('invalid configuration - zoom start is greater than zoom stop')
seed_conf['seeds']['tileset_seed']['levels']['from'] = tileset.layer_zoom_start
seed_conf['seeds']['tileset_seed']['levels']['to'] = tileset.layer_zoom_stop
# any specified refresh before for mbtiles will result in regeneration of the tile set
seed_conf['seeds']['tileset_seed']['refresh_before']['minutes'] = 0
if tileset.geom:
geom_type = 'other'
if tileset.geom.startswith('{"'):
geom_type = 'geojson'
elif tileset.geom.lower().startswith('polygon') or tileset.geom.lower().startswith('multipolygon'):
geom_type = 'txt'
elif tileset.geom.startswith('['):
geom_type = 'bbox'
if geom_type in ['geojson', 'txt']:
geom_dir = '{}/geoms'.format(get_tileset_dir())
if not os.path.exists(geom_dir):
os.makedirs(geom_dir)
# TODO: remove geom files when done or pair them up with the actual tileset files?
geom_filename = '{}/geoms/{}.{}'.format(get_tileset_dir(), tileset.name, geom_type)
with open(geom_filename, 'w+') as geom_file:
geom_file.write(tileset.geom)
seed_conf['coverages']['tileset_geom']['datasource'] = geom_filename
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
elif geom_type is 'bbox':
seed_conf['coverages']['tileset_geom']['bbox'] = yaml.safe_load(tileset.geom)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
else:
# if not bbox or file, just set it as is to the datasource since mapproxy can handle other datastores
# and they should work as is
seed_conf['coverages']['tileset_geom']['datasource'] = yaml.safe_load(tileset.geom)
print '---- tileset geom_type: {}, geom: {}'.format(geom_type, tileset.geom)
else:
# if a geom is not specified, remove the coverages key from tileset_seed
seed_conf['seeds']['tileset_seed'].pop('coverages', None)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
print '--[ mapproxy_conf: '
print yaml.dump(mapproxy_conf)
print '--[ seed_conf: '
print yaml.dump(seed_conf)
if tileset.server_username and tileset.server_password:
"""
"http":{
"headers":{
"Authorization":"Basic YWRtaW46Z2Vvc2VydmVy"
}
}
"""
encoded = base64.b64encode('{}:{}'.format(tileset.server_username, tileset.server_password))
mapproxy_conf['sources']['tileset_source']['http'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers']['Authorization'] = 'Basic {}'.format(encoded)
errors, informal_only = validate_mapproxy_conf(mapproxy_conf)
for error in errors:
print error
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors)))
cf = ProxyConfiguration(mapproxy_conf, conf_base_dir=get_tileset_dir(), seed=seed, renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
for error in errors:
print error
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=cf)
return cf, seed_cf
def generate_confs(tileset, ignore_warnings=True, renderd=False):
"""
Takes a Tileset object and returns mapproxy and seed config files
"""
mapproxy_conf_json = """
{
"services":{
"wms":{
"on_source_errors":"raise",
"image_formats": ["image/png"]
}
},
"layers":[
{
"name":"",
"title":"",
"sources":[
"tileset_cache"
]
}
],
"caches":{
"tileset_cache":{
"grids":[
"webmercator"
],
"sources":[
"tileset_source"
],
"cache":{
"type":"mbtiles",
"filename": "/provide/valid/path/to/file.mbtiles"
}
}
},
"sources":{
"tileset_source":{
}
},
"grids":{
"webmercator":{
"base":"GLOBAL_MERCATOR"
}
},
"globals": {
"image": {
"paletted": false
}
}
}
"""
seed_conf_json = """
{
"coverages": {
"tileset_geom": {
"bbox": [-77.47, 38.72, -76.72, 39.08],
"datasource": "path/to/geom/file.xxx",
"srs": "EPSG:4326"
}
},
"seeds": {
"tileset_seed": {
"refresh_before": {
"minutes": 0
},
"caches": [
"tileset_cache"
],
"levels": {
"from": 0,
"to": 2
},
"coverages": ["tileset_geom"]
}
}
}
"""
seed_conf = yaml.safe_load(seed_conf_json)
mapproxy_conf = yaml.safe_load(mapproxy_conf_json)
print '---- mbtiles file to generate: {}'.format(
get_tileset_filename(tileset.name))
mapproxy_conf['sources']['tileset_source']['type'] = u_to_str(
tileset.server_service_type)
if u_to_str(tileset.server_service_type) == 'wms':
"""
"req":{
"url":"http://*****:*****@192.168.99.100/geoserver/wms?",
"layers":"geonode:ne_50m_admin_0_countries"
},
"""
mapproxy_conf['sources']['tileset_source']['req'] = {}
mapproxy_conf['sources']['tileset_source']['req']['url'] = u_to_str(
tileset.server_url)
mapproxy_conf['sources']['tileset_source']['req']['layers'] = u_to_str(
tileset.layer_name)
elif u_to_str(tileset.server_service_type) == 'tile':
"""
"url": "http://a.tile.openstreetmap.org/%(z)s/%(x)s/%(y)s.png",
"""
mapproxy_conf['sources']['tileset_source']['url'] = u_to_str(
tileset.server_url)
mapproxy_conf['layers'][0]['name'] = u_to_str(tileset.layer_name)
mapproxy_conf['layers'][0]['title'] = u_to_str(tileset.layer_name)
mapproxy_conf['caches']['tileset_cache']['cache'][
'filename'] = get_tileset_filename(tileset.name, 'generating')
if tileset.layer_zoom_start > tileset.layer_zoom_stop:
raise ConfigurationError(
'invalid configuration - zoom start is greater than zoom stop')
seed_conf['seeds']['tileset_seed']['levels'][
'from'] = tileset.layer_zoom_start
seed_conf['seeds']['tileset_seed']['levels'][
'to'] = tileset.layer_zoom_stop
# any specified refresh before for mbtiles will result in regeneration of the tile set
seed_conf['seeds']['tileset_seed']['refresh_before']['minutes'] = 0
if tileset.geom:
geom_type = 'other'
if tileset.geom.startswith('{"'):
geom_type = 'geojson'
elif tileset.geom.lower().startswith(
'polygon') or tileset.geom.lower().startswith('multipolygon'):
geom_type = 'txt'
elif tileset.geom.startswith('['):
geom_type = 'bbox'
if geom_type in ['geojson', 'txt']:
geom_dir = '{}/geoms'.format(get_tileset_dir())
if not os.path.exists(geom_dir):
os.makedirs(geom_dir)
# TODO: remove geom files when done or pair them up with the actual tileset files?
geom_filename = '{}/geoms/{}.{}'.format(get_tileset_dir(),
tileset.name, geom_type)
with open(geom_filename, 'w+') as geom_file:
geom_file.write(tileset.geom)
seed_conf['coverages']['tileset_geom'][
'datasource'] = geom_filename
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
elif geom_type is 'bbox':
seed_conf['coverages']['tileset_geom']['bbox'] = yaml.safe_load(
tileset.geom)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
else:
# if not bbox or file, just set it as is to the datasource since mapproxy can handle other datastores
# and they should work as is
seed_conf['coverages']['tileset_geom'][
'datasource'] = yaml.safe_load(tileset.geom)
print '---- tileset geom_type: {}, geom: {}'.format(
geom_type, tileset.geom)
else:
# if a geom is not specified, remove the coverages key from tileset_seed
seed_conf['seeds']['tileset_seed'].pop('coverages', None)
seed_conf['coverages']['tileset_geom'].pop('datasource', None)
seed_conf['coverages']['tileset_geom'].pop('bbox', None)
print '--[ mapproxy_conf: '
print yaml.dump(mapproxy_conf)
print '--[ seed_conf: '
print yaml.dump(seed_conf)
if tileset.server_username and tileset.server_password:
"""
"http":{
"headers":{
"Authorization":"Basic YWRtaW46Z2Vvc2VydmVy"
}
}
"""
encoded = base64.b64encode('{}:{}'.format(tileset.server_username,
tileset.server_password))
mapproxy_conf['sources']['tileset_source']['http'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers'] = {}
mapproxy_conf['sources']['tileset_source']['http']['headers'][
'Authorization'] = 'Basic {}'.format(encoded)
errors, informal_only = validate_mapproxy_conf(mapproxy_conf)
for error in errors:
print error
if not informal_only or (errors and not ignore_warnings):
raise ConfigurationError('invalid configuration - {}'.format(
', '.join(errors)))
cf = ProxyConfiguration(mapproxy_conf,
conf_base_dir=get_tileset_dir(),
seed=seed,
renderd=renderd)
errors, informal_only = validate_seed_conf(seed_conf)
for error in errors:
print error
if not informal_only:
raise SeedConfigurationError('invalid seed configuration - {}'.format(
', '.join(errors)))
seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=cf)
return cf, seed_cf
def convert(self, ):
"""
Convert external service to gpkg.
"""
from ..tasks.task_process import TaskProcess
from .geopackage import remove_empty_zoom_levels
if self.config:
conf_dict = yaml.load(self.config)
else:
conf_dict = create_conf_from_url(self.service_url)
if not conf_dict.get('grids'):
conf_dict['grids'] = {
'geodetic': {
'srs': 'EPSG:4326',
'tile_size': [256, 256],
'origin': 'nw'
},
'webmercator': {
'srs': 'EPSG:3857',
'tile_size': [256, 256],
'origin': 'nw'
}
}
# If user provides a cache setup then use that and substitute in the geopackage file for the placeholder.
conf_dict['caches'] = conf_dict.get('caches', {})
try:
conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile
except KeyError:
conf_dict['caches']['cache'] = get_cache_template(
["{0}_{1}".format(self.layer, self.service_type)],
[grids for grids in conf_dict.get('grids')], self.gpkgfile)
# Prevent the service from failing if source has missing tiles.
for source in conf_dict.get('sources'):
if 'wmts' in source:
conf_dict['sources'][source]['transparent'] = True
conf_dict['sources'][source]['on_error'] = {
"other": {
"response": "transparent",
"cache": False
}
}
# disable SSL cert checks
if getattr(settings, "DISABLE_SSL_VERIFICATION", False):
conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}}
# Add autoconfiguration to base_config
# default = load_default_config()
mapproxy_config = load_default_config()
load_config(mapproxy_config, config_dict=conf_dict)
# Create a configuration object
mapproxy_configuration = ProxyConfiguration(mapproxy_config,
seed=seed,
renderd=None)
# # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error.
if isclose(self.bbox[0], self.bbox[2], rel_tol=0.01) or isclose(
self.bbox[0], self.bbox[2], rel_tol=0.01):
logger.warn(
'Using bbox instead of selection, because the area is too small'
)
self.selection = None
seed_dict = get_seed_template(bbox=self.bbox,
level_from=self.level_from,
level_to=self.level_to,
coverage_file=self.selection)
# Create a seed configuration object
seed_configuration = SeedingConfiguration(
seed_dict, mapproxy_conf=mapproxy_configuration)
logger.info("Beginning seeding to {0}".format(self.gpkgfile))
logger.error(mapproxy_config)
try:
check_service(conf_dict)
progress_logger = CustomLogger(verbose=True,
task_uid=self.task_uid)
task_process = TaskProcess(task_uid=self.task_uid)
task_process.start_process(
billiard=True,
target=seeder.seed,
kwargs={
"tasks":
seed_configuration.seeds(['seed']),
"concurrency":
int(getattr(settings, 'MAPPROXY_CONCURRENCY', 1)),
"progress_logger":
progress_logger
})
remove_empty_zoom_levels(self.gpkgfile)
except Exception as e:
logger.error("Export failed for url {}.".format(self.service_url))
errors, informal_only = validate_options(mapproxy_config)
if not informal_only:
logger.error("MapProxy configuration failed.")
logger.error("Using Configuration:")
logger.error(mapproxy_config)
errors, informal_only = validate_seed_conf(seed_dict)
if not informal_only:
logger.error("Mapproxy Seed failed.")
logger.error("Using Seed Configuration:")
logger.error(seed_dict)
raise SeedConfigurationError(
'MapProxy seed configuration error - {}'.format(
', '.join(errors)))
raise e
finally:
connections.close_all()
return self.gpkgfile
