def generate_confs(tileset, layer, title, ignore_warnings=True, renderd=False): """ Default para productos SATMO Takes a Tileset object and returns mapproxy and seed config files """ # Start with a sane configuration using MapProxy's defaults mapproxy_config = load_default_config() tileset_conf_json = get_mapproxy_conf(tileset, layer, title) tileset_conf = yaml.safe_load(tileset_conf_json) # print tileset_conf_json # merge our config load_config(mapproxy_config, config_dict=tileset_conf) seed_conf_json = get_seed_conf(tileset) seed_conf = yaml.safe_load(seed_conf_json) errors, informal_only = validate_options(mapproxy_config) if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration - {}'.format( ', '.join(errors))) mapproxy_cf = ProxyConfiguration(mapproxy_config, seed=seed, renderd=renderd) errors, informal_only = validate_seed_conf(seed_conf) if not informal_only: raise SeedConfigurationError('invalid seed configuration - {}'.format( ', '.join(errors))) seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf) return mapproxy_cf, seed_cf
def generate_confs(tileset, ignore_warnings=True, renderd=False): """ Takes a Tileset object and returns mapproxy and seed config files """ # Start with a sane configuration using MapProxy's defaults mapproxy_config = load_default_config() tileset_conf_json = get_mapproxy_conf(tileset) tileset_conf = yaml.safe_load(tileset_conf_json) # merge our config load_config(mapproxy_config, config_dict=tileset_conf) seed_conf_json = get_seed_conf(tileset) seed_conf = yaml.safe_load(seed_conf_json) errors, informal_only = validate_options(mapproxy_config) if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors))) mapproxy_cf = ProxyConfiguration(mapproxy_config, seed=seed, renderd=renderd) errors, informal_only = validate_seed_conf(seed_conf) if not informal_only: raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors))) seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=mapproxy_cf) return mapproxy_cf, seed_cf
def test_union(self): conf = { 'coverages': { 'covname': { 'union': [ {'bbox': [0, 0, 10, 10], 'srs': 'EPSG:4326'}, {'bbox': [10, 0, 20, 10], 'srs': 'EPSG:4326', 'unknown': True}, ], }, }, } errors, informal_only = validate_seed_conf(conf) assert informal_only assert len(errors) == 1 assert errors[0] == "unknown 'unknown' in coverages.covname.union[1]"
def test_union(self): conf = { 'coverages': { 'covname': { 'union': [ {'bbox': [0, 0, 10, 10], 'srs': 'EPSG:4326'}, {'bbox': [10, 0, 20, 10], 'srs': 'EPSG:4326', 'unknown': True}, ], }, }, } errors, informal_only = validate_seed_conf(conf) assert informal_only assert len(errors) == 1 eq_(errors[0], "unknown 'unknown' in coverages.covname.union[1]")
def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf): try: conf = load_yaml_file(seed_conf_filename) except YAMLError as ex: raise SeedConfigurationError(ex) if 'views' in conf: # TODO: deprecate old config seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) else: errors, informal_only = validate_seed_conf(conf) for error in errors: log.warn(error) if not informal_only: raise SeedConfigurationError('invalid configuration') seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) return seed_conf
import logging log = logging.getLogger('mapproxy.seed.config') def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf): try: conf = load_yaml_file(seed_conf_filename) except YAMLError, ex: raise SeedConfigurationError(ex) if 'views' in conf: # TODO: deprecate old config seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) else: errors, informal_only = validate_seed_conf(conf) for error in errors: log.warn(error) if not informal_only: raise SeedConfigurationError('invalid configuration') seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) return seed_conf class LegacySeedingConfiguration(object): """ Read old seed.yaml configuration (with seed and views). """ def __init__(self, seed_conf, mapproxy_conf): self.conf = seed_conf self.mapproxy_conf = mapproxy_conf self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in self.mapproxy_conf.grids.iteritems())
import logging log = logging.getLogger('mapproxy.seed.config') def load_seed_tasks_conf(seed_conf_filename, mapproxy_conf): try: conf = load_yaml_file(seed_conf_filename) except YAMLError, ex: raise SeedConfigurationError(ex) if 'views' in conf: # TODO: deprecate old config seed_conf = LegacySeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) else: errors, informal_only = validate_seed_conf(conf) for error in errors: log.warn(error) if not informal_only: raise SeedConfigurationError('invalid configuration') seed_conf = SeedingConfiguration(conf, mapproxy_conf=mapproxy_conf) return seed_conf class LegacySeedingConfiguration(object): """ Read old seed.yaml configuration (with seed and views). """ def __init__(self, seed_conf, mapproxy_conf): self.conf = seed_conf self.mapproxy_conf = mapproxy_conf
def generate_confs(tileset, ignore_warnings=True, renderd=False): """ Takes a Tileset object and returns mapproxy and seed config files """ mapproxy_conf_json = """ { "services":{ "wms":{ "on_source_errors":"raise", "image_formats": ["image/png"] } }, "layers":[ { "name":"", "title":"", "sources":[ "tileset_cache" ] } ], "caches":{ "tileset_cache":{ "grids":[ "webmercator" ], "sources":[ "tileset_source" ], "cache":{ "type":"mbtiles", "filename": "/provide/valid/path/to/file.mbtiles" } } }, "sources":{ "tileset_source":{ } }, "grids":{ "webmercator":{ "base":"GLOBAL_MERCATOR" } }, "globals": { "image": { "paletted": false } } } """ seed_conf_json = """ { "coverages": { "tileset_geom": { "bbox": [-77.47, 38.72, -76.72, 39.08], "datasource": "path/to/geom/file.xxx", "srs": "EPSG:4326" } }, "seeds": { "tileset_seed": { "refresh_before": { "minutes": 0 }, "caches": [ "tileset_cache" ], "levels": { "from": 0, "to": 2 }, "coverages": ["tileset_geom"] } } } """ seed_conf = yaml.safe_load(seed_conf_json) mapproxy_conf = yaml.safe_load(mapproxy_conf_json) print '---- mbtiles file to generate: {}'.format(get_tileset_filename(tileset.name)) mapproxy_conf['sources']['tileset_source']['type'] = u_to_str(tileset.server_service_type) if u_to_str(tileset.server_service_type) == 'wms': """ "req":{ "url":"http://*****:*****@192.168.99.100/geoserver/wms?", "layers":"geonode:ne_50m_admin_0_countries" }, """ mapproxy_conf['sources']['tileset_source']['req'] = {} mapproxy_conf['sources']['tileset_source']['req']['url'] = u_to_str(tileset.server_url) mapproxy_conf['sources']['tileset_source']['req']['layers'] = u_to_str(tileset.layer_name) elif u_to_str(tileset.server_service_type) == 'tile': """ "url": "http://a.tile.openstreetmap.org/%(z)s/%(x)s/%(y)s.png", """ mapproxy_conf['sources']['tileset_source']['url'] = u_to_str(tileset.server_url) mapproxy_conf['layers'][0]['name'] = u_to_str(tileset.layer_name) mapproxy_conf['layers'][0]['title'] = u_to_str(tileset.layer_name) mapproxy_conf['caches']['tileset_cache']['cache']['filename'] = get_tileset_filename(tileset.name, 'generating') if tileset.layer_zoom_start > tileset.layer_zoom_stop: raise ConfigurationError('invalid configuration - zoom start is greater than zoom stop') seed_conf['seeds']['tileset_seed']['levels']['from'] = tileset.layer_zoom_start seed_conf['seeds']['tileset_seed']['levels']['to'] = tileset.layer_zoom_stop # any specified refresh before for mbtiles will result in regeneration of the tile set seed_conf['seeds']['tileset_seed']['refresh_before']['minutes'] = 0 if tileset.geom: geom_type = 'other' if tileset.geom.startswith('{"'): geom_type = 'geojson' elif tileset.geom.lower().startswith('polygon') or tileset.geom.lower().startswith('multipolygon'): geom_type = 'txt' elif tileset.geom.startswith('['): geom_type = 'bbox' if geom_type in ['geojson', 'txt']: geom_dir = '{}/geoms'.format(get_tileset_dir()) if not os.path.exists(geom_dir): os.makedirs(geom_dir) # TODO: remove geom files when done or pair them up with the actual tileset files? geom_filename = '{}/geoms/{}.{}'.format(get_tileset_dir(), tileset.name, geom_type) with open(geom_filename, 'w+') as geom_file: geom_file.write(tileset.geom) seed_conf['coverages']['tileset_geom']['datasource'] = geom_filename seed_conf['coverages']['tileset_geom'].pop('bbox', None) elif geom_type is 'bbox': seed_conf['coverages']['tileset_geom']['bbox'] = yaml.safe_load(tileset.geom) seed_conf['coverages']['tileset_geom'].pop('datasource', None) else: # if not bbox or file, just set it as is to the datasource since mapproxy can handle other datastores # and they should work as is seed_conf['coverages']['tileset_geom']['datasource'] = yaml.safe_load(tileset.geom) print '---- tileset geom_type: {}, geom: {}'.format(geom_type, tileset.geom) else: # if a geom is not specified, remove the coverages key from tileset_seed seed_conf['seeds']['tileset_seed'].pop('coverages', None) seed_conf['coverages']['tileset_geom'].pop('datasource', None) seed_conf['coverages']['tileset_geom'].pop('bbox', None) print '--[ mapproxy_conf: ' print yaml.dump(mapproxy_conf) print '--[ seed_conf: ' print yaml.dump(seed_conf) if tileset.server_username and tileset.server_password: """ "http":{ "headers":{ "Authorization":"Basic YWRtaW46Z2Vvc2VydmVy" } } """ encoded = base64.b64encode('{}:{}'.format(tileset.server_username, tileset.server_password)) mapproxy_conf['sources']['tileset_source']['http'] = {} mapproxy_conf['sources']['tileset_source']['http']['headers'] = {} mapproxy_conf['sources']['tileset_source']['http']['headers']['Authorization'] = 'Basic {}'.format(encoded) errors, informal_only = validate_mapproxy_conf(mapproxy_conf) for error in errors: print error if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration - {}'.format(', '.join(errors))) cf = ProxyConfiguration(mapproxy_conf, conf_base_dir=get_tileset_dir(), seed=seed, renderd=renderd) errors, informal_only = validate_seed_conf(seed_conf) for error in errors: print error if not informal_only: raise SeedConfigurationError('invalid seed configuration - {}'.format(', '.join(errors))) seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=cf) return cf, seed_cf
def generate_confs(tileset, ignore_warnings=True, renderd=False): """ Takes a Tileset object and returns mapproxy and seed config files """ mapproxy_conf_json = """ { "services":{ "wms":{ "on_source_errors":"raise", "image_formats": ["image/png"] } }, "layers":[ { "name":"", "title":"", "sources":[ "tileset_cache" ] } ], "caches":{ "tileset_cache":{ "grids":[ "webmercator" ], "sources":[ "tileset_source" ], "cache":{ "type":"mbtiles", "filename": "/provide/valid/path/to/file.mbtiles" } } }, "sources":{ "tileset_source":{ } }, "grids":{ "webmercator":{ "base":"GLOBAL_MERCATOR" } }, "globals": { "image": { "paletted": false } } } """ seed_conf_json = """ { "coverages": { "tileset_geom": { "bbox": [-77.47, 38.72, -76.72, 39.08], "datasource": "path/to/geom/file.xxx", "srs": "EPSG:4326" } }, "seeds": { "tileset_seed": { "refresh_before": { "minutes": 0 }, "caches": [ "tileset_cache" ], "levels": { "from": 0, "to": 2 }, "coverages": ["tileset_geom"] } } } """ seed_conf = yaml.safe_load(seed_conf_json) mapproxy_conf = yaml.safe_load(mapproxy_conf_json) print '---- mbtiles file to generate: {}'.format( get_tileset_filename(tileset.name)) mapproxy_conf['sources']['tileset_source']['type'] = u_to_str( tileset.server_service_type) if u_to_str(tileset.server_service_type) == 'wms': """ "req":{ "url":"http://*****:*****@192.168.99.100/geoserver/wms?", "layers":"geonode:ne_50m_admin_0_countries" }, """ mapproxy_conf['sources']['tileset_source']['req'] = {} mapproxy_conf['sources']['tileset_source']['req']['url'] = u_to_str( tileset.server_url) mapproxy_conf['sources']['tileset_source']['req']['layers'] = u_to_str( tileset.layer_name) elif u_to_str(tileset.server_service_type) == 'tile': """ "url": "http://a.tile.openstreetmap.org/%(z)s/%(x)s/%(y)s.png", """ mapproxy_conf['sources']['tileset_source']['url'] = u_to_str( tileset.server_url) mapproxy_conf['layers'][0]['name'] = u_to_str(tileset.layer_name) mapproxy_conf['layers'][0]['title'] = u_to_str(tileset.layer_name) mapproxy_conf['caches']['tileset_cache']['cache'][ 'filename'] = get_tileset_filename(tileset.name, 'generating') if tileset.layer_zoom_start > tileset.layer_zoom_stop: raise ConfigurationError( 'invalid configuration - zoom start is greater than zoom stop') seed_conf['seeds']['tileset_seed']['levels'][ 'from'] = tileset.layer_zoom_start seed_conf['seeds']['tileset_seed']['levels'][ 'to'] = tileset.layer_zoom_stop # any specified refresh before for mbtiles will result in regeneration of the tile set seed_conf['seeds']['tileset_seed']['refresh_before']['minutes'] = 0 if tileset.geom: geom_type = 'other' if tileset.geom.startswith('{"'): geom_type = 'geojson' elif tileset.geom.lower().startswith( 'polygon') or tileset.geom.lower().startswith('multipolygon'): geom_type = 'txt' elif tileset.geom.startswith('['): geom_type = 'bbox' if geom_type in ['geojson', 'txt']: geom_dir = '{}/geoms'.format(get_tileset_dir()) if not os.path.exists(geom_dir): os.makedirs(geom_dir) # TODO: remove geom files when done or pair them up with the actual tileset files? geom_filename = '{}/geoms/{}.{}'.format(get_tileset_dir(), tileset.name, geom_type) with open(geom_filename, 'w+') as geom_file: geom_file.write(tileset.geom) seed_conf['coverages']['tileset_geom'][ 'datasource'] = geom_filename seed_conf['coverages']['tileset_geom'].pop('bbox', None) elif geom_type is 'bbox': seed_conf['coverages']['tileset_geom']['bbox'] = yaml.safe_load( tileset.geom) seed_conf['coverages']['tileset_geom'].pop('datasource', None) else: # if not bbox or file, just set it as is to the datasource since mapproxy can handle other datastores # and they should work as is seed_conf['coverages']['tileset_geom'][ 'datasource'] = yaml.safe_load(tileset.geom) print '---- tileset geom_type: {}, geom: {}'.format( geom_type, tileset.geom) else: # if a geom is not specified, remove the coverages key from tileset_seed seed_conf['seeds']['tileset_seed'].pop('coverages', None) seed_conf['coverages']['tileset_geom'].pop('datasource', None) seed_conf['coverages']['tileset_geom'].pop('bbox', None) print '--[ mapproxy_conf: ' print yaml.dump(mapproxy_conf) print '--[ seed_conf: ' print yaml.dump(seed_conf) if tileset.server_username and tileset.server_password: """ "http":{ "headers":{ "Authorization":"Basic YWRtaW46Z2Vvc2VydmVy" } } """ encoded = base64.b64encode('{}:{}'.format(tileset.server_username, tileset.server_password)) mapproxy_conf['sources']['tileset_source']['http'] = {} mapproxy_conf['sources']['tileset_source']['http']['headers'] = {} mapproxy_conf['sources']['tileset_source']['http']['headers'][ 'Authorization'] = 'Basic {}'.format(encoded) errors, informal_only = validate_mapproxy_conf(mapproxy_conf) for error in errors: print error if not informal_only or (errors and not ignore_warnings): raise ConfigurationError('invalid configuration - {}'.format( ', '.join(errors))) cf = ProxyConfiguration(mapproxy_conf, conf_base_dir=get_tileset_dir(), seed=seed, renderd=renderd) errors, informal_only = validate_seed_conf(seed_conf) for error in errors: print error if not informal_only: raise SeedConfigurationError('invalid seed configuration - {}'.format( ', '.join(errors))) seed_cf = SeedingConfiguration(seed_conf, mapproxy_conf=cf) return cf, seed_cf
def convert(self, ): """ Convert external service to gpkg. """ from ..tasks.task_process import TaskProcess from .geopackage import remove_empty_zoom_levels if self.config: conf_dict = yaml.load(self.config) else: conf_dict = create_conf_from_url(self.service_url) if not conf_dict.get('grids'): conf_dict['grids'] = { 'geodetic': { 'srs': 'EPSG:4326', 'tile_size': [256, 256], 'origin': 'nw' }, 'webmercator': { 'srs': 'EPSG:3857', 'tile_size': [256, 256], 'origin': 'nw' } } # If user provides a cache setup then use that and substitute in the geopackage file for the placeholder. conf_dict['caches'] = conf_dict.get('caches', {}) try: conf_dict['caches']['cache']['cache']['filename'] = self.gpkgfile except KeyError: conf_dict['caches']['cache'] = get_cache_template( ["{0}_{1}".format(self.layer, self.service_type)], [grids for grids in conf_dict.get('grids')], self.gpkgfile) # Prevent the service from failing if source has missing tiles. for source in conf_dict.get('sources'): if 'wmts' in source: conf_dict['sources'][source]['transparent'] = True conf_dict['sources'][source]['on_error'] = { "other": { "response": "transparent", "cache": False } } # disable SSL cert checks if getattr(settings, "DISABLE_SSL_VERIFICATION", False): conf_dict['globals'] = {'http': {'ssl_no_cert_checks': True}} # Add autoconfiguration to base_config # default = load_default_config() mapproxy_config = load_default_config() load_config(mapproxy_config, config_dict=conf_dict) # Create a configuration object mapproxy_configuration = ProxyConfiguration(mapproxy_config, seed=seed, renderd=None) # # As of Mapproxy 1.9.x, datasource files covering a small area cause a bbox error. if isclose(self.bbox[0], self.bbox[2], rel_tol=0.01) or isclose( self.bbox[0], self.bbox[2], rel_tol=0.01): logger.warn( 'Using bbox instead of selection, because the area is too small' ) self.selection = None seed_dict = get_seed_template(bbox=self.bbox, level_from=self.level_from, level_to=self.level_to, coverage_file=self.selection) # Create a seed configuration object seed_configuration = SeedingConfiguration( seed_dict, mapproxy_conf=mapproxy_configuration) logger.info("Beginning seeding to {0}".format(self.gpkgfile)) logger.error(mapproxy_config) try: check_service(conf_dict) progress_logger = CustomLogger(verbose=True, task_uid=self.task_uid) task_process = TaskProcess(task_uid=self.task_uid) task_process.start_process( billiard=True, target=seeder.seed, kwargs={ "tasks": seed_configuration.seeds(['seed']), "concurrency": int(getattr(settings, 'MAPPROXY_CONCURRENCY', 1)), "progress_logger": progress_logger }) remove_empty_zoom_levels(self.gpkgfile) except Exception as e: logger.error("Export failed for url {}.".format(self.service_url)) errors, informal_only = validate_options(mapproxy_config) if not informal_only: logger.error("MapProxy configuration failed.") logger.error("Using Configuration:") logger.error(mapproxy_config) errors, informal_only = validate_seed_conf(seed_dict) if not informal_only: logger.error("Mapproxy Seed failed.") logger.error("Using Seed Configuration:") logger.error(seed_dict) raise SeedConfigurationError( 'MapProxy seed configuration error - {}'.format( ', '.join(errors))) raise e finally: connections.close_all() return self.gpkgfile