def update_config(conf, overwrites):
wildcard_keys = []
for k, v in iteritems(overwrites):
if k == '__all__':
continue
if k.startswith('___') or k.endswith('___'):
wildcard_keys.append(k)
continue
if k.endswith('__extend__'):
k = k[:-len('__extend__')]
if k not in conf:
conf[k] = v
elif isinstance(v, list):
conf[k].extend(v)
else:
raise ValueError('cannot extend non-list:', v)
elif k not in conf:
conf[k] = copy(v)
else:
if isinstance(conf[k], dict) and isinstance(v, dict):
conf[k] = update_config(conf[k], v)
else:
conf[k] = copy(v)
if '__all__' in overwrites:
v = overwrites['__all__']
for conf_k, conf_v in iteritems(conf):
if isinstance(conf_v, dict):
conf[conf_k] = update_config(conf_v, v)
else:
conf[conf_k] = v
if wildcard_keys:
for key in wildcard_keys:
v = overwrites[key]
if key.startswith('___'):
key = key[3:]
key_check = lambda x: x.endswith(key)
else:
key = key[:-3]
key_check = lambda x: x.startswith(key)
for conf_k, conf_v in iteritems(conf):
if not key_check(conf_k):
continue
if isinstance(conf_v, dict):
conf[conf_k] = update_config(conf_v, v)
else:
conf[conf_k] = v
return conf
def load_base_config(config_file=None, clear_existing=False):
"""
Load system wide base configuration.
:param config_file: the file name of the mapproxy.yaml configuration.
if ``None``, load the internal proxylib/default.yaml conf
:param clear_existing: if ``True`` remove the existing configuration settings,
else overwrite the settings.
"""
if config_file is None:
from mapproxy.config import defaults
config_dict = {}
for k, v in iteritems(defaults.__dict__):
if k.startswith('_'): continue
config_dict[k] = v
conf_base_dir = os.getcwd()
load_config(base_config(), config_dict=config_dict, clear_existing=clear_existing)
else:
conf_base_dir = os.path.abspath(os.path.dirname(config_file))
load_config(base_config(), config_file=config_file, clear_existing=clear_existing)
bc = base_config()
finish_base_config(bc)
bc.conf_base_dir = conf_base_dir
def layer_srs_bbox(self, layer, epsg_axis_order=False):
for srs, extent in iteritems(self.srs_extents):
# is_default is True when no explicit bbox is defined for this srs
# use layer extent
if extent.is_default:
bbox = layer.extent.bbox_for(SRS(srs))
elif layer.extent.is_default:
bbox = extent.bbox_for(SRS(srs))
else:
# Use intersection of srs_extent and layer.extent.
bbox = extent.intersection(layer.extent).bbox_for(SRS(srs))
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, srs)
if srs in self.srs:
yield srs, bbox
# add native srs
layer_srs_code = layer.extent.srs.srs_code
if layer_srs_code not in self.srs_extents:
bbox = layer.extent.bbox
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, layer_srs_code)
if layer_srs_code in self.srs:
yield layer_srs_code, bbox
def restart_with_reloader():
"""Spawn a new Python interpreter with the same arguments as this one,
but running the reloader thread.
"""
while 1:
_log('info', ' * Restarting with reloader')
args = [sys.executable] + sys.argv
if os.name == 'nt':
# pip installs commands as .exe, but sys.argv[0]
# can miss the prefix.
# Add .exe to avoid file-not-found in subprocess call.
# Also, recent pip versions create .exe commands that are not
# executable by Python, but there is a -script.py which
# we need to call in this case. Check for this first.
if os.path.exists(args[1] + '-script.py'):
args[1] = args[1] + '-script.py'
elif not args[1].endswith('.exe'):
args[1] = args[1] + '.exe'
new_environ = os.environ.copy()
new_environ['WERKZEUG_RUN_MAIN'] = 'true'
# a weird bug on windows. sometimes unicode strings end up in the
# environment and subprocess.call does not like this, encode them
# to latin1 and continue.
if os.name == 'nt' and PY2:
for key, value in iteritems(new_environ):
if isinstance(value, text_type):
new_environ[key] = value.encode('iso-8859-1')
exit_code = subprocess.call(args, env=new_environ)
if exit_code != 3:
return exit_code
def __repr__(self):
items = [
(k, v) for k, v in iteritems(self)]
items.sort()
return '<%s %s>' % (
self.__class__.__name__,
' '.join(['%s=%r' % (k, v) for k, v in items]))
def fixed_headers(self):
headers = []
for key, value in iteritems(self.headers):
if PY2 and isinstance(value, unicode):
value = value.encode('utf-8')
headers.append((key, value))
return headers
def restart_with_reloader():
"""Spawn a new Python interpreter with the same arguments as this one,
but running the reloader thread.
"""
while 1:
_log('info', ' * Restarting with reloader')
args = [sys.executable] + sys.argv
# pip installs commands as .exe, but sys.argv[0]
# can miss the prefix. add .exe to avoid file-not-found
# in subprocess call
if os.name == 'nt' and '.' not in args[1]:
args[1] = args[1] + '.exe'
new_environ = os.environ.copy()
new_environ['WERKZEUG_RUN_MAIN'] = 'true'
# a weird bug on windows. sometimes unicode strings end up in the
# environment and subprocess.call does not like this, encode them
# to latin1 and continue.
if os.name == 'nt' and PY2:
for key, value in iteritems(new_environ):
if isinstance(value, text_type):
new_environ[key] = value.encode('iso-8859-1')
exit_code = subprocess.call(args, env=new_environ)
if exit_code != 3:
return exit_code
def __init__(self, seed_conf, mapproxy_conf):
self.conf = seed_conf
self.mapproxy_conf = mapproxy_conf
self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in iteritems(self.mapproxy_conf.grids))
self.seed_tasks = []
self.cleanup_tasks = []
self._init_tasks()
def seed_tasks(self):
for grid_name in self.grids:
for cache_name, cache in iteritems(self.caches):
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage is False:
coverage = False
elif self.coverage:
coverage = self.coverage.transform_to(grid.srs)
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
try:
if coverage is not False:
coverage.extent.llbbox
except TransformationError:
raise SeedConfigurationError('%s: coverage transformation error' % self.name)
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(range(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
if self.refresh_timestamp:
# remove everything
self.refresh_timestamp = 0
md = dict(name=self.name, cache_name=cache_name, grid_name=grid_name)
yield SeedTask(md, tile_manager, levels, self.refresh_timestamp, coverage)
def remove_tile(self, tile):
if tile.coord is None:
return True
for bundle_file, bundle_tiles in iteritems(self._get_bundle_tiles([tile])):
with BundleDataV2(bundle_file, mode="write") as bundledata:
for tile in bundle_tiles:
return bundledata.remove_tile(tile)
def adapt_params_to_version(self):
params = self.params.copy()
for key, value in iteritems(self.fixed_params):
params[key] = value
if "styles" not in params:
params["styles"] = ""
return params
def is_cached(self, tile):
if tile.source or tile.coord is None:
return True
for bundle_file, bundle_tiles in iteritems(self._get_bundle_tiles([tile])):
with BundleDataV2(bundle_file) as bundledata:
for tile in bundle_tiles:
return bundledata.tile_size(tile.coord) != 0
def adapt_params_to_version(self):
params = self.params.copy()
for key, value in iteritems(self.fixed_params):
params[key] = value
if 'styles' not in params:
params['styles'] = ''
return params
def __getitem__(self, key):
if self.defaults is None:
defaults = {}
for epsg, bbox in iteritems(self._defaults):
defaults[SRS(epsg)] = bbox
self.defaults = defaults
return self.defaults[key]
def print_layers(self, capabilities, indent=None, root=False):
if root:
print("# Note: This is not a valid MapProxy configuration!")
print('Capabilities Document Version %s' % (self.version,))
print('Root-Layer:')
layer_list = capabilities.layers()['layers']
else:
layer_list = capabilities['layers']
indent = indent or self.indent
for layer in layer_list:
marked_first = False
# print ordered items first
for item in self.print_order:
if layer.get(item, False):
if not marked_first:
marked_first = True
self._format_output(item, layer[item], indent, mark_first=marked_first)
else:
self._format_output(item, layer[item], indent)
# print remaining items except sublayers
for key, value in iteritems(layer):
if key in self.print_order or key == 'layers':
continue
self._format_output(key, value, indent)
# print the sublayers now
if layer.get('layers', False):
self.print_line(indent, 'layers')
self.print_layers(layer, indent=indent+self.indent)
def check_request_dimensions(self, tile_layer, request):
# check that unknown dimension for this layer are set to default
if request.dimensions:
for dimension, value in iteritems(request.dimensions):
dimension = dimension.lower()
if dimension not in tile_layer.dimensions and value != 'default':
raise RequestError('unknown dimension: ' + str(dimension),
code='InvalidParameterValue', request=request)
def needs_reload(self, app_name, timestamps):
if not timestamps:
return True
for conf_file, timestamp in iteritems(timestamps):
m_time = os.path.getmtime(conf_file)
if m_time > timestamp:
return True
return False
def _format_output(self, key, value, indent, mark_first=False):
if key == 'bbox':
self.print_line(indent, key)
for srs_code, bbox in iteritems(value):
self.print_line(indent+self.indent, srs_code, value=bbox, mark_first=mark_first)
else:
if isinstance(value, set):
value = list(value)
self.print_line(indent, key, value=value, mark_first=mark_first)
def __init__(self, attributes):
self.attributes = attributes
for key, value in iteritems(attributes):
if value == '{{timestamp}}':
self.timestamp_key = key
break
else:
attributes['timestamp'] = '{{timestamp}}'
self.timestamp_key = 'timestamp'
def create_is_x_functions():
for type_, magic in iteritems(magic_bytes):
def create_is_type(type_, magic):
def is_type(fileobj):
if not hasattr(fileobj, 'read'):
fileobj = BytesIO(fileobj)
return has_magic_bytes(fileobj, magic)
return is_type
globals()['is_' + type_] = create_is_type(type_, magic)
def caches(cap, sources, srs_grids):
caches = {}
for name, source in iteritems(sources):
conf = for_source(name, source, srs_grids)
if not conf:
continue
caches[name[:-len('_wms')] + '_cache'] = conf
return caches
def load_default_config():
from mapproxy.config import defaults
config_dict = {}
for k, v in iteritems(defaults.__dict__):
if k.startswith('_'): continue
config_dict[k] = v
default_conf = Options()
load_config(default_conf, config_dict=config_dict)
return default_conf
def fixed_headers(self):
headers = []
for key, value in iteritems(self.headers):
if type(value) != text_type:
# for str subclasses like ImageFormat
value = str(value)
if PY2 and isinstance(value, unicode):
value = value.encode('utf-8')
headers.append((key, value))
return headers
def load_default_config():
from mapproxy.config import defaults
config_dict = {}
for k, v in iteritems(defaults.__dict__):
if k.startswith('_'): continue
config_dict[k] = v
default_conf = Options()
load_config(default_conf, config_dict=config_dict)
return default_conf
def _to_options_map(mapping):
if isinstance(mapping, dict):
opt = Options()
for key, value in iteritems(mapping):
opt[key] = _to_options_map(value)
return opt
elif isinstance(mapping, list):
return [_to_options_map(m) for m in mapping]
else:
return mapping
def _to_options_map(mapping):
if isinstance(mapping, dict):
opt = Options()
for key, value in iteritems(mapping):
opt[key] = _to_options_map(value)
return opt
elif isinstance(mapping, list):
return [_to_options_map(m) for m in mapping]
else:
return mapping
def dimensions_for_params(self, params):
"""
Return subset of the dimensions.
>>> mq = MapQuery(None, None, None, dimensions={'Foo': 1, 'bar': 2})
>>> mq.dimensions_for_params(set(['FOO', 'baz']))
{'Foo': 1}
"""
params = [p.lower() for p in params]
return dict((k, v) for k, v in iteritems(self.dimensions) if k.lower() in params)
def print_items(data, title='Commands'):
name_len = max(len(name) for name in data)
if title:
print('%s:' % (title, ), file=sys.stdout)
for name, item in iteritems(data):
help = item.get('help', '')
name = ('%%-%ds' % name_len) % name
if help:
help = ' ' + help
print(' %s%s' % (name, help), file=sys.stdout)
def load_tiles(self, tiles, with_metadata=False):
tiles_to_load = [t for t in tiles if not (t.source or t.coord is None)]
tiles_loaded = 0
if not tiles_to_load:
return True
for bundle_file, bundle_tiles in iteritems(self._get_bundle_tiles(tiles_to_load)):
with BundleDataV2(bundle_file) as bundledata:
for tile in bundle_tiles:
tiles_loaded += 1 if bundledata.load_tile(tile) else 0
log.debug("%i loaded tiles" % tiles_loaded)
return tiles_loaded == len(tiles_to_load)
def print_items(data, title='Commands'):
name_len = max(len(name) for name in data)
if title:
print('%s:' % (title, ), file=sys.stdout)
for name, item in iteritems(data):
help = item.get('help', '')
name = ('%%-%ds' % name_len) % name
if help:
help = ' ' + help
print(' %s%s' % (name, help), file=sys.stdout)
def print_items(data, title="Commands"):
name_len = max(len(name) for name in data)
if title:
print("%s:" % (title,), file=sys.stdout)
for name, item in iteritems(data):
help = item.get("help", "")
name = ("%%-%ds" % name_len) % name
if help:
help = " " + help
print(" %s%s" % (name, help), file=sys.stdout)
def _init_tasks(self):
for cache_name, options in iteritems(self.conf['seeds']):
remove_before = None
if 'remove_before' in options:
remove_before = before_timestamp_from_options(options['remove_before'])
try:
caches = self.mapproxy_conf.caches[cache_name].caches()
except KeyError:
print('error: cache %s not found. available caches: %s' % (
cache_name, ','.join(self.mapproxy_conf.caches.keys())), file=sys.stderr)
return
caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
for view in options['views']:
view_conf = self.conf['views'][view]
coverage = load_coverage(view_conf)
cache_srs = view_conf.get('srs', None)
if cache_srs is not None:
cache_srs = [SRS(s) for s in cache_srs]
level = view_conf.get('level', None)
assert len(level) == 2
for grid, tile_mgr in iteritems(caches):
if cache_srs and grid.srs not in cache_srs: continue
md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
levels = list(range(level[0], level[1]+1))
if coverage:
if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
continue
seed_coverage = coverage.transform_to(grid.srs)
else:
seed_coverage = BBOXCoverage(grid.bbox, grid.srs)
self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))
if remove_before:
levels = list(range(grid.levels))
complete_extent = bool(coverage)
self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
seed_coverage, complete_extent=complete_extent))
def _init_tasks(self):
for cache_name, options in iteritems(self.conf['seeds']):
remove_before = None
if 'remove_before' in options:
remove_before = before_timestamp_from_options(options['remove_before'])
try:
caches = self.mapproxy_conf.caches[cache_name].caches()
except KeyError:
print('error: cache %s not found. available caches: %s' % (
cache_name, ','.join(self.mapproxy_conf.caches.keys())), file=sys.stderr)
return
caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
for view in options['views']:
view_conf = self.conf['views'][view]
coverage = load_coverage(view_conf)
cache_srs = view_conf.get('srs', None)
if cache_srs is not None:
cache_srs = [SRS(s) for s in cache_srs]
level = view_conf.get('level', None)
assert len(level) == 2
for grid, tile_mgr in iteritems(caches):
if cache_srs and grid.srs not in cache_srs: continue
md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
levels = list(range(level[0], level[1]+1))
if coverage:
if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
continue
seed_coverage = coverage.transform_to(grid.srs)
else:
seed_coverage = BBOXCoverage(grid.bbox, grid.srs)
self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))
if remove_before:
levels = list(range(grid.levels))
complete_extent = bool(coverage)
self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
seed_coverage, complete_extent=complete_extent))
def _format_output(self, key, value, indent, mark_first=False):
if key == 'bbox':
self.print_line(indent, key)
for srs_code, bbox in iteritems(value):
self.print_line(indent + self.indent,
srs_code,
value=bbox,
mark_first=mark_first)
else:
if isinstance(value, set):
value = list(value)
self.print_line(indent, key, value=value, mark_first=mark_first)
def create_is_x_functions():
for type_, magic in iteritems(magic_bytes):
def create_is_type(type_, magic):
def is_type(fileobj):
if not hasattr(fileobj, 'read'):
fileobj = BytesIO(fileobj)
return has_magic_bytes(fileobj, magic)
return is_type
globals()['is_' + type_] = create_is_type(type_, magic)
def store_tiles(self, tiles):
tiles_to_store=[t for t in tiles if not t.stored]
tiles_stored=0
for bundle_file, bundle_tiles in iteritems(self._get_bundle_tiles(tiles_to_store)):
records=[]
for tile in bundle_tiles:
with tile_buffer(tile) as buf: records.append((buf.read(), tile))
with BundleDataV2(bundle_file, mode = "write") as bundledata:
for record in records:
tiles_stored+=1 if bundledata.store_tile(*record) else 0
return tiles_stored==len(tiles_to_store)
def _parse_signature(self, args, kw):
values = {}
sig_args, var_args, var_kw, defaults = self._func_signature
extra_kw = {}
for name, value in iteritems(kw):
if not var_kw and name not in sig_args:
raise TypeError(
'Unexpected argument %s' % name)
if name in sig_args:
values[sig_args] = value
else:
extra_kw[name] = value
args = list(args)
sig_args = list(sig_args)
while args:
while sig_args and sig_args[0] in values:
sig_args.pop(0)
if sig_args:
name = sig_args.pop(0)
values[name] = args.pop(0)
elif var_args:
values[var_args] = tuple(args)
break
else:
raise TypeError(
'Extra position arguments: %s'
% ', '.join(repr(v) for v in args))
for name, value_expr in iteritems(defaults):
if name not in values:
values[name] = self._template._eval(
value_expr, self._ns, self._pos)
for name in sig_args:
if name not in values:
raise TypeError(
'Missing argument: %s' % name)
if var_kw:
values[var_kw] = extra_kw
return values
def update(self, *args, **kwargs):
sources = []
if len(args) == 1:
if hasattr(args[0], 'iteritems') or hasattr(args[0], 'items'):
sources.append(iteritems(args[0]))
else:
sources.append(iter(args[0]))
elif args:
raise TypeError('expected at most one positional argument')
if kwargs:
sources.append(kwargs.iteritems())
for iterable in sources:
for key, val in iterable:
self[key] = val
def merge_dict(base, other):
"""
Return `base` dict with values from `conf` merged in.
"""
for k, v in iteritems(other):
if k not in base:
base[k] = v
else:
if isinstance(base[k], dict):
merge_dict(base[k], v)
elif isinstance(base[k], list):
base[k].extend(v)
else:
base[k] = v
return base
def checked_dimensions(self, tile_request):
dimensions = {}
for dimension, values in iteritems(self.dimensions):
value = tile_request.dimensions.get(dimension)
if value in values:
dimensions[dimension] = value
elif not value or value == 'default':
dimensions[dimension] = values.default
else:
raise RequestError('invalid dimension value (%s=%s).' %
(dimension, value),
request=tile_request,
code='InvalidParameterValue')
return dimensions
def _interpret_inherit(self, body, defs, inherit_template, ns):
__traceback_hide__ = True
if not self.get_template:
raise TemplateError(
'You cannot use inheritance without passing in get_template',
position=None,
name=self.name)
templ = self.get_template(inherit_template, self)
self_ = TemplateObject(self.name)
for name, value in iteritems(defs):
setattr(self_, name, value)
self_.body = body
ns = ns.copy()
ns['self'] = self_
return templ.substitute(ns)
def _validate_cache(self, name, cache):
if isinstance(cache.get('sources', []), dict):
self._validate_bands(name, set(cache['sources'].keys()))
for band, confs in iteritems(cache['sources']):
for conf in confs:
band_source = conf['source']
self._validate_cache_source(name, band_source)
else:
for cache_source in cache.get('sources', []):
self._validate_cache_source(name, cache_source)
for grid in cache.get('grids', []):
if grid not in self.known_grids:
self.errors.append(
"Grid '%s' for cache '%s' not found in config" %
(grid, name))
def _gen_dict(self, mapping=()):
"""A `NoCaseMultiDict` can be constructed from an iterable of
``(key, value)`` tuples or a dict.
"""
tmp = {}
if isinstance(mapping, NoCaseMultiDict):
for key, value in mapping.iteritems(): #pylint: disable-msg=E1103
tmp.setdefault(key.lower(), (key, []))[1].extend(value)
else:
if isinstance(mapping, dict):
itr = iteritems(mapping)
else:
itr = iter(mapping)
for key, value in itr:
tmp.setdefault(key.lower(), (key, []))[1].append(value)
return tmp
def cleanup_tasks(self):
for grid_name in self.grids:
for cache_name, cache in iteritems(self.caches):
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage is False:
coverage = False
complete_extent = False
elif self.coverage:
coverage = self.coverage.transform_to(grid.srs)
complete_extent = False
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
complete_extent = True
try:
if coverage is not False:
coverage.extent.llbbox
except TransformationError:
raise SeedConfigurationError(
'%s: coverage transformation error' % self.name)
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(range(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
# for caches without timestamp support (like MBTiles)
if self.remove_timestamp is self.init_time or self.remove_timestamp == 0:
# remove everything
self.remove_timestamp = 0
else:
raise SeedConfigurationError(
"cleanup does not support remove_before for '%s'"
" because cache '%s' does not support timestamps" %
(self.name, cache_name))
md = dict(name=self.name,
cache_name=cache_name,
grid_name=grid_name)
yield CleanupTask(md,
tile_manager,
levels,
self.remove_timestamp,
coverage=coverage,
complete_extent=complete_extent)
def layer_srs_bbox(self, layer, epsg_axis_order=False):
layer_srs_code = layer.extent.srs.srs_code
for srs, extent in iteritems(self.srs_extents):
if extent.is_default:
bbox = layer.extent.bbox_for(SRS(srs))
else:
bbox = extent.bbox_for(SRS(srs))
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, srs)
yield srs, bbox
# add native srs
if layer_srs_code not in self.srs_extents:
bbox = layer.extent.bbox
if epsg_axis_order:
bbox = switch_bbox_epsg_axis_order(bbox, layer_srs_code)
yield layer_srs_code, bbox
def seeds(cap, caches):
seeds = {}
cleanups = {}
for cache_name, cache in iteritems(caches):
for grid in cache['grids']:
seeds[cache_name + '_' + grid] = {
'caches': [cache_name],
'grids': [grid],
}
cleanups[cache_name + '_' + grid] = {
'caches': [cache_name],
'grids': [grid],
'remove_before': {
'time': '1900-01-01T00:00:00',
}
}
return seeds, cleanups
def load_config(config,
config_file=None,
config_dict=None,
clear_existing=False):
if clear_existing:
for key in list(config.keys()):
del config[key]
if config_dict is None:
config_dict = load_yaml_file(config_file)
defaults = _to_options_map(config_dict)
if defaults:
for key, value in iteritems(defaults):
if key in config and hasattr(config[key], 'update'):
config[key].update(value)
else:
config[key] = value
def seed_tasks(self):
for grid_name in self.grids:
for cache_name, cache in iteritems(self.caches):
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage is False:
coverage = False
elif self.coverage:
coverage = self.coverage.transform_to(grid.srs)
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
try:
if coverage is not False:
coverage.extent.llbbox
except TransformationError:
raise SeedConfigurationError(
'%s: coverage transformation error' % self.name)
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(range(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
if self.refresh_timestamp:
# remove everything
self.refresh_timestamp = 0
md = dict(name=self.name,
cache_name=cache_name,
grid_name=grid_name)
if tile_manager.rescale_tiles:
if tile_manager.rescale_tiles > 0:
levels = levels[::-1]
for l in levels:
yield SeedTask(md, tile_manager, [l],
self.refresh_timestamp, coverage)
else:
yield SeedTask(md, tile_manager, levels,
self.refresh_timestamp, coverage)
def restart_with_reloader():
"""Spawn a new Python interpreter with the same arguments as this one,
but running the reloader thread.
"""
while 1:
_log('info', ' * Restarting with reloader')
args = [sys.executable] + sys.argv
new_environ = os.environ.copy()
new_environ['WERKZEUG_RUN_MAIN'] = 'true'
# a weird bug on windows. sometimes unicode strings end up in the
# environment and subprocess.call does not like this, encode them
# to latin1 and continue.
if os.name == 'nt' and PY2:
for key, value in iteritems(new_environ):
if isinstance(value, text_type):
new_environ[key] = value.encode('iso-8859-1')
exit_code = subprocess.call(args, env=new_environ)
if exit_code != 3:
return exit_code
def authorized_layers(self, feature, layers, env, query_extent):
if 'mapproxy.authorize' in env:
result = env['mapproxy.authorize']('wms.' + feature, layers[:],
environ=env, query_extent=query_extent)
if result['authorized'] == 'unauthenticated':
raise RequestError('unauthorized', status=401)
if result['authorized'] == 'full':
return PERMIT_ALL_LAYERS, None
layers = {}
if result['authorized'] == 'partial':
for layer_name, permissions in iteritems(result['layers']):
if permissions.get(feature, False) == True:
layers[layer_name] = permissions.get('limited_to')
limited_to = result.get('limited_to')
if limited_to:
coverage = load_limited_to(limited_to)
else:
coverage = None
return layers, coverage
else:
return PERMIT_ALL_LAYERS, None
def doc(self, tile, grid):
doc = {}
x, y, z = tile.coord
for key, value in iteritems(self.attributes):
if not isinstance(value, string_type) or not value.startswith('{{'):
doc[key] = value
continue
if value == '{{timestamp}}':
doc[key] = time.time()
elif value == '{{x}}':
doc[key] = x
elif value == '{{y}}':
doc[key] = y
elif value in ('{{z}}', '{{level}}'):
doc[key] = z
elif value == '{{utc_iso}}':
doc[key] = utc_now_isoformat()
elif value == '{{wgs_tile_centroid}}':
tile_bbox = grid.tile_bbox(tile.coord)
centroid = (
tile_bbox[0] + (tile_bbox[2]-tile_bbox[0])/2,
tile_bbox[1] + (tile_bbox[3]-tile_bbox[1])/2
)
centroid = grid.srs.transform_to(SRS(4326), centroid)
doc[key] = centroid
elif value == '{{tile_centroid}}':
tile_bbox = grid.tile_bbox(tile.coord)
centroid = (
tile_bbox[0] + (tile_bbox[2]-tile_bbox[0])/2,
tile_bbox[1] + (tile_bbox[3]-tile_bbox[1])/2
)
doc[key] = centroid
else:
raise ValueError('unknown CouchDB tile_metadata value: %r' % (value, ))
return doc
def _needs_reload(self):
for conf_file, timestamp in iteritems(self.app.config_files):
m_time = os.path.getmtime(conf_file)
if m_time > timestamp:
return True
return False
def __repr__(self):
items = [(k, v) for k, v in iteritems(self)]
items.sort()
return '<%s %s>' % (self.__class__.__name__, ' '.join(
['%s=%r' % (k, v) for k, v in items]))
def __init__(self, **kw):
for name, value in iteritems(kw):
setattr(self, name, value)
def raw_params(self):
params = {}
for key, value in iteritems(self.params):
params[key] = value
return params
def update(self, mapping=(), append=False):
"""A `NoCaseMultiDict` can be updated from an iterable of
``(key, value)`` tuples or a dict.
"""
for _, (key, values) in iteritems(self._gen_dict(mapping)):
self.set(key, values, append=append, unpack=True)
def config_command(args):
parser = optparse.OptionParser("usage: %prog autoconfig [options]")
parser.add_option(
'--capabilities',
help="URL or filename of WMS 1.1.1/1.3.0 capabilities document")
parser.add_option('--output',
help="filename for created MapProxy config [default: -]",
default="-")
parser.add_option('--output-seed',
help="filename for created seeding config")
parser.add_option('--base',
help='base config to include in created MapProxy config')
parser.add_option(
'--overwrite',
help='YAML file with overwrites for the created MapProxy config')
parser.add_option(
'--overwrite-seed',
help='YAML file with overwrites for the created seeding config')
parser.add_option('--force',
default=False,
action='store_true',
help="overwrite existing files")
options, args = parser.parse_args(args)
if not options.capabilities:
parser.print_help()
print("\nERROR: --capabilities required", file=sys.stderr)
return 2
if not options.output and not options.output_seed:
parser.print_help()
print("\nERROR: --output and/or --output-seed required",
file=sys.stderr)
return 2
if not options.force:
if options.output and options.output != '-' and os.path.exists(
options.output):
print("\nERROR: %s already exists, use --force to overwrite" %
options.output,
file=sys.stderr)
return 2
if options.output_seed and options.output_seed != '-' and os.path.exists(
options.output_seed):
print("\nERROR: %s already exists, use --force to overwrite" %
options.output_seed,
file=sys.stderr)
return 2
log = logging.getLogger('mapproxy_conf_cmd')
log.addHandler(logging.StreamHandler())
setup_logging(logging.WARNING)
srs_grids = {}
if options.base:
base = load_configuration(options.base)
for name, grid_conf in iteritems(base.grids):
if name.startswith('GLOBAL_'):
continue
srs_grids[grid_conf.tile_grid().srs.srs_code] = name
cap_doc = options.capabilities
if cap_doc.startswith(('http://', 'https://')):
cap_doc = download_capabilities(options.capabilities).read()
else:
cap_doc = open(cap_doc, 'rb').read()
try:
cap = parse_capabilities(BytesIO(cap_doc))
except (xml.etree.ElementTree.ParseError, ValueError) as ex:
print(ex, file=sys.stderr)
print(cap_doc[:1000] + ('...' if len(cap_doc) > 1000 else ''),
file=sys.stderr)
return 3
overwrite = None
if options.overwrite:
with open(options.overwrite, 'rb') as f:
overwrite = yaml.safe_load(f)
overwrite_seed = None
if options.overwrite_seed:
with open(options.overwrite_seed, 'rb') as f:
overwrite_seed = yaml.safe_load(f)
conf = {}
if options.base:
conf['base'] = os.path.abspath(options.base)
conf['services'] = {'wms': {'md': {'title': cap.metadata()['title']}}}
if overwrite:
conf['services'] = update_config(conf['services'],
overwrite.pop('service', {}))
conf['sources'] = sources(cap)
if overwrite:
conf['sources'] = update_config(conf['sources'],
overwrite.pop('sources', {}))
conf['caches'] = caches(cap, conf['sources'], srs_grids=srs_grids)
if overwrite:
conf['caches'] = update_config(conf['caches'],
overwrite.pop('caches', {}))
conf['layers'] = layers(cap, conf['caches'])
if overwrite:
conf['layers'] = update_config(conf['layers'],
overwrite.pop('layers', {}))
if overwrite:
conf = update_config(conf, overwrite)
seed_conf = {}
seed_conf['seeds'], seed_conf['cleanups'] = seeds(cap, conf['caches'])
if overwrite_seed:
seed_conf = update_config(seed_conf, overwrite_seed)
if options.output:
with file_or_stdout(options.output) as f:
write_header(f, options.capabilities)
yaml.dump(conf,
f,
default_flow_style=False,
Dumper=MapProxyYAMLDumper)
if options.output_seed:
with file_or_stdout(options.output_seed) as f:
write_header(f, options.capabilities)
yaml.dump(seed_conf,
f,
default_flow_style=False,
Dumper=MapProxyYAMLDumper)
return 0
def start_response(self, resp):
self.send_response(int(resp.get('status', '200')))
if 'headers' in resp:
for key, value in iteritems(resp['headers']):
self.send_header(key, value)
self.end_headers()
def __init__(self, seed_conf, mapproxy_conf):
self.conf = seed_conf
self.mapproxy_conf = mapproxy_conf
self.grids = bidict((name, grid_conf.tile_grid()) for name, grid_conf in iteritems(self.mapproxy_conf.grids))
