def test_with_warnings(object):
with TempFile() as f:
open(f, 'w').write("""
services:
unknown:
""")
load_configuration(f) # defaults to ignore_warnings=True
assert_raises(ConfigurationError, load_configuration, f, ignore_warnings=False)
def test_with_warnings(object):
with TempFile() as f:
open(f, 'wb').write(b"""
services:
unknown:
""")
load_configuration(f) # defaults to ignore_warnings=True
with pytest.raises(ConfigurationError):
load_configuration(f, ignore_warnings=False)
def test_loading(self):
with TempFile() as gp:
open(gp, 'w').write(self.yaml_grand_parent)
self.yaml_parent = """
base:
- %s
%s
""" % (gp, self.yaml_parent)
with TempFile() as p:
open(p, 'w').write(self.yaml_parent)
self.yaml_string = """
base: [%s]
%s
""" % (p, self.yaml_string)
with TempFile() as cfg:
open(cfg, 'w').write(self.yaml_string)
config = load_configuration(cfg)
http = config.globals.get_value('http')
eq_(http['client_timeout'], 1)
eq_(http['headers']['bar'], 'qux')
eq_(http['headers']['foo'], 'bar')
eq_(http['headers']['baz'], 'quux')
eq_(http['method'], 'GET')
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
:param reloader: reload mapproxy.yaml when it changed
"""
if sys.version_info[0] == 2 and sys.version_info[1] == 5:
warnings.warn('Support for Python 2.5 is deprecated since 1.7.0 and will be dropped with 1.8.0', FutureWarning)
if reloader:
make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug,
reloader=False)
return ReloaderApp(services_conf, make_app)
try:
conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError as e:
log.fatal(e)
raise
config_files = conf.config_files()
app = MapProxyApp(services, conf.base_config)
if debug:
app = wrap_wsgi_debug(app, conf)
app.config_files = config_files
return app
def test_loading(self):
with TempFile() as gp:
open(gp, 'w').write(self.yaml_grand_parent)
self.yaml_parent = """
base:
- %s
%s
""" % (gp, self.yaml_parent)
with TempFile() as p:
open(p, 'w').write(self.yaml_parent)
self.yaml_string = """
base: [%s]
%s
""" % (p, self.yaml_string)
with TempFile() as cfg:
open(cfg, 'w').write(self.yaml_string)
config = load_configuration(cfg)
http = config.globals.get_value('http')
eq_(http['client_timeout'], 1)
eq_(http['headers']['bar'], 'qux')
eq_(http['headers']['foo'], 'bar')
eq_(http['headers']['baz'], 'quux')
eq_(http['method'], 'GET')
def make_wsgi_app(services_conf=None,
debug=False,
ignore_config_warnings=True,
reloader=False):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
:param reloader: reload mapproxy.yaml when it changed
"""
if reloader:
make_app = lambda: make_wsgi_app(
services_conf=services_conf, debug=debug, reloader=False)
return ReloaderApp(services_conf, make_app)
try:
conf = load_configuration(mapproxy_conf=services_conf,
ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError as e:
log.fatal(e)
raise
config_files = conf.config_files()
app = MapProxyApp(services, conf.base_config)
if debug:
app = wrap_wsgi_debug(app, conf)
app.config_files = config_files
return app
def test_loading(self):
with TempFile() as gp:
open(gp, 'wb').write(self.yaml_grand_parent.encode('utf-8'))
self.yaml_parent = """
base:
- %s
%s
""" % (gp, self.yaml_parent)
with TempFile() as p:
open(p, 'wb').write(self.yaml_parent.encode("utf-8"))
self.yaml_string = """
base: [%s]
%s
""" % (p, self.yaml_string)
with TempFile() as cfg:
open(cfg, 'wb').write(self.yaml_string.encode("utf-8"))
config = load_configuration(cfg)
http = config.globals.get_value('http')
assert http['client_timeout'] == 1
assert http['headers']['bar'] == 'qux'
assert http['headers']['foo'] == 'bar'
assert http['headers']['baz'] == 'quux'
assert http['method'] == 'GET'
config_files = config.config_files()
assert set(config_files.keys()) == set([gp, p, cfg])
assert abs(config_files[gp] - time.time()) < 10
assert abs(config_files[p] - time.time()) < 10
assert abs(config_files[cfg] - time.time()) < 10
def test_loading(self):
with TempFile() as gp:
open(gp, 'w').write(self.yaml_grand_parent)
self.yaml_parent = """
base:
- %s
%s
""" % (gp, self.yaml_parent)
with TempFile() as p:
open(p, 'w').write(self.yaml_parent)
self.yaml_string = """
base: [%s]
%s
""" % (p, self.yaml_string)
with TempFile() as cfg:
open(cfg, 'w').write(self.yaml_string)
config = load_configuration(cfg)
http = config.globals.get_value('http')
eq_(http['client_timeout'], 1)
eq_(http['headers']['bar'], 'qux')
eq_(http['headers']['foo'], 'bar')
eq_(http['headers']['baz'], 'quux')
eq_(http['method'], 'GET')
config_files = config.config_files()
eq_(set(config_files.keys()), set([gp, p, cfg]))
assert abs(config_files[gp] - time.time()) < 10
assert abs(config_files[p] - time.time()) < 10
assert abs(config_files[cfg] - time.time()) < 10
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
:param reloader: reload mapproxy.yaml when it changed
"""
if reloader:
make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug,
reloader=False)
return ReloaderApp(services_conf, make_app)
try:
conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError as e:
log.fatal(e)
raise
config_files = conf.config_files()
app = MapProxyApp(services, conf.base_config)
if debug:
app = wrap_wsgi_debug(app, conf)
app.config_files = config_files
return app
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging()
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError, ex:
print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
:param reloader: reload mapproxy.yaml when it changed
"""
if sys.version_info[0] == 2 and sys.version_info[1] == 5:
warnings.warn('Support for Python 2.5 is deprecated since 1.7.0 and will be dropped with 1.8.0', FutureWarning)
if reloader:
make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug,
reloader=False)
return ReloaderApp(services_conf, make_app)
try:
conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError as e:
log.fatal(e)
raise
config_files = conf.config_files()
app = MapProxyApp(services, conf.base_config)
if debug:
app = wrap_wsgi_debug(app, conf)
app.config_files = config_files
return app
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option("-g", "--grid", dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option("--all", dest="show_all", action="store_true", default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage")
coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined")
coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError, e:
print >>sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename)
sys.exit(2)
def setup(self):
self.dir = tempfile.mkdtemp()
shutil.copy(os.path.join(FIXTURE_DIR, self.seed_conf_name), self.dir)
shutil.copy(os.path.join(FIXTURE_DIR, self.mapproxy_conf_name), self.dir)
self.seed_conf_file = os.path.join(self.dir, self.seed_conf_name)
self.mapproxy_conf_file = os.path.join(self.dir, self.mapproxy_conf_name)
self.mapproxy_conf = load_configuration(self.mapproxy_conf_file, seed=True)
def config_command(args):
parser = optparse.OptionParser("usage: %prog autoconfig [options]")
parser.add_option("--capabilities", help="URL or filename of WMS 1.1.1/1.3.0 capabilities document")
parser.add_option("--output", help="filename for created MapProxy config [default: -]", default="-")
parser.add_option("--output-seed", help="filename for created seeding config")
parser.add_option("--base", help="base config to include in created MapProxy config")
parser.add_option("--overwrite", help="YAML file with overwrites for the created MapProxy config")
parser.add_option("--overwrite-seed", help="YAML file with overwrites for the created seeding config")
parser.add_option("--force", default=False, action="store_true", help="overwrite existing files")
options, args = parser.parse_args(args)
if not options.capabilities:
parser.print_help()
print >>sys.stderr, "\nERROR: --capabilities required"
return 2
if not options.output and not options.output_seed:
parser.print_help()
print >>sys.stderr, "\nERROR: --output and/or --output-seed required"
return 2
if not options.force:
if options.output and options.output != "-" and os.path.exists(options.output):
print >>sys.stderr, "\nERROR: %s already exists, use --force to overwrite" % options.output
return 2
if options.output_seed and options.output_seed != "-" and os.path.exists(options.output_seed):
print >>sys.stderr, "\nERROR: %s already exists, use --force to overwrite" % options.output_seed
return 2
log = logging.getLogger("mapproxy_conf_cmd")
log.addHandler(logging.StreamHandler())
setup_logging(logging.WARNING)
srs_grids = {}
if options.base:
base = load_configuration(options.base)
for name, grid_conf in base.grids.iteritems():
if name.startswith("GLOBAL_"):
continue
srs_grids[grid_conf.tile_grid().srs.srs_code] = name
cap_doc = options.capabilities
if cap_doc.startswith(("http://", "https://")):
cap_doc = download_capabilities(options.capabilities).read()
else:
cap_doc = open(cap_doc, "rb").read()
try:
cap = parse_capabilities(StringIO(cap_doc))
except (xml.etree.ElementTree.ParseError, ValueError), ex:
print >>sys.stderr, ex
print >>sys.stderr, cap_doc[:1000] + ("..." if len(cap_doc) > 1000 else "")
return 3
def setup(self):
self.dir = tempfile.mkdtemp()
shutil.copy(os.path.join(FIXTURE_DIR, self.seed_conf_name), self.dir)
shutil.copy(os.path.join(FIXTURE_DIR, self.mapproxy_conf_name), self.dir)
shutil.copy(os.path.join(FIXTURE_DIR, self.empty_ogrdata), self.dir)
self.seed_conf_file = os.path.join(self.dir, self.seed_conf_name)
self.mapproxy_conf_file = os.path.join(self.dir, self.mapproxy_conf_name)
self.mapproxy_conf = load_configuration(self.mapproxy_conf_file, seed=True)
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option(
"-g",
"--grid",
dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option(
"--all",
dest="show_all",
action="store_true",
default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option(
"-l",
"--list",
dest="list_grids",
action="store_true",
default=False,
help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group(
"Approximate the number of tiles within a given coverage")
coverage_group.add_option(
"-s",
"--seed-conf",
dest="seed_config",
help="Seed configuration, where the coverage is defined")
coverage_group.add_option(
"-c",
"--coverage-name",
dest="coverage",
help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError, e:
print >> sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename)
sys.exit(2)
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
"""
try:
conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError, e:
log.fatal(e)
raise
def make_wsgi_app(services_conf=None,
debug=False,
ignore_config_warnings=True):
"""
Create a MapProxyApp with the given services conf.
:param services_conf: the file name of the mapproxy.yaml configuration
"""
try:
conf = load_configuration(mapproxy_conf=services_conf,
ignore_warnings=ignore_config_warnings)
services = conf.configured_services()
except ConfigurationError, e:
log.fatal(e)
raise
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error('missing seed_conf file as last argument or --seed-conf option')
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging()
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError, ex:
print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
def export_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration")
parser.add_option("--source", dest="source", help="source to export (source or cache)")
parser.add_option(
"--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string"
)
parser.add_option("--dest", help="destination of the export (directory or filename)")
parser.add_option("--type", help="type of the export format")
parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option(
"--fetch-missing-tiles",
dest="fetch_missing_tiles",
action="store_true",
default=False,
help="if missing tiles should be fetched from the sources",
)
parser.add_option(
"--force", action="store_true", default=False, help="overwrite/append to existing --dest files/directories"
)
parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output")
parser.add_option(
"-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes"
)
parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource")
parser.add_option("--srs", help="the SRS of the coverage")
parser.add_option("--where", help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ["mapproxy_conf", "grid", "source", "dest", "levels"]
for required in required_options:
if not getattr(options, required):
print("ERROR: missing required option --%s" % required.replace("_", "-"), file=sys.stderr)
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError as e:
print("ERROR: ", "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print("ERROR: invalid configuration (see above)", file=sys.stderr)
sys.exit(2)
if "=" in options.grid:
try:
grid_conf = parse_grid_definition(options.grid)
except ValidationError as ex:
print("ERROR: invalid grid configuration", file=sys.stderr)
for error in ex.errors:
print(" ", error, file=sys.stderr)
sys.exit(2)
except ValueError:
print("ERROR: invalid grid configuration", file=sys.stderr)
sys.exit(2)
options.grid = "tmp_mapproxy_export_grid"
grid_conf["name"] = options.grid
custom_grid = True
conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
else:
custom_grid = False
if os.path.exists(options.dest) and not options.force:
print("ERROR: destination exists, remove first or use --force", file=sys.stderr)
sys.exit(2)
cache_conf = {"name": "export", "grids": [options.grid], "sources": [options.source]}
if options.type == "mbtile":
cache_conf["cache"] = {"type": "mbtiles", "filename": options.dest}
elif options.type in ("tc", "mapproxy"):
cache_conf["cache"] = {"type": "file", "directory": options.dest}
elif options.type in ("tms", None): # default
cache_conf["cache"] = {"type": "file", "directory_layout": "tms", "directory": options.dest}
else:
print("ERROR: unsupported --type %s" % (options.type,), file=sys.stderr)
sys.exit(2)
if not options.fetch_missing_tiles:
for source in conf.sources.values():
source.conf["seed_only"] = True
tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]
levels = parse_levels(options.levels)
if levels[-1] >= tile_grid.levels:
print("ERROR: destination grid only has %d levels" % tile_grid.levels, file=sys.stderr)
sys.exit(2)
if options.srs:
srs = SRS(options.srs)
else:
srs = tile_grid.srs
if options.coverage:
seed_coverage = load_coverage(
{"datasource": options.coverage, "srs": srs, "where": options.where}, base_path=os.getcwd()
)
else:
seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
if not supports_tiled_access(mgr):
print("WARN: grids are incompatible. needs to scale/reproject tiles for export.", file=sys.stderr)
md = dict(name="export", cache_name="cache", grid_name=options.grid, dest=options.dest)
task = SeedTask(md, mgr, levels, None, seed_coverage)
print(format_export_task(task, custom_grid=custom_grid))
logger = ProgressLog(verbose=True, silent=False)
try:
seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency)
except KeyboardInterrupt:
print("stopping...", file=sys.stderr)
sys.exit(2)
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option("-g", "--grid", dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option("--all", dest="show_all", action="store_true", default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage")
coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined")
coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
if options.show_all or options.grid_name:
grids = proxy_configuration.grids
else:
caches = proxy_configuration.caches
grids = {}
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict((key.lower(), value) for (key, value) in iteritems(grids))
if not grids.get(options.grid_name, False):
print('grid not found: %s' % (options.grid_name,))
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
try:
seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
except SeedConfigurationError as e:
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print('Old seed configuration format not supported')
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
print('--coverage and --seed-conf can only be used together')
sys.exit(1)
if options.list_grids:
display_grids_list(grids)
elif options.grid_name:
display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage)
else:
display_grids(grids, coverage=coverage)
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def config_command(args):
parser = optparse.OptionParser("usage: %prog autoconfig [options]")
parser.add_option(
'--capabilities',
help="URL or filename of WMS 1.1.1/1.3.0 capabilities document")
parser.add_option('--output',
help="filename for created MapProxy config [default: -]",
default="-")
parser.add_option('--output-seed',
help="filename for created seeding config")
parser.add_option('--base',
help='base config to include in created MapProxy config')
parser.add_option(
'--overwrite',
help='YAML file with overwrites for the created MapProxy config')
parser.add_option(
'--overwrite-seed',
help='YAML file with overwrites for the created seeding config')
parser.add_option('--force',
default=False,
action='store_true',
help="overwrite existing files")
options, args = parser.parse_args(args)
if not options.capabilities:
parser.print_help()
print("\nERROR: --capabilities required", file=sys.stderr)
return 2
if not options.output and not options.output_seed:
parser.print_help()
print("\nERROR: --output and/or --output-seed required",
file=sys.stderr)
return 2
if not options.force:
if options.output and options.output != '-' and os.path.exists(
options.output):
print("\nERROR: %s already exists, use --force to overwrite" %
options.output,
file=sys.stderr)
return 2
if options.output_seed and options.output_seed != '-' and os.path.exists(
options.output_seed):
print("\nERROR: %s already exists, use --force to overwrite" %
options.output_seed,
file=sys.stderr)
return 2
log = logging.getLogger('mapproxy_conf_cmd')
log.addHandler(logging.StreamHandler())
setup_logging(logging.WARNING)
srs_grids = {}
if options.base:
base = load_configuration(options.base)
for name, grid_conf in iteritems(base.grids):
if name.startswith('GLOBAL_'):
continue
srs_grids[grid_conf.tile_grid().srs.srs_code] = name
cap_doc = options.capabilities
if cap_doc.startswith(('http://', 'https://')):
cap_doc = download_capabilities(options.capabilities).read()
else:
cap_doc = open(cap_doc, 'rb').read()
try:
cap = parse_capabilities(BytesIO(cap_doc))
except (xml.etree.ElementTree.ParseError, ValueError) as ex:
print(ex, file=sys.stderr)
print(cap_doc[:1000] + ('...' if len(cap_doc) > 1000 else ''),
file=sys.stderr)
return 3
overwrite = None
if options.overwrite:
with open(options.overwrite, 'rb') as f:
overwrite = yaml.safe_load(f)
overwrite_seed = None
if options.overwrite_seed:
with open(options.overwrite_seed, 'rb') as f:
overwrite_seed = yaml.safe_load(f)
conf = {}
if options.base:
conf['base'] = os.path.abspath(options.base)
conf['services'] = {'wms': {'md': {'title': cap.metadata()['title']}}}
if overwrite:
conf['services'] = update_config(conf['services'],
overwrite.pop('service', {}))
conf['sources'] = sources(cap)
if overwrite:
conf['sources'] = update_config(conf['sources'],
overwrite.pop('sources', {}))
conf['caches'] = caches(cap, conf['sources'], srs_grids=srs_grids)
if overwrite:
conf['caches'] = update_config(conf['caches'],
overwrite.pop('caches', {}))
conf['layers'] = layers(cap, conf['caches'])
if overwrite:
conf['layers'] = update_config(conf['layers'],
overwrite.pop('layers', {}))
if overwrite:
conf = update_config(conf, overwrite)
seed_conf = {}
seed_conf['seeds'], seed_conf['cleanups'] = seeds(cap, conf['caches'])
if overwrite_seed:
seed_conf = update_config(seed_conf, overwrite_seed)
if options.output:
with file_or_stdout(options.output) as f:
write_header(f, options.capabilities)
yaml.dump(conf,
f,
default_flow_style=False,
Dumper=MapProxyYAMLDumper)
if options.output_seed:
with file_or_stdout(options.output_seed) as f:
write_header(f, options.capabilities)
yaml.dump(seed_conf,
f,
default_flow_style=False,
Dumper=MapProxyYAMLDumper)
return 0
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option(
"-g",
"--grid",
dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option(
"--all",
dest="show_all",
action="store_true",
default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option(
"-l",
"--list",
dest="list_grids",
action="store_true",
default=False,
help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group(
"Approximate the number of tiles within a given coverage")
coverage_group.add_option(
"-s",
"--seed-conf",
dest="seed_config",
help="Seed configuration, where the coverage is defined")
coverage_group.add_option(
"-c",
"--coverage-name",
dest="coverage",
help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ',
"%s: '%s'" % (e.strerror, e.filename),
file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
if options.show_all or options.grid_name:
grids = proxy_configuration.grids
else:
caches = proxy_configuration.caches
grids = {}
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict(
(key.lower(), value) for (key, value) in iteritems(grids))
if not grids.get(options.grid_name, False):
print('grid not found: %s' % (options.grid_name, ))
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
try:
seed_conf = load_seed_tasks_conf(options.seed_config,
proxy_configuration)
except SeedConfigurationError as e:
print('ERROR: invalid configuration (see above)',
file=sys.stderr)
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print('Old seed configuration format not supported')
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage
and not options.seed_config) or (not options.coverage
and options.seed_config):
print('--coverage and --seed-conf can only be used together')
sys.exit(1)
if options.list_grids:
display_grids_list(grids)
elif options.grid_name:
display_grids({options.grid_name: grids[options.grid_name]},
coverage=coverage)
else:
display_grids(grids, coverage=coverage)
def export_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration")
parser.add_option("--source", dest="source",
help="source to export (source or cache)")
parser.add_option("--grid",
help="grid for export. either the name of an existing grid or "
"the grid definition as a string")
parser.add_option("--dest",
help="destination of the export (directory or filename)")
parser.add_option("--type",
help="type of the export format")
parser.add_option("--levels",
help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles",
action='store_true', default=False,
help="if missing tiles should be fetched from the sources")
parser.add_option("--force",
action='store_true', default=False,
help="overwrite/append to existing --dest files/directories")
parser.add_option("-n", "--dry-run",
action="store_true", default=False,
help="do not export, just print output")
parser.add_option("-c", "--concurrency", type="int",
dest="concurrency", default=1,
help="number of parallel export processes")
parser.add_option("--coverage",
help="the coverage for the export as a BBOX string, WKT file "
"or OGR datasource")
parser.add_option("--srs",
help="the SRS of the coverage")
parser.add_option("--where",
help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
for required in required_options:
if not getattr(options, required):
print >>sys.stderr, 'ERROR: missing required option --%s' % required.replace('_', '-')
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError, e:
print >>sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename)
sys.exit(2)
def export_command(args=None):
parser = optparse.OptionParser("%prog export [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration")
parser.add_option("-q", "--quiet",
action="count", dest="quiet", default=0,
help="reduce number of messages to stdout, repeat to disable progress output")
parser.add_option("--source", dest="source",
help="source to export (source or cache)")
parser.add_option("--grid",
help="grid for export. either the name of an existing grid or "
"the grid definition as a string")
parser.add_option("--dest",
help="destination of the export (directory or filename)")
parser.add_option("--type",
help="type of the export format")
parser.add_option("--levels",
help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles",
action='store_true', default=False,
help="if missing tiles should be fetched from the sources")
parser.add_option("--force",
action='store_true', default=False,
help="overwrite/append to existing --dest files/directories")
parser.add_option("-n", "--dry-run",
action="store_true", default=False,
help="do not export, just print output")
parser.add_option("-c", "--concurrency", type="int",
dest="concurrency", default=1,
help="number of parallel export processes")
parser.add_option("--coverage",
help="the coverage for the export as a BBOX string, WKT file "
"or OGR datasource")
parser.add_option("--srs",
help="the SRS of the coverage")
parser.add_option("--where",
help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
for required in required_options:
if not getattr(options, required):
print('ERROR: missing required option --%s' % required.replace('_', '-'), file=sys.stderr)
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
if '=' in options.grid:
try:
grid_conf = parse_grid_definition(options.grid)
except ValidationError as ex:
print('ERROR: invalid grid configuration', file=sys.stderr)
for error in ex.errors:
print(' ', error, file=sys.stderr)
sys.exit(2)
except ValueError:
print('ERROR: invalid grid configuration', file=sys.stderr)
sys.exit(2)
options.grid = 'tmp_mapproxy_export_grid'
grid_conf['name'] = options.grid
custom_grid = True
conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
else:
custom_grid = False
if os.path.exists(options.dest) and not options.force:
print('ERROR: destination exists, remove first or use --force', file=sys.stderr)
sys.exit(2)
cache_conf = {
'name': 'export',
'grids': [options.grid],
'sources': [options.source],
}
if options.type == 'mbtile':
cache_conf['cache'] = {
'type': 'mbtiles',
'filename': options.dest,
}
elif options.type == 'sqlite':
cache_conf['cache'] = {
'type': 'sqlite',
'directory': options.dest,
}
elif options.type == 'geopackage':
cache_conf['cache'] = {
'type': 'geopackage',
'filename': options.dest,
}
elif options.type == 'compact-v1':
cache_conf['cache'] = {
'type': 'compact',
'version': 1,
'directory': options.dest,
}
elif options.type == 'compact-v2':
cache_conf['cache'] = {
'type': 'compact',
'version': 2,
'directory': options.dest,
}
elif options.type in ('tc', 'mapproxy'):
cache_conf['cache'] = {
'type': 'file',
'directory': options.dest,
}
elif options.type == 'arcgis':
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'arcgis',
'directory': options.dest,
}
elif options.type in ('tms', None): # default
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'tms',
'directory': options.dest,
}
else:
print('ERROR: unsupported --type %s' % (options.type, ), file=sys.stderr)
sys.exit(2)
if not options.fetch_missing_tiles:
for source in conf.sources.values():
source.conf['seed_only'] = True
tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]
levels = parse_levels(options.levels)
if levels[-1] >= tile_grid.levels:
print('ERROR: destination grid only has %d levels' % tile_grid.levels, file=sys.stderr)
sys.exit(2)
if options.srs:
srs = SRS(options.srs)
else:
srs = tile_grid.srs
if options.coverage:
seed_coverage = load_coverage(
{'datasource': options.coverage, 'srs': srs, 'where': options.where},
base_path=os.getcwd())
else:
seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
if not supports_tiled_access(mgr):
print('WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr)
md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
task = SeedTask(md, mgr, levels, 1, seed_coverage)
print(format_export_task(task, custom_grid=custom_grid))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
try:
seed_task(task, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency)
except KeyboardInterrupt:
print('stopping...', file=sys.stderr)
sys.exit(2)
def load_services(conf_file):
conf = load_configuration(conf_file)
return conf.configured_services()
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
if options.duration:
# calls with --duration are handled in call_with_duration
sys.exit(self.call_with_duration(options, args))
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
progress = None
if options.continue_seed or options.progress_file:
if not options.progress_file:
options.progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(options.progress_file,
continue_seed=options.continue_seed)
if options.reseed_file:
if not os.path.exists(options.reseed_file):
# create --reseed-file if missing
with open(options.reseed_file, 'w'):
pass
else:
if progress and not os.path.exists(options.progress_file):
# we have an existing --reseed-file but no --progress-file
# meaning the last seed call was completed
if options.reseed_interval and (
os.path.getmtime(options.reseed_file) >
(time.time() - options.reseed_interval)):
print("no need for re-seeding")
sys.exit(1)
os.utime(options.reseed_file, (time.time(), time.time()))
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels,
skip_uncached=options.skip_uncached)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def load_mapproxy_config(mapproxy_config_file):
try:
mapproxy_config = load_configuration(mapproxy_conf=mapproxy_config_file)
except ConfigurationError, e:
log.fatal(e)
raise
def export_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration")
parser.add_option("--source", dest="source", help="source to export (source or cache)")
parser.add_option(
"--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string"
)
parser.add_option("--dest", help="destination of the export (directory or filename)")
parser.add_option("--type", help="type of the export format")
parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option(
"--fetch-missing-tiles",
dest="fetch_missing_tiles",
action="store_true",
default=False,
help="if missing tiles should be fetched from the sources",
)
parser.add_option(
"--force", action="store_true", default=False, help="overwrite/append to existing --dest files/directories"
)
parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output")
parser.add_option(
"-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes"
)
parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource")
parser.add_option("--srs", help="the SRS of the coverage")
parser.add_option("--where", help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ["mapproxy_conf", "grid", "source", "dest", "levels"]
for required in required_options:
if not getattr(options, required):
print >> sys.stderr, "ERROR: missing required option --%s" % required.replace("_", "-")
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError, e:
print >> sys.stderr, "ERROR: ", "%s: '%s'" % (e.strerror, e.filename)
sys.exit(2)
def export_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration")
parser.add_option(
"-q",
"--quiet",
action="count",
dest="quiet",
default=0,
help=
"reduce number of messages to stdout, repeat to disable progress output"
)
parser.add_option("--source",
dest="source",
help="source to export (source or cache)")
parser.add_option(
"--grid",
help="grid for export. either the name of an existing grid or "
"the grid definition as a string")
parser.add_option("--dest",
help="destination of the export (directory or filename)")
parser.add_option("--type", help="type of the export format")
parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option(
"--fetch-missing-tiles",
dest="fetch_missing_tiles",
action='store_true',
default=False,
help="if missing tiles should be fetched from the sources")
parser.add_option(
"--force",
action='store_true',
default=False,
help="overwrite/append to existing --dest files/directories")
parser.add_option("-n",
"--dry-run",
action="store_true",
default=False,
help="do not export, just print output")
parser.add_option("-c",
"--concurrency",
type="int",
dest="concurrency",
default=1,
help="number of parallel export processes")
parser.add_option(
"--coverage",
help="the coverage for the export as a BBOX string, WKT file "
"or OGR datasource")
parser.add_option("--srs", help="the SRS of the coverage")
parser.add_option("--where", help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
for required in required_options:
if not getattr(options, required):
print('ERROR: missing required option --%s' %
required.replace('_', '-'),
file=sys.stderr)
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ',
"%s: '%s'" % (e.strerror, e.filename),
file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
if '=' in options.grid:
try:
grid_conf = parse_grid_definition(options.grid)
except ValidationError as ex:
print('ERROR: invalid grid configuration', file=sys.stderr)
for error in ex.errors:
print(' ', error, file=sys.stderr)
sys.exit(2)
except ValueError:
print('ERROR: invalid grid configuration', file=sys.stderr)
sys.exit(2)
options.grid = 'tmp_mapproxy_export_grid'
grid_conf['name'] = options.grid
custom_grid = True
conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
else:
custom_grid = False
if os.path.exists(options.dest) and not options.force:
print('ERROR: destination exists, remove first or use --force',
file=sys.stderr)
sys.exit(2)
cache_conf = {
'name': 'export',
'grids': [options.grid],
'sources': [options.source],
}
if options.type == 'mbtile':
cache_conf['cache'] = {
'type': 'mbtiles',
'filename': options.dest,
}
elif options.type == 'sqlite':
cache_conf['cache'] = {
'type': 'sqlite',
'directory': options.dest,
}
elif options.type == 'geopackage':
cache_conf['cache'] = {
'type': 'geopackage',
'filename': options.dest,
}
elif options.type == 'compact-v1':
cache_conf['cache'] = {
'type': 'compact',
'version': 1,
'directory': options.dest,
}
elif options.type == 'compact-v2':
cache_conf['cache'] = {
'type': 'compact',
'version': 2,
'directory': options.dest,
}
elif options.type in ('tc', 'mapproxy'):
cache_conf['cache'] = {
'type': 'file',
'directory': options.dest,
}
elif options.type == 'arcgis':
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'arcgis',
'directory': options.dest,
}
elif options.type in ('tms', None): # default
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'tms',
'directory': options.dest,
}
else:
print('ERROR: unsupported --type %s' % (options.type, ),
file=sys.stderr)
sys.exit(2)
if not options.fetch_missing_tiles:
for source in conf.sources.values():
source.conf['seed_only'] = True
tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]
levels = parse_levels(options.levels)
if levels[-1] >= tile_grid.levels:
print('ERROR: destination grid only has %d levels' % tile_grid.levels,
file=sys.stderr)
sys.exit(2)
if options.srs:
srs = SRS(options.srs)
else:
srs = tile_grid.srs
if options.coverage:
seed_coverage = load_coverage(
{
'datasource': options.coverage,
'srs': srs,
'where': options.where
},
base_path=os.getcwd())
else:
seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
if not supports_tiled_access(mgr):
print(
'WARN: grids are incompatible. needs to scale/reproject tiles for export.',
file=sys.stderr)
md = dict(name='export',
cache_name='cache',
grid_name=options.grid,
dest=options.dest)
task = SeedTask(md, mgr, levels, 1, seed_coverage)
print(format_export_task(task, custom_grid=custom_grid))
logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2)
try:
seed_task(task,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency)
except KeyboardInterrupt:
print('stopping...', file=sys.stderr)
sys.exit(2)
def config_command(args):
parser = optparse.OptionParser("usage: %prog autoconfig [options]")
parser.add_option("--capabilities", help="URL or filename of WMS 1.1.1/1.3.0 capabilities document")
parser.add_option("--output", help="filename for created MapProxy config [default: -]", default="-")
parser.add_option("--output-seed", help="filename for created seeding config")
parser.add_option("--base", help="base config to include in created MapProxy config")
parser.add_option("--overwrite", help="YAML file with overwrites for the created MapProxy config")
parser.add_option("--overwrite-seed", help="YAML file with overwrites for the created seeding config")
parser.add_option("--force", default=False, action="store_true", help="overwrite existing files")
options, args = parser.parse_args(args)
if not options.capabilities:
parser.print_help()
print("\nERROR: --capabilities required", file=sys.stderr)
return 2
if not options.output and not options.output_seed:
parser.print_help()
print("\nERROR: --output and/or --output-seed required", file=sys.stderr)
return 2
if not options.force:
if options.output and options.output != "-" and os.path.exists(options.output):
print("\nERROR: %s already exists, use --force to overwrite" % options.output, file=sys.stderr)
return 2
if options.output_seed and options.output_seed != "-" and os.path.exists(options.output_seed):
print("\nERROR: %s already exists, use --force to overwrite" % options.output_seed, file=sys.stderr)
return 2
log = logging.getLogger("mapproxy_conf_cmd")
log.addHandler(logging.StreamHandler())
setup_logging(logging.WARNING)
srs_grids = {}
if options.base:
base = load_configuration(options.base)
for name, grid_conf in iteritems(base.grids):
if name.startswith("GLOBAL_"):
continue
srs_grids[grid_conf.tile_grid().srs.srs_code] = name
cap_doc = options.capabilities
if cap_doc.startswith(("http://", "https://")):
cap_doc = download_capabilities(options.capabilities).read()
else:
cap_doc = open(cap_doc, "rb").read()
try:
cap = parse_capabilities(BytesIO(cap_doc))
except (xml.etree.ElementTree.ParseError, ValueError) as ex:
print(ex, file=sys.stderr)
print(cap_doc[:1000] + ("..." if len(cap_doc) > 1000 else ""), file=sys.stderr)
return 3
overwrite = None
if options.overwrite:
with open(options.overwrite, "rb") as f:
overwrite = yaml.load(f)
overwrite_seed = None
if options.overwrite_seed:
with open(options.overwrite_seed, "rb") as f:
overwrite_seed = yaml.load(f)
conf = {}
if options.base:
conf["base"] = os.path.abspath(options.base)
conf["services"] = {"wms": {"md": {"title": cap.metadata()["title"]}}}
if overwrite:
conf["services"] = update_config(conf["services"], overwrite.pop("service", {}))
conf["sources"] = sources(cap)
if overwrite:
conf["sources"] = update_config(conf["sources"], overwrite.pop("sources", {}))
conf["caches"] = caches(cap, conf["sources"], srs_grids=srs_grids)
if overwrite:
conf["caches"] = update_config(conf["caches"], overwrite.pop("caches", {}))
conf["layers"] = layers(cap, conf["caches"])
if overwrite:
conf["layers"] = update_config(conf["layers"], overwrite.pop("layers", {}))
if overwrite:
conf = update_config(conf, overwrite)
seed_conf = {}
seed_conf["seeds"], seed_conf["cleanups"] = seeds(cap, conf["caches"])
if overwrite_seed:
seed_conf = update_config(seed_conf, overwrite_seed)
if options.output:
with file_or_stdout(options.output) as f:
write_header(f, options.capabilities)
yaml.dump(conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper)
if options.output_seed:
with file_or_stdout(options.output_seed) as f:
write_header(f, options.capabilities)
yaml.dump(seed_conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper)
return 0
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error('missing seed_conf file as last argument or --seed-conf option')
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print('Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
progress_store=progress)
seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency, cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' % (
len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
concurrency=options.concurrency, progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def main():
parser = optparse.OptionParser()
parser.add_option("-f",
"--mapproxy-conf",
dest="conf_file",
default='mapproxy.yaml',
help="MapProxy configuration")
parser.add_option("--renderer",
default=None,
type=int,
help="Number of render processes.")
parser.add_option("--max-seed-renderer",
default=None,
type=int,
help="Maximum --renderer used for seeding.")
parser.add_option("--pidfile")
parser.add_option("--log-config", dest="log_config_file")
parser.add_option("--verbose", action="store_true", default=False)
options, args = parser.parse_args()
init_logging(options.log_config_file, options.verbose)
conf = load_configuration(options.conf_file, renderd=True)
broker_address = conf.globals.renderd_address
if not broker_address:
fatal('mapproxy config (%s) does not define renderd address' %
(options.conf_file))
broker_address = broker_address.replace('localhost', '127.0.0.1')
broker_port = int(broker_address.rsplit(':', 1)[1]) # TODO
tile_managers = {}
with conf:
for mapproxy_cache in conf.caches.itervalues():
for tile_grid_, extent_, tile_manager in mapproxy_cache.caches():
tile_manager._expire_timestamp = 2**32 # future ~2106
tile_managers[tile_manager.identifier] = tile_manager
if options.renderer is None:
pool_size = multiprocessing.cpu_count()
else:
pool_size = options.renderer
if options.max_seed_renderer is None:
max_seed_renderer = pool_size
else:
max_seed_renderer = min(options.max_seed_renderer, pool_size)
non_seed_renderer = pool_size - max_seed_renderer
process_priorities = [50] * non_seed_renderer + [0] * max_seed_renderer
log.debug('starting %d processes with the following min priorities: %r',
pool_size, process_priorities)
def worker_factory(in_queue, out_queue):
return SeedWorker(tile_managers,
conf.base_config,
in_queue=in_queue,
out_queue=out_queue)
worker_pool = WorkerPool(worker_factory, pool_size=pool_size)
task_queue = RenderQueue(process_priorities)
if options.pidfile:
with open(options.pidfile, 'w') as f:
f.write(str(os.getpid()))
def remove_pid():
os.unlink(options.pidfile)
atexit.register(remove_pid)
try:
broker = Broker(worker_pool, task_queue)
broker.start()
app = RenderdApp(broker)
server = CherryPyWSGIServer(
('127.0.0.1', broker_port),
app,
numthreads=64,
request_queue_size=256,
)
server.start()
except (KeyboardInterrupt, SystemExit):
print >> sys.stderr, 'exiting...'
if server:
server.stop()
return 0
except Exception:
log.fatal('fatal error, terminating', exc_info=True)
raise
def defrag_command(args=None):
parser = optparse.OptionParser(
"%prog defrag-compact [options] -f mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option(
"--min-percent",
type=float,
default=10.0,
help=
"Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)"
)
parser.add_option(
"--min-mb",
type=float,
default=1.0,
help=
"Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)"
)
parser.add_option("--dry-run",
"-n",
action="store_true",
help="Do not de-fragment, only print output")
parser.add_option("--caches",
dest="cache_names",
metavar='cache1,cache2,...',
help="only defragment the named caches")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
parser.print_help()
sys.exit(1)
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ',
"%s: '%s'" % (e.strerror, e.filename),
file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
available_caches = OrderedDict()
for name, cache_conf in proxy_configuration.caches.items():
for grid, extent, tile_mgr in cache_conf.caches():
if isinstance(tile_mgr.cache,
(CompactCacheV1, CompactCacheV2)):
available_caches.setdefault(name,
[]).append(tile_mgr.cache)
if options.cache_names:
defrag_caches = options.cache_names.split(',')
missing = set(defrag_caches).difference(available_caches.keys())
if missing:
print('unknown caches: %s' % (', '.join(missing), ))
print('available compact caches: %s' %
(', '.join(available_caches.keys()), ))
sys.exit(1)
else:
defrag_caches = None
for name, caches in available_caches.items():
if defrag_caches and name not in defrag_caches:
continue
for cache in caches:
logger = DefragLog(name)
defrag_compact_cache(
cache,
min_percent=options.min_percent / 100,
min_bytes=options.min_mb * 1024 * 1024,
dry_run=options.dry_run,
log_progress=logger,
)
def defrag_command(args=None):
parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option("--min-percent", type=float, default=10.0,
help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)")
parser.add_option("--min-mb", type=float, default=1.0,
help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)")
parser.add_option("--dry-run", "-n", action="store_true",
help="Do not de-fragment, only print output")
parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...',
help="only defragment the named caches")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
parser.print_help()
sys.exit(1)
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
available_caches = OrderedDict()
for name, cache_conf in proxy_configuration.caches.items():
for grid, extent, tile_mgr in cache_conf.caches():
if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)):
available_caches.setdefault(name, []).append(tile_mgr.cache)
if options.cache_names:
defrag_caches = options.cache_names.split(',')
missing = set(defrag_caches).difference(available_caches.keys())
if missing:
print('unknown caches: %s' % (', '.join(missing), ))
print('available compact caches: %s' %
(', '.join(available_caches.keys()), ))
sys.exit(1)
else:
defrag_caches = None
for name, caches in available_caches.items():
if defrag_caches and name not in defrag_caches:
continue
for cache in caches:
logger = DefragLog(name)
defrag_compact_cache(cache,
min_percent=options.min_percent/100,
min_bytes=options.min_mb*1024*1024,
dry_run=options.dry_run,
log_progress=logger,
)
def main():
parser = optparse.OptionParser()
parser.add_option("-f", "--mapproxy-conf",
dest="conf_file", default='mapproxy.yaml',
help="MapProxy configuration")
parser.add_option("--renderer", default=None, type=int,
help="Number of render processes.")
parser.add_option("--max-seed-renderer", default=None, type=int,
help="Maximum --renderer used for seeding.")
parser.add_option("--pidfile")
parser.add_option("--log-config", dest="log_config_file")
parser.add_option("--verbose", action="store_true", default=False)
options, args = parser.parse_args()
init_logging(options.log_config_file, options.verbose)
conf = load_configuration(options.conf_file, renderd=True)
broker_address = conf.globals.renderd_address
if not broker_address:
fatal('mapproxy config (%s) does not define renderd address' % (
options.conf_file))
broker_address = broker_address.replace('localhost', '127.0.0.1')
broker_port = int(broker_address.rsplit(':', 1)[1]) # TODO
tile_managers = {}
with conf:
for mapproxy_cache in conf.caches.itervalues():
for tile_grid_, extent_, tile_manager in mapproxy_cache.caches():
tile_manager._expire_timestamp = 2**32 # future ~2106
tile_managers[tile_manager.identifier] = tile_manager
if options.renderer is None:
pool_size = multiprocessing.cpu_count()
else:
pool_size = options.renderer
if options.max_seed_renderer is None:
max_seed_renderer = pool_size
else:
max_seed_renderer = min(options.max_seed_renderer, pool_size)
non_seed_renderer = pool_size - max_seed_renderer
process_priorities = [50] * non_seed_renderer + [0] * max_seed_renderer
log.debug('starting %d processes with the following min priorities: %r',
pool_size, process_priorities)
def worker_factory(in_queue, out_queue):
return SeedWorker(tile_managers, conf.base_config,
in_queue=in_queue,
out_queue=out_queue)
worker_pool = WorkerPool(worker_factory, pool_size=pool_size)
task_queue = RenderQueue(process_priorities)
if options.pidfile:
with open(options.pidfile, 'w') as f:
f.write(str(os.getpid()))
def remove_pid():
os.unlink(options.pidfile)
atexit.register(remove_pid)
try:
broker = Broker(worker_pool, task_queue)
broker.start()
app = RenderdApp(broker)
server = CherryPyWSGIServer(
('127.0.0.1', broker_port), app,
numthreads=64,
request_queue_size=256,
)
server.start()
except (KeyboardInterrupt, SystemExit):
print >>sys.stderr, 'exiting...'
if server:
server.stop()
return 0
except Exception:
log.fatal('fatal error, terminating', exc_info=True)
raise
