def test_with_warnings(object): with TempFile() as f: open(f, 'w').write(""" services: unknown: """) load_configuration(f) # defaults to ignore_warnings=True assert_raises(ConfigurationError, load_configuration, f, ignore_warnings=False)
def test_with_warnings(object): with TempFile() as f: open(f, 'wb').write(b""" services: unknown: """) load_configuration(f) # defaults to ignore_warnings=True with pytest.raises(ConfigurationError): load_configuration(f, ignore_warnings=False)
def test_loading(self): with TempFile() as gp: open(gp, 'w').write(self.yaml_grand_parent) self.yaml_parent = """ base: - %s %s """ % (gp, self.yaml_parent) with TempFile() as p: open(p, 'w').write(self.yaml_parent) self.yaml_string = """ base: [%s] %s """ % (p, self.yaml_string) with TempFile() as cfg: open(cfg, 'w').write(self.yaml_string) config = load_configuration(cfg) http = config.globals.get_value('http') eq_(http['client_timeout'], 1) eq_(http['headers']['bar'], 'qux') eq_(http['headers']['foo'], 'bar') eq_(http['headers']['baz'], 'quux') eq_(http['method'], 'GET')
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False): """ Create a MapProxyApp with the given services conf. :param services_conf: the file name of the mapproxy.yaml configuration :param reloader: reload mapproxy.yaml when it changed """ if sys.version_info[0] == 2 and sys.version_info[1] == 5: warnings.warn('Support for Python 2.5 is deprecated since 1.7.0 and will be dropped with 1.8.0', FutureWarning) if reloader: make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug, reloader=False) return ReloaderApp(services_conf, make_app) try: conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings) services = conf.configured_services() except ConfigurationError as e: log.fatal(e) raise config_files = conf.config_files() app = MapProxyApp(services, conf.base_config) if debug: app = wrap_wsgi_debug(app, conf) app.config_files = config_files return app
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False): """ Create a MapProxyApp with the given services conf. :param services_conf: the file name of the mapproxy.yaml configuration :param reloader: reload mapproxy.yaml when it changed """ if reloader: make_app = lambda: make_wsgi_app( services_conf=services_conf, debug=debug, reloader=False) return ReloaderApp(services_conf, make_app) try: conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings) services = conf.configured_services() except ConfigurationError as e: log.fatal(e) raise config_files = conf.config_files() app = MapProxyApp(services, conf.base_config) if debug: app = wrap_wsgi_debug(app, conf) app.config_files = config_files return app
def test_loading(self): with TempFile() as gp: open(gp, 'wb').write(self.yaml_grand_parent.encode('utf-8')) self.yaml_parent = """ base: - %s %s """ % (gp, self.yaml_parent) with TempFile() as p: open(p, 'wb').write(self.yaml_parent.encode("utf-8")) self.yaml_string = """ base: [%s] %s """ % (p, self.yaml_string) with TempFile() as cfg: open(cfg, 'wb').write(self.yaml_string.encode("utf-8")) config = load_configuration(cfg) http = config.globals.get_value('http') assert http['client_timeout'] == 1 assert http['headers']['bar'] == 'qux' assert http['headers']['foo'] == 'bar' assert http['headers']['baz'] == 'quux' assert http['method'] == 'GET' config_files = config.config_files() assert set(config_files.keys()) == set([gp, p, cfg]) assert abs(config_files[gp] - time.time()) < 10 assert abs(config_files[p] - time.time()) < 10 assert abs(config_files[cfg] - time.time()) < 10
def test_loading(self): with TempFile() as gp: open(gp, 'w').write(self.yaml_grand_parent) self.yaml_parent = """ base: - %s %s """ % (gp, self.yaml_parent) with TempFile() as p: open(p, 'w').write(self.yaml_parent) self.yaml_string = """ base: [%s] %s """ % (p, self.yaml_string) with TempFile() as cfg: open(cfg, 'w').write(self.yaml_string) config = load_configuration(cfg) http = config.globals.get_value('http') eq_(http['client_timeout'], 1) eq_(http['headers']['bar'], 'qux') eq_(http['headers']['foo'], 'bar') eq_(http['headers']['baz'], 'quux') eq_(http['method'], 'GET') config_files = config.config_files() eq_(set(config_files.keys()), set([gp, p, cfg])) assert abs(config_files[gp] - time.time()) < 10 assert abs(config_files[p] - time.time()) < 10 assert abs(config_files[cfg] - time.time()) < 10
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True, reloader=False): """ Create a MapProxyApp with the given services conf. :param services_conf: the file name of the mapproxy.yaml configuration :param reloader: reload mapproxy.yaml when it changed """ if reloader: make_app = lambda: make_wsgi_app(services_conf=services_conf, debug=debug, reloader=False) return ReloaderApp(services_conf, make_app) try: conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings) services = conf.configured_services() except ConfigurationError as e: log.fatal(e) raise config_files = conf.config_files() app = MapProxyApp(services, conf.base_config) if debug: app = wrap_wsgi_debug(app, conf) app.config_files = config_files return app
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging() try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError, ex: print "ERROR: " + '\n\t'.join(str(ex).split('\n')) sys.exit(2)
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option("-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option("--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage") coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError, e: print >>sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename) sys.exit(2)
def setup(self): self.dir = tempfile.mkdtemp() shutil.copy(os.path.join(FIXTURE_DIR, self.seed_conf_name), self.dir) shutil.copy(os.path.join(FIXTURE_DIR, self.mapproxy_conf_name), self.dir) self.seed_conf_file = os.path.join(self.dir, self.seed_conf_name) self.mapproxy_conf_file = os.path.join(self.dir, self.mapproxy_conf_name) self.mapproxy_conf = load_configuration(self.mapproxy_conf_file, seed=True)
def config_command(args): parser = optparse.OptionParser("usage: %prog autoconfig [options]") parser.add_option("--capabilities", help="URL or filename of WMS 1.1.1/1.3.0 capabilities document") parser.add_option("--output", help="filename for created MapProxy config [default: -]", default="-") parser.add_option("--output-seed", help="filename for created seeding config") parser.add_option("--base", help="base config to include in created MapProxy config") parser.add_option("--overwrite", help="YAML file with overwrites for the created MapProxy config") parser.add_option("--overwrite-seed", help="YAML file with overwrites for the created seeding config") parser.add_option("--force", default=False, action="store_true", help="overwrite existing files") options, args = parser.parse_args(args) if not options.capabilities: parser.print_help() print >>sys.stderr, "\nERROR: --capabilities required" return 2 if not options.output and not options.output_seed: parser.print_help() print >>sys.stderr, "\nERROR: --output and/or --output-seed required" return 2 if not options.force: if options.output and options.output != "-" and os.path.exists(options.output): print >>sys.stderr, "\nERROR: %s already exists, use --force to overwrite" % options.output return 2 if options.output_seed and options.output_seed != "-" and os.path.exists(options.output_seed): print >>sys.stderr, "\nERROR: %s already exists, use --force to overwrite" % options.output_seed return 2 log = logging.getLogger("mapproxy_conf_cmd") log.addHandler(logging.StreamHandler()) setup_logging(logging.WARNING) srs_grids = {} if options.base: base = load_configuration(options.base) for name, grid_conf in base.grids.iteritems(): if name.startswith("GLOBAL_"): continue srs_grids[grid_conf.tile_grid().srs.srs_code] = name cap_doc = options.capabilities if cap_doc.startswith(("http://", "https://")): cap_doc = download_capabilities(options.capabilities).read() else: cap_doc = open(cap_doc, "rb").read() try: cap = parse_capabilities(StringIO(cap_doc)) except (xml.etree.ElementTree.ParseError, ValueError), ex: print >>sys.stderr, ex print >>sys.stderr, cap_doc[:1000] + ("..." if len(cap_doc) > 1000 else "") return 3
def setup(self): self.dir = tempfile.mkdtemp() shutil.copy(os.path.join(FIXTURE_DIR, self.seed_conf_name), self.dir) shutil.copy(os.path.join(FIXTURE_DIR, self.mapproxy_conf_name), self.dir) shutil.copy(os.path.join(FIXTURE_DIR, self.empty_ogrdata), self.dir) self.seed_conf_file = os.path.join(self.dir, self.seed_conf_name) self.mapproxy_conf_file = os.path.join(self.dir, self.mapproxy_conf_name) self.mapproxy_conf = load_configuration(self.mapproxy_conf_file, seed=True)
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option( "-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option( "--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option( "-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group( "Approximate the number of tiles within a given coverage") coverage_group.add_option( "-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option( "-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError, e: print >> sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename) sys.exit(2)
def make_wsgi_app(services_conf=None, debug=False, ignore_config_warnings=True): """ Create a MapProxyApp with the given services conf. :param services_conf: the file name of the mapproxy.yaml configuration """ try: conf = load_configuration(mapproxy_conf=services_conf, ignore_warnings=ignore_config_warnings) services = conf.configured_services() except ConfigurationError, e: log.fatal(e) raise
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error('missing seed_conf file as last argument or --seed-conf option') else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging() try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError, ex: print "ERROR: " + '\n\t'.join(str(ex).split('\n')) sys.exit(2)
def export_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration") parser.add_option("--source", dest="source", help="source to export (source or cache)") parser.add_option( "--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string" ) parser.add_option("--dest", help="destination of the export (directory or filename)") parser.add_option("--type", help="type of the export format") parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10") parser.add_option( "--fetch-missing-tiles", dest="fetch_missing_tiles", action="store_true", default=False, help="if missing tiles should be fetched from the sources", ) parser.add_option( "--force", action="store_true", default=False, help="overwrite/append to existing --dest files/directories" ) parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output") parser.add_option( "-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes" ) parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource") parser.add_option("--srs", help="the SRS of the coverage") parser.add_option("--where", help="filter for OGR coverages") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] required_options = ["mapproxy_conf", "grid", "source", "dest", "levels"] for required in required_options: if not getattr(options, required): print("ERROR: missing required option --%s" % required.replace("_", "-"), file=sys.stderr) parser.print_help() sys.exit(1) try: conf = load_configuration(options.mapproxy_conf) except IOError as e: print("ERROR: ", "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print("ERROR: invalid configuration (see above)", file=sys.stderr) sys.exit(2) if "=" in options.grid: try: grid_conf = parse_grid_definition(options.grid) except ValidationError as ex: print("ERROR: invalid grid configuration", file=sys.stderr) for error in ex.errors: print(" ", error, file=sys.stderr) sys.exit(2) except ValueError: print("ERROR: invalid grid configuration", file=sys.stderr) sys.exit(2) options.grid = "tmp_mapproxy_export_grid" grid_conf["name"] = options.grid custom_grid = True conf.grids[options.grid] = GridConfiguration(grid_conf, conf) else: custom_grid = False if os.path.exists(options.dest) and not options.force: print("ERROR: destination exists, remove first or use --force", file=sys.stderr) sys.exit(2) cache_conf = {"name": "export", "grids": [options.grid], "sources": [options.source]} if options.type == "mbtile": cache_conf["cache"] = {"type": "mbtiles", "filename": options.dest} elif options.type in ("tc", "mapproxy"): cache_conf["cache"] = {"type": "file", "directory": options.dest} elif options.type in ("tms", None): # default cache_conf["cache"] = {"type": "file", "directory_layout": "tms", "directory": options.dest} else: print("ERROR: unsupported --type %s" % (options.type,), file=sys.stderr) sys.exit(2) if not options.fetch_missing_tiles: for source in conf.sources.values(): source.conf["seed_only"] = True tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0] levels = parse_levels(options.levels) if levels[-1] >= tile_grid.levels: print("ERROR: destination grid only has %d levels" % tile_grid.levels, file=sys.stderr) sys.exit(2) if options.srs: srs = SRS(options.srs) else: srs = tile_grid.srs if options.coverage: seed_coverage = load_coverage( {"datasource": options.coverage, "srs": srs, "where": options.where}, base_path=os.getcwd() ) else: seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs) if not supports_tiled_access(mgr): print("WARN: grids are incompatible. needs to scale/reproject tiles for export.", file=sys.stderr) md = dict(name="export", cache_name="cache", grid_name=options.grid, dest=options.dest) task = SeedTask(md, mgr, levels, None, seed_coverage) print(format_export_task(task, custom_grid=custom_grid)) logger = ProgressLog(verbose=True, silent=False) try: seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency) except KeyboardInterrupt: print("stopping...", file=sys.stderr) sys.exit(2)
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option("-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option("--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage") coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): if options.show_all or options.grid_name: grids = proxy_configuration.grids else: caches = proxy_configuration.caches grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict((key.lower(), value) for (key, value) in iteritems(grids)) if not grids.get(options.grid_name, False): print('grid not found: %s' % (options.grid_name,)) sys.exit(1) coverage = None if options.coverage and options.seed_config: try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError as e: print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print('Old seed configuration format not supported') sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print('--coverage and --seed-conf can only be used together') sys.exit(1) if options.list_grids: display_grids_list(grids) elif options.grid_name: display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage) else: display_grids(grids, coverage=coverage)
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 progress = None if options.continue_seed or options.progress_file: if options.progress_file: progress_file = options.progress_file else: progress_file = '.mapproxy_seed_progress' progress = ProgressStore(progress_file, continue_seed=options.continue_seed) try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def config_command(args): parser = optparse.OptionParser("usage: %prog autoconfig [options]") parser.add_option( '--capabilities', help="URL or filename of WMS 1.1.1/1.3.0 capabilities document") parser.add_option('--output', help="filename for created MapProxy config [default: -]", default="-") parser.add_option('--output-seed', help="filename for created seeding config") parser.add_option('--base', help='base config to include in created MapProxy config') parser.add_option( '--overwrite', help='YAML file with overwrites for the created MapProxy config') parser.add_option( '--overwrite-seed', help='YAML file with overwrites for the created seeding config') parser.add_option('--force', default=False, action='store_true', help="overwrite existing files") options, args = parser.parse_args(args) if not options.capabilities: parser.print_help() print("\nERROR: --capabilities required", file=sys.stderr) return 2 if not options.output and not options.output_seed: parser.print_help() print("\nERROR: --output and/or --output-seed required", file=sys.stderr) return 2 if not options.force: if options.output and options.output != '-' and os.path.exists( options.output): print("\nERROR: %s already exists, use --force to overwrite" % options.output, file=sys.stderr) return 2 if options.output_seed and options.output_seed != '-' and os.path.exists( options.output_seed): print("\nERROR: %s already exists, use --force to overwrite" % options.output_seed, file=sys.stderr) return 2 log = logging.getLogger('mapproxy_conf_cmd') log.addHandler(logging.StreamHandler()) setup_logging(logging.WARNING) srs_grids = {} if options.base: base = load_configuration(options.base) for name, grid_conf in iteritems(base.grids): if name.startswith('GLOBAL_'): continue srs_grids[grid_conf.tile_grid().srs.srs_code] = name cap_doc = options.capabilities if cap_doc.startswith(('http://', 'https://')): cap_doc = download_capabilities(options.capabilities).read() else: cap_doc = open(cap_doc, 'rb').read() try: cap = parse_capabilities(BytesIO(cap_doc)) except (xml.etree.ElementTree.ParseError, ValueError) as ex: print(ex, file=sys.stderr) print(cap_doc[:1000] + ('...' if len(cap_doc) > 1000 else ''), file=sys.stderr) return 3 overwrite = None if options.overwrite: with open(options.overwrite, 'rb') as f: overwrite = yaml.safe_load(f) overwrite_seed = None if options.overwrite_seed: with open(options.overwrite_seed, 'rb') as f: overwrite_seed = yaml.safe_load(f) conf = {} if options.base: conf['base'] = os.path.abspath(options.base) conf['services'] = {'wms': {'md': {'title': cap.metadata()['title']}}} if overwrite: conf['services'] = update_config(conf['services'], overwrite.pop('service', {})) conf['sources'] = sources(cap) if overwrite: conf['sources'] = update_config(conf['sources'], overwrite.pop('sources', {})) conf['caches'] = caches(cap, conf['sources'], srs_grids=srs_grids) if overwrite: conf['caches'] = update_config(conf['caches'], overwrite.pop('caches', {})) conf['layers'] = layers(cap, conf['caches']) if overwrite: conf['layers'] = update_config(conf['layers'], overwrite.pop('layers', {})) if overwrite: conf = update_config(conf, overwrite) seed_conf = {} seed_conf['seeds'], seed_conf['cleanups'] = seeds(cap, conf['caches']) if overwrite_seed: seed_conf = update_config(seed_conf, overwrite_seed) if options.output: with file_or_stdout(options.output) as f: write_header(f, options.capabilities) yaml.dump(conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper) if options.output_seed: with file_or_stdout(options.output_seed) as f: write_header(f, options.capabilities) yaml.dump(seed_conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper) return 0
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option( "-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option( "--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option( "-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group( "Approximate the number of tiles within a given coverage") coverage_group.add_option( "-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option( "-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): if options.show_all or options.grid_name: grids = proxy_configuration.grids else: caches = proxy_configuration.caches grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict( (key.lower(), value) for (key, value) in iteritems(grids)) if not grids.get(options.grid_name, False): print('grid not found: %s' % (options.grid_name, )) sys.exit(1) coverage = None if options.coverage and options.seed_config: try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError as e: print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print('Old seed configuration format not supported') sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print('--coverage and --seed-conf can only be used together') sys.exit(1) if options.list_grids: display_grids_list(grids) elif options.grid_name: display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage) else: display_grids(grids, coverage=coverage)
def export_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration") parser.add_option("--source", dest="source", help="source to export (source or cache)") parser.add_option("--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string") parser.add_option("--dest", help="destination of the export (directory or filename)") parser.add_option("--type", help="type of the export format") parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10") parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles", action='store_true', default=False, help="if missing tiles should be fetched from the sources") parser.add_option("--force", action='store_true', default=False, help="overwrite/append to existing --dest files/directories") parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes") parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource") parser.add_option("--srs", help="the SRS of the coverage") parser.add_option("--where", help="filter for OGR coverages") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels'] for required in required_options: if not getattr(options, required): print >>sys.stderr, 'ERROR: missing required option --%s' % required.replace('_', '-') parser.print_help() sys.exit(1) try: conf = load_configuration(options.mapproxy_conf) except IOError, e: print >>sys.stderr, 'ERROR: ', "%s: '%s'" % (e.strerror, e.filename) sys.exit(2)
def export_command(args=None): parser = optparse.OptionParser("%prog export [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration") parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, help="reduce number of messages to stdout, repeat to disable progress output") parser.add_option("--source", dest="source", help="source to export (source or cache)") parser.add_option("--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string") parser.add_option("--dest", help="destination of the export (directory or filename)") parser.add_option("--type", help="type of the export format") parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10") parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles", action='store_true', default=False, help="if missing tiles should be fetched from the sources") parser.add_option("--force", action='store_true', default=False, help="overwrite/append to existing --dest files/directories") parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes") parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource") parser.add_option("--srs", help="the SRS of the coverage") parser.add_option("--where", help="filter for OGR coverages") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels'] for required in required_options: if not getattr(options, required): print('ERROR: missing required option --%s' % required.replace('_', '-'), file=sys.stderr) parser.print_help() sys.exit(1) try: conf = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if '=' in options.grid: try: grid_conf = parse_grid_definition(options.grid) except ValidationError as ex: print('ERROR: invalid grid configuration', file=sys.stderr) for error in ex.errors: print(' ', error, file=sys.stderr) sys.exit(2) except ValueError: print('ERROR: invalid grid configuration', file=sys.stderr) sys.exit(2) options.grid = 'tmp_mapproxy_export_grid' grid_conf['name'] = options.grid custom_grid = True conf.grids[options.grid] = GridConfiguration(grid_conf, conf) else: custom_grid = False if os.path.exists(options.dest) and not options.force: print('ERROR: destination exists, remove first or use --force', file=sys.stderr) sys.exit(2) cache_conf = { 'name': 'export', 'grids': [options.grid], 'sources': [options.source], } if options.type == 'mbtile': cache_conf['cache'] = { 'type': 'mbtiles', 'filename': options.dest, } elif options.type == 'sqlite': cache_conf['cache'] = { 'type': 'sqlite', 'directory': options.dest, } elif options.type == 'geopackage': cache_conf['cache'] = { 'type': 'geopackage', 'filename': options.dest, } elif options.type == 'compact-v1': cache_conf['cache'] = { 'type': 'compact', 'version': 1, 'directory': options.dest, } elif options.type == 'compact-v2': cache_conf['cache'] = { 'type': 'compact', 'version': 2, 'directory': options.dest, } elif options.type in ('tc', 'mapproxy'): cache_conf['cache'] = { 'type': 'file', 'directory': options.dest, } elif options.type == 'arcgis': cache_conf['cache'] = { 'type': 'file', 'directory_layout': 'arcgis', 'directory': options.dest, } elif options.type in ('tms', None): # default cache_conf['cache'] = { 'type': 'file', 'directory_layout': 'tms', 'directory': options.dest, } else: print('ERROR: unsupported --type %s' % (options.type, ), file=sys.stderr) sys.exit(2) if not options.fetch_missing_tiles: for source in conf.sources.values(): source.conf['seed_only'] = True tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0] levels = parse_levels(options.levels) if levels[-1] >= tile_grid.levels: print('ERROR: destination grid only has %d levels' % tile_grid.levels, file=sys.stderr) sys.exit(2) if options.srs: srs = SRS(options.srs) else: srs = tile_grid.srs if options.coverage: seed_coverage = load_coverage( {'datasource': options.coverage, 'srs': srs, 'where': options.where}, base_path=os.getcwd()) else: seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs) if not supports_tiled_access(mgr): print('WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr) md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest) task = SeedTask(md, mgr, levels, 1, seed_coverage) print(format_export_task(task, custom_grid=custom_grid)) logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2) try: seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency) except KeyboardInterrupt: print('stopping...', file=sys.stderr) sys.exit(2)
def load_services(conf_file): conf = load_configuration(conf_file) return conf.configured_services()
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) if options.duration: # calls with --duration are handled in call_with_duration sys.exit(self.call_with_duration(options, args)) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 progress = None if options.continue_seed or options.progress_file: if not options.progress_file: options.progress_file = '.mapproxy_seed_progress' progress = ProgressStore(options.progress_file, continue_seed=options.continue_seed) if options.reseed_file: if not os.path.exists(options.reseed_file): # create --reseed-file if missing with open(options.reseed_file, 'w'): pass else: if progress and not os.path.exists(options.progress_file): # we have an existing --reseed-file but no --progress-file # meaning the last seed call was completed if options.reseed_interval and ( os.path.getmtime(options.reseed_file) > (time.time() - options.reseed_interval)): print("no need for re-seeding") sys.exit(1) os.utime(options.reseed_file, (time.time(), time.time())) with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels, skip_uncached=options.skip_uncached) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def load_mapproxy_config(mapproxy_config_file): try: mapproxy_config = load_configuration(mapproxy_conf=mapproxy_config_file) except ConfigurationError, e: log.fatal(e) raise
def export_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration") parser.add_option("--source", dest="source", help="source to export (source or cache)") parser.add_option( "--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string" ) parser.add_option("--dest", help="destination of the export (directory or filename)") parser.add_option("--type", help="type of the export format") parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10") parser.add_option( "--fetch-missing-tiles", dest="fetch_missing_tiles", action="store_true", default=False, help="if missing tiles should be fetched from the sources", ) parser.add_option( "--force", action="store_true", default=False, help="overwrite/append to existing --dest files/directories" ) parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output") parser.add_option( "-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes" ) parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource") parser.add_option("--srs", help="the SRS of the coverage") parser.add_option("--where", help="filter for OGR coverages") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] required_options = ["mapproxy_conf", "grid", "source", "dest", "levels"] for required in required_options: if not getattr(options, required): print >> sys.stderr, "ERROR: missing required option --%s" % required.replace("_", "-") parser.print_help() sys.exit(1) try: conf = load_configuration(options.mapproxy_conf) except IOError, e: print >> sys.stderr, "ERROR: ", "%s: '%s'" % (e.strerror, e.filename) sys.exit(2)
def export_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration") parser.add_option( "-q", "--quiet", action="count", dest="quiet", default=0, help= "reduce number of messages to stdout, repeat to disable progress output" ) parser.add_option("--source", dest="source", help="source to export (source or cache)") parser.add_option( "--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string") parser.add_option("--dest", help="destination of the export (directory or filename)") parser.add_option("--type", help="type of the export format") parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10") parser.add_option( "--fetch-missing-tiles", dest="fetch_missing_tiles", action='store_true', default=False, help="if missing tiles should be fetched from the sources") parser.add_option( "--force", action='store_true', default=False, help="overwrite/append to existing --dest files/directories") parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes") parser.add_option( "--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource") parser.add_option("--srs", help="the SRS of the coverage") parser.add_option("--where", help="filter for OGR coverages") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels'] for required in required_options: if not getattr(options, required): print('ERROR: missing required option --%s' % required.replace('_', '-'), file=sys.stderr) parser.print_help() sys.exit(1) try: conf = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if '=' in options.grid: try: grid_conf = parse_grid_definition(options.grid) except ValidationError as ex: print('ERROR: invalid grid configuration', file=sys.stderr) for error in ex.errors: print(' ', error, file=sys.stderr) sys.exit(2) except ValueError: print('ERROR: invalid grid configuration', file=sys.stderr) sys.exit(2) options.grid = 'tmp_mapproxy_export_grid' grid_conf['name'] = options.grid custom_grid = True conf.grids[options.grid] = GridConfiguration(grid_conf, conf) else: custom_grid = False if os.path.exists(options.dest) and not options.force: print('ERROR: destination exists, remove first or use --force', file=sys.stderr) sys.exit(2) cache_conf = { 'name': 'export', 'grids': [options.grid], 'sources': [options.source], } if options.type == 'mbtile': cache_conf['cache'] = { 'type': 'mbtiles', 'filename': options.dest, } elif options.type == 'sqlite': cache_conf['cache'] = { 'type': 'sqlite', 'directory': options.dest, } elif options.type == 'geopackage': cache_conf['cache'] = { 'type': 'geopackage', 'filename': options.dest, } elif options.type == 'compact-v1': cache_conf['cache'] = { 'type': 'compact', 'version': 1, 'directory': options.dest, } elif options.type == 'compact-v2': cache_conf['cache'] = { 'type': 'compact', 'version': 2, 'directory': options.dest, } elif options.type in ('tc', 'mapproxy'): cache_conf['cache'] = { 'type': 'file', 'directory': options.dest, } elif options.type == 'arcgis': cache_conf['cache'] = { 'type': 'file', 'directory_layout': 'arcgis', 'directory': options.dest, } elif options.type in ('tms', None): # default cache_conf['cache'] = { 'type': 'file', 'directory_layout': 'tms', 'directory': options.dest, } else: print('ERROR: unsupported --type %s' % (options.type, ), file=sys.stderr) sys.exit(2) if not options.fetch_missing_tiles: for source in conf.sources.values(): source.conf['seed_only'] = True tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0] levels = parse_levels(options.levels) if levels[-1] >= tile_grid.levels: print('ERROR: destination grid only has %d levels' % tile_grid.levels, file=sys.stderr) sys.exit(2) if options.srs: srs = SRS(options.srs) else: srs = tile_grid.srs if options.coverage: seed_coverage = load_coverage( { 'datasource': options.coverage, 'srs': srs, 'where': options.where }, base_path=os.getcwd()) else: seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs) if not supports_tiled_access(mgr): print( 'WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr) md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest) task = SeedTask(md, mgr, levels, 1, seed_coverage) print(format_export_task(task, custom_grid=custom_grid)) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) try: seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency) except KeyboardInterrupt: print('stopping...', file=sys.stderr) sys.exit(2)
def config_command(args): parser = optparse.OptionParser("usage: %prog autoconfig [options]") parser.add_option("--capabilities", help="URL or filename of WMS 1.1.1/1.3.0 capabilities document") parser.add_option("--output", help="filename for created MapProxy config [default: -]", default="-") parser.add_option("--output-seed", help="filename for created seeding config") parser.add_option("--base", help="base config to include in created MapProxy config") parser.add_option("--overwrite", help="YAML file with overwrites for the created MapProxy config") parser.add_option("--overwrite-seed", help="YAML file with overwrites for the created seeding config") parser.add_option("--force", default=False, action="store_true", help="overwrite existing files") options, args = parser.parse_args(args) if not options.capabilities: parser.print_help() print("\nERROR: --capabilities required", file=sys.stderr) return 2 if not options.output and not options.output_seed: parser.print_help() print("\nERROR: --output and/or --output-seed required", file=sys.stderr) return 2 if not options.force: if options.output and options.output != "-" and os.path.exists(options.output): print("\nERROR: %s already exists, use --force to overwrite" % options.output, file=sys.stderr) return 2 if options.output_seed and options.output_seed != "-" and os.path.exists(options.output_seed): print("\nERROR: %s already exists, use --force to overwrite" % options.output_seed, file=sys.stderr) return 2 log = logging.getLogger("mapproxy_conf_cmd") log.addHandler(logging.StreamHandler()) setup_logging(logging.WARNING) srs_grids = {} if options.base: base = load_configuration(options.base) for name, grid_conf in iteritems(base.grids): if name.startswith("GLOBAL_"): continue srs_grids[grid_conf.tile_grid().srs.srs_code] = name cap_doc = options.capabilities if cap_doc.startswith(("http://", "https://")): cap_doc = download_capabilities(options.capabilities).read() else: cap_doc = open(cap_doc, "rb").read() try: cap = parse_capabilities(BytesIO(cap_doc)) except (xml.etree.ElementTree.ParseError, ValueError) as ex: print(ex, file=sys.stderr) print(cap_doc[:1000] + ("..." if len(cap_doc) > 1000 else ""), file=sys.stderr) return 3 overwrite = None if options.overwrite: with open(options.overwrite, "rb") as f: overwrite = yaml.load(f) overwrite_seed = None if options.overwrite_seed: with open(options.overwrite_seed, "rb") as f: overwrite_seed = yaml.load(f) conf = {} if options.base: conf["base"] = os.path.abspath(options.base) conf["services"] = {"wms": {"md": {"title": cap.metadata()["title"]}}} if overwrite: conf["services"] = update_config(conf["services"], overwrite.pop("service", {})) conf["sources"] = sources(cap) if overwrite: conf["sources"] = update_config(conf["sources"], overwrite.pop("sources", {})) conf["caches"] = caches(cap, conf["sources"], srs_grids=srs_grids) if overwrite: conf["caches"] = update_config(conf["caches"], overwrite.pop("caches", {})) conf["layers"] = layers(cap, conf["caches"]) if overwrite: conf["layers"] = update_config(conf["layers"], overwrite.pop("layers", {})) if overwrite: conf = update_config(conf, overwrite) seed_conf = {} seed_conf["seeds"], seed_conf["cleanups"] = seeds(cap, conf["caches"]) if overwrite_seed: seed_conf = update_config(seed_conf, overwrite_seed) if options.output: with file_or_stdout(options.output) as f: write_header(f, options.capabilities) yaml.dump(conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper) if options.output_seed: with file_or_stdout(options.output_seed) as f: write_header(f, options.capabilities) yaml.dump(seed_conf, f, default_flow_style=False, Dumper=MapProxyYAMLDumper) return 0
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error('missing seed_conf file as last argument or --seed-conf option') else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 progress = None if options.continue_seed or options.progress_file: if options.progress_file: progress_file = options.progress_file else: progress_file = '.mapproxy_seed_progress' progress = ProgressStore(progress_file, continue_seed=options.continue_seed) try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print('Start seeding process (%d task%s)' % ( len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % ( len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2) cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def main(): parser = optparse.OptionParser() parser.add_option("-f", "--mapproxy-conf", dest="conf_file", default='mapproxy.yaml', help="MapProxy configuration") parser.add_option("--renderer", default=None, type=int, help="Number of render processes.") parser.add_option("--max-seed-renderer", default=None, type=int, help="Maximum --renderer used for seeding.") parser.add_option("--pidfile") parser.add_option("--log-config", dest="log_config_file") parser.add_option("--verbose", action="store_true", default=False) options, args = parser.parse_args() init_logging(options.log_config_file, options.verbose) conf = load_configuration(options.conf_file, renderd=True) broker_address = conf.globals.renderd_address if not broker_address: fatal('mapproxy config (%s) does not define renderd address' % (options.conf_file)) broker_address = broker_address.replace('localhost', '127.0.0.1') broker_port = int(broker_address.rsplit(':', 1)[1]) # TODO tile_managers = {} with conf: for mapproxy_cache in conf.caches.itervalues(): for tile_grid_, extent_, tile_manager in mapproxy_cache.caches(): tile_manager._expire_timestamp = 2**32 # future ~2106 tile_managers[tile_manager.identifier] = tile_manager if options.renderer is None: pool_size = multiprocessing.cpu_count() else: pool_size = options.renderer if options.max_seed_renderer is None: max_seed_renderer = pool_size else: max_seed_renderer = min(options.max_seed_renderer, pool_size) non_seed_renderer = pool_size - max_seed_renderer process_priorities = [50] * non_seed_renderer + [0] * max_seed_renderer log.debug('starting %d processes with the following min priorities: %r', pool_size, process_priorities) def worker_factory(in_queue, out_queue): return SeedWorker(tile_managers, conf.base_config, in_queue=in_queue, out_queue=out_queue) worker_pool = WorkerPool(worker_factory, pool_size=pool_size) task_queue = RenderQueue(process_priorities) if options.pidfile: with open(options.pidfile, 'w') as f: f.write(str(os.getpid())) def remove_pid(): os.unlink(options.pidfile) atexit.register(remove_pid) try: broker = Broker(worker_pool, task_queue) broker.start() app = RenderdApp(broker) server = CherryPyWSGIServer( ('127.0.0.1', broker_port), app, numthreads=64, request_queue_size=256, ) server.start() except (KeyboardInterrupt, SystemExit): print >> sys.stderr, 'exiting...' if server: server.stop() return 0 except Exception: log.fatal('fatal error, terminating', exc_info=True) raise
def defrag_command(args=None): parser = optparse.OptionParser( "%prog defrag-compact [options] -f mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option( "--min-percent", type=float, default=10.0, help= "Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)" ) parser.add_option( "--min-mb", type=float, default=1.0, help= "Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)" ) parser.add_option("--dry-run", "-n", action="store_true", help="Do not de-fragment, only print output") parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...', help="only defragment the named caches") from mapproxy.script.util import setup_logging import logging setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s") if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: parser.print_help() sys.exit(1) try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): available_caches = OrderedDict() for name, cache_conf in proxy_configuration.caches.items(): for grid, extent, tile_mgr in cache_conf.caches(): if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)): available_caches.setdefault(name, []).append(tile_mgr.cache) if options.cache_names: defrag_caches = options.cache_names.split(',') missing = set(defrag_caches).difference(available_caches.keys()) if missing: print('unknown caches: %s' % (', '.join(missing), )) print('available compact caches: %s' % (', '.join(available_caches.keys()), )) sys.exit(1) else: defrag_caches = None for name, caches in available_caches.items(): if defrag_caches and name not in defrag_caches: continue for cache in caches: logger = DefragLog(name) defrag_compact_cache( cache, min_percent=options.min_percent / 100, min_bytes=options.min_mb * 1024 * 1024, dry_run=options.dry_run, log_progress=logger, )
def defrag_command(args=None): parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option("--min-percent", type=float, default=10.0, help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)") parser.add_option("--min-mb", type=float, default=1.0, help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)") parser.add_option("--dry-run", "-n", action="store_true", help="Do not de-fragment, only print output") parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...', help="only defragment the named caches") from mapproxy.script.util import setup_logging import logging setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s") if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: parser.print_help() sys.exit(1) try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): available_caches = OrderedDict() for name, cache_conf in proxy_configuration.caches.items(): for grid, extent, tile_mgr in cache_conf.caches(): if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)): available_caches.setdefault(name, []).append(tile_mgr.cache) if options.cache_names: defrag_caches = options.cache_names.split(',') missing = set(defrag_caches).difference(available_caches.keys()) if missing: print('unknown caches: %s' % (', '.join(missing), )) print('available compact caches: %s' % (', '.join(available_caches.keys()), )) sys.exit(1) else: defrag_caches = None for name, caches in available_caches.items(): if defrag_caches and name not in defrag_caches: continue for cache in caches: logger = DefragLog(name) defrag_compact_cache(cache, min_percent=options.min_percent/100, min_bytes=options.min_mb*1024*1024, dry_run=options.dry_run, log_progress=logger, )
def main(): parser = optparse.OptionParser() parser.add_option("-f", "--mapproxy-conf", dest="conf_file", default='mapproxy.yaml', help="MapProxy configuration") parser.add_option("--renderer", default=None, type=int, help="Number of render processes.") parser.add_option("--max-seed-renderer", default=None, type=int, help="Maximum --renderer used for seeding.") parser.add_option("--pidfile") parser.add_option("--log-config", dest="log_config_file") parser.add_option("--verbose", action="store_true", default=False) options, args = parser.parse_args() init_logging(options.log_config_file, options.verbose) conf = load_configuration(options.conf_file, renderd=True) broker_address = conf.globals.renderd_address if not broker_address: fatal('mapproxy config (%s) does not define renderd address' % ( options.conf_file)) broker_address = broker_address.replace('localhost', '127.0.0.1') broker_port = int(broker_address.rsplit(':', 1)[1]) # TODO tile_managers = {} with conf: for mapproxy_cache in conf.caches.itervalues(): for tile_grid_, extent_, tile_manager in mapproxy_cache.caches(): tile_manager._expire_timestamp = 2**32 # future ~2106 tile_managers[tile_manager.identifier] = tile_manager if options.renderer is None: pool_size = multiprocessing.cpu_count() else: pool_size = options.renderer if options.max_seed_renderer is None: max_seed_renderer = pool_size else: max_seed_renderer = min(options.max_seed_renderer, pool_size) non_seed_renderer = pool_size - max_seed_renderer process_priorities = [50] * non_seed_renderer + [0] * max_seed_renderer log.debug('starting %d processes with the following min priorities: %r', pool_size, process_priorities) def worker_factory(in_queue, out_queue): return SeedWorker(tile_managers, conf.base_config, in_queue=in_queue, out_queue=out_queue) worker_pool = WorkerPool(worker_factory, pool_size=pool_size) task_queue = RenderQueue(process_priorities) if options.pidfile: with open(options.pidfile, 'w') as f: f.write(str(os.getpid())) def remove_pid(): os.unlink(options.pidfile) atexit.register(remove_pid) try: broker = Broker(worker_pool, task_queue) broker.start() app = RenderdApp(broker) server = CherryPyWSGIServer( ('127.0.0.1', broker_port), app, numthreads=64, request_queue_size=256, ) server.start() except (KeyboardInterrupt, SystemExit): print >>sys.stderr, 'exiting...' if server: server.stop() return 0 except Exception: log.fatal('fatal error, terminating', exc_info=True) raise