Exemple #1
0
    def coverage(self, name):
        coverage_conf = (self.conf.get('coverages') or {}).get(name)
        if coverage_conf is None:
            raise SeedConfigurationError('coverage %s not found. available coverages: %s' % (
                name, ','.join((self.conf.get('coverages') or {}).keys())))

        return load_coverage(coverage_conf)
Exemple #2
0
    def coverage(self, name):
        coverage_conf = (self.conf.get('coverages') or {}).get(name)
        if coverage_conf is None:
            raise SeedConfigurationError('coverage %s not found. available coverages: %s' % (
                name, ','.join((self.conf.get('coverages') or {}).keys())))

        try:
            coverage = load_coverage(coverage_conf)
        except CoverageReadError as ex:
            raise SeedConfigurationError("can't load coverage '%s'. %s" % (name, ex))
        except GeometryError as ex:
            raise SeedConfigurationError("invalid geometry in coverage '%s'. %s" % (name, ex))
        except EmptyGeometryError as ex:
            raise EmptyCoverageError("coverage '%s' contains no geometries. %s" % (name, ex))

        # without extend we have an empty coverage
        if not coverage.extent.llbbox:
            raise EmptyCoverageError("coverage '%s' contains no geometries." % name)
        return coverage
Exemple #3
0
    def _init_tasks(self):
        for cache_name, options in self.conf['seeds'].iteritems():
            remove_before = None
            if 'remove_before' in options:
                remove_before = before_timestamp_from_options(options['remove_before'])
            try:
                caches = self.mapproxy_conf.caches[cache_name].caches()
            except KeyError:
                print >>sys.stderr, 'error: cache %s not found. available caches: %s' % (
                    cache_name, ','.join(self.mapproxy_conf.caches.keys()))
                return
            caches = dict((grid, tile_mgr) for grid, extent, tile_mgr in caches)
            for view in options['views']:
                view_conf = self.conf['views'][view]
                coverage = load_coverage(view_conf)

                cache_srs = view_conf.get('srs', None)
                if cache_srs is not None:
                    cache_srs = [SRS(s) for s in cache_srs]

                level = view_conf.get('level', None)
                assert len(level) == 2

                for grid, tile_mgr in caches.iteritems():
                    if cache_srs and grid.srs not in cache_srs: continue
                    md = dict(name=view, cache_name=cache_name, grid_name=self.grids[grid])
                    levels = range(level[0], level[1]+1)
                    if coverage:
                        if isinstance(coverage, GeomCoverage) and coverage.geom.is_empty:
                            continue
                        seed_coverage = coverage.transform_to(grid.srs)
                    else:
                        seed_coverage = BBOXCoverage(grid.bbox, grid.srs)

                    self.seed_tasks.append(SeedTask(md, tile_mgr, levels, remove_before, seed_coverage))

                    if remove_before:
                        levels = range(grid.levels)
                        complete_extent = bool(coverage)
                        self.cleanup_tasks.append(CleanupTask(md, tile_mgr, levels, remove_before,
                            seed_coverage, complete_extent=complete_extent))
Exemple #4
0
def export_command(args=None):
    parser = optparse.OptionParser("%prog export [options] mapproxy_conf")
    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
        help="MapProxy configuration")

    parser.add_option("-q", "--quiet",
                      action="count", dest="quiet", default=0,
                      help="reduce number of messages to stdout, repeat to disable progress output")

    parser.add_option("--source", dest="source",
        help="source to export (source or cache)")

    parser.add_option("--grid",
        help="grid for export. either the name of an existing grid or "
        "the grid definition as a string")

    parser.add_option("--dest",
        help="destination of the export (directory or filename)")

    parser.add_option("--type",
        help="type of the export format")

    parser.add_option("--levels",
        help="levels to export: e.g 1,2,3 or 1..10")

    parser.add_option("--fetch-missing-tiles", dest="fetch_missing_tiles",
        action='store_true', default=False,
        help="if missing tiles should be fetched from the sources")

    parser.add_option("--force",
        action='store_true', default=False,
        help="overwrite/append to existing --dest files/directories")

    parser.add_option("-n", "--dry-run",
        action="store_true", default=False,
        help="do not export, just print output")

    parser.add_option("-c", "--concurrency", type="int",
        dest="concurrency", default=1,
        help="number of parallel export processes")

    parser.add_option("--coverage",
        help="the coverage for the export as a BBOX string, WKT file "
        "or OGR datasource")
    parser.add_option("--srs",
        help="the SRS of the coverage")
    parser.add_option("--where",
        help="filter for OGR coverages")

    from mapproxy.script.util import setup_logging
    import logging
    setup_logging(logging.WARN)

    if args:
        args = args[1:] # remove script name

    (options, args) = parser.parse_args(args)

    if not options.mapproxy_conf:
        if len(args) != 1:
            parser.print_help()
            sys.exit(1)
        else:
            options.mapproxy_conf = args[0]

    required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
    for required in required_options:
        if not getattr(options, required):
            print('ERROR: missing required option --%s' % required.replace('_', '-'), file=sys.stderr)
            parser.print_help()
            sys.exit(1)

    try:
        conf = load_configuration(options.mapproxy_conf)
    except IOError as e:
        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
        sys.exit(2)
    except ConfigurationError as e:
        print(e, file=sys.stderr)
        print('ERROR: invalid configuration (see above)', file=sys.stderr)
        sys.exit(2)


    if '=' in options.grid:
        try:
            grid_conf = parse_grid_definition(options.grid)
        except ValidationError as ex:
            print('ERROR: invalid grid configuration', file=sys.stderr)
            for error in ex.errors:
                print(' ', error, file=sys.stderr)
            sys.exit(2)
        except ValueError:
            print('ERROR: invalid grid configuration', file=sys.stderr)
            sys.exit(2)
        options.grid = 'tmp_mapproxy_export_grid'
        grid_conf['name'] = options.grid
        custom_grid = True
        conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
    else:
        custom_grid = False

    if os.path.exists(options.dest) and not options.force:
        print('ERROR: destination exists, remove first or use --force', file=sys.stderr)
        sys.exit(2)


    cache_conf = {
        'name': 'export',
        'grids': [options.grid],
        'sources': [options.source],
    }
    if options.type == 'mbtile':
        cache_conf['cache'] = {
            'type': 'mbtiles',
            'filename': options.dest,
        }
    elif options.type == 'sqlite':
        cache_conf['cache'] = {
            'type': 'sqlite',
            'directory': options.dest,
        }
    elif options.type == 'geopackage':
        cache_conf['cache'] = {
            'type': 'geopackage',
            'filename': options.dest,
        }
    elif options.type == 'compact-v1':
        cache_conf['cache'] = {
            'type': 'compact',
            'version': 1,
            'directory': options.dest,
        }
    elif options.type == 'compact-v2':
        cache_conf['cache'] = {
            'type': 'compact',
            'version': 2,
            'directory': options.dest,
        }
    elif options.type in ('tc', 'mapproxy'):
        cache_conf['cache'] = {
            'type': 'file',
            'directory': options.dest,
        }
    elif options.type == 'arcgis':
        cache_conf['cache'] = {
            'type': 'file',
            'directory_layout': 'arcgis',
            'directory': options.dest,
        }
    elif options.type in ('tms', None): # default
        cache_conf['cache'] = {
            'type': 'file',
            'directory_layout': 'tms',
            'directory': options.dest,
        }
    else:
        print('ERROR: unsupported --type %s' % (options.type, ), file=sys.stderr)
        sys.exit(2)

    if not options.fetch_missing_tiles:
        for source in conf.sources.values():
            source.conf['seed_only'] = True

    tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]


    levels = parse_levels(options.levels)
    if levels[-1] >= tile_grid.levels:
        print('ERROR: destination grid only has %d levels' % tile_grid.levels, file=sys.stderr)
        sys.exit(2)

    if options.srs:
        srs = SRS(options.srs)
    else:
        srs = tile_grid.srs

    if options.coverage:
        seed_coverage = load_coverage(
            {'datasource': options.coverage, 'srs': srs, 'where': options.where},
            base_path=os.getcwd())
    else:
        seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)

    if not supports_tiled_access(mgr):
        print('WARN: grids are incompatible. needs to scale/reproject tiles for export.', file=sys.stderr)

    md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
    task = SeedTask(md, mgr, levels, 1, seed_coverage)

    print(format_export_task(task, custom_grid=custom_grid))

    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
    try:
        seed_task(task, progress_logger=logger, dry_run=options.dry_run,
             concurrency=options.concurrency)
    except KeyboardInterrupt:
        print('stopping...', file=sys.stderr)
        sys.exit(2)
Exemple #5
0
def export_command(args=None):
    parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration")

    parser.add_option("--source", dest="source", help="source to export (source or cache)")

    parser.add_option(
        "--grid", help="grid for export. either the name of an existing grid or " "the grid definition as a string"
    )

    parser.add_option("--dest", help="destination of the export (directory or filename)")

    parser.add_option("--type", help="type of the export format")

    parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10")

    parser.add_option(
        "--fetch-missing-tiles",
        dest="fetch_missing_tiles",
        action="store_true",
        default=False,
        help="if missing tiles should be fetched from the sources",
    )

    parser.add_option(
        "--force", action="store_true", default=False, help="overwrite/append to existing --dest files/directories"
    )

    parser.add_option("-n", "--dry-run", action="store_true", default=False, help="do not export, just print output")

    parser.add_option(
        "-c", "--concurrency", type="int", dest="concurrency", default=1, help="number of parallel export processes"
    )

    parser.add_option("--coverage", help="the coverage for the export as a BBOX string, WKT file " "or OGR datasource")
    parser.add_option("--srs", help="the SRS of the coverage")
    parser.add_option("--where", help="filter for OGR coverages")

    from mapproxy.script.util import setup_logging
    import logging

    setup_logging(logging.WARN)

    if args:
        args = args[1:]  # remove script name

    (options, args) = parser.parse_args(args)

    if not options.mapproxy_conf:
        if len(args) != 1:
            parser.print_help()
            sys.exit(1)
        else:
            options.mapproxy_conf = args[0]

    required_options = ["mapproxy_conf", "grid", "source", "dest", "levels"]
    for required in required_options:
        if not getattr(options, required):
            print("ERROR: missing required option --%s" % required.replace("_", "-"), file=sys.stderr)
            parser.print_help()
            sys.exit(1)

    try:
        conf = load_configuration(options.mapproxy_conf)
    except IOError as e:
        print("ERROR: ", "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
        sys.exit(2)
    except ConfigurationError as e:
        print(e, file=sys.stderr)
        print("ERROR: invalid configuration (see above)", file=sys.stderr)
        sys.exit(2)

    if "=" in options.grid:
        try:
            grid_conf = parse_grid_definition(options.grid)
        except ValidationError as ex:
            print("ERROR: invalid grid configuration", file=sys.stderr)
            for error in ex.errors:
                print(" ", error, file=sys.stderr)
            sys.exit(2)
        except ValueError:
            print("ERROR: invalid grid configuration", file=sys.stderr)
            sys.exit(2)
        options.grid = "tmp_mapproxy_export_grid"
        grid_conf["name"] = options.grid
        custom_grid = True
        conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
    else:
        custom_grid = False

    if os.path.exists(options.dest) and not options.force:
        print("ERROR: destination exists, remove first or use --force", file=sys.stderr)
        sys.exit(2)

    cache_conf = {"name": "export", "grids": [options.grid], "sources": [options.source]}
    if options.type == "mbtile":
        cache_conf["cache"] = {"type": "mbtiles", "filename": options.dest}
    elif options.type in ("tc", "mapproxy"):
        cache_conf["cache"] = {"type": "file", "directory": options.dest}
    elif options.type in ("tms", None):  # default
        cache_conf["cache"] = {"type": "file", "directory_layout": "tms", "directory": options.dest}
    else:
        print("ERROR: unsupported --type %s" % (options.type,), file=sys.stderr)
        sys.exit(2)

    if not options.fetch_missing_tiles:
        for source in conf.sources.values():
            source.conf["seed_only"] = True

    tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]

    levels = parse_levels(options.levels)
    if levels[-1] >= tile_grid.levels:
        print("ERROR: destination grid only has %d levels" % tile_grid.levels, file=sys.stderr)
        sys.exit(2)

    if options.srs:
        srs = SRS(options.srs)
    else:
        srs = tile_grid.srs

    if options.coverage:
        seed_coverage = load_coverage(
            {"datasource": options.coverage, "srs": srs, "where": options.where}, base_path=os.getcwd()
        )
    else:
        seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)

    if not supports_tiled_access(mgr):
        print("WARN: grids are incompatible. needs to scale/reproject tiles for export.", file=sys.stderr)

    md = dict(name="export", cache_name="cache", grid_name=options.grid, dest=options.dest)
    task = SeedTask(md, mgr, levels, None, seed_coverage)

    print(format_export_task(task, custom_grid=custom_grid))

    logger = ProgressLog(verbose=True, silent=False)
    try:
        seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency)
    except KeyboardInterrupt:
        print("stopping...", file=sys.stderr)
        sys.exit(2)
Exemple #6
0
    def coverage(self, name):
        coverage_conf = self.conf.get('coverages', {}).get(name)
        if coverage_conf is None:
            raise ValueError('no coverage %s configured' % name)

        return load_coverage(coverage_conf)
Exemple #7
0
    tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]

    levels = parse_levels(options.levels)
    if levels[-1] >= tile_grid.levels:
        print >> sys.stderr, "ERROR: destination grid only has %d levels" % tile_grid.levels
        sys.exit(2)

    if options.srs:
        srs = SRS(options.srs)
    else:
        srs = tile_grid.srs

    if options.coverage:
        seed_coverage = load_coverage(
            {"datasource": options.coverage, "srs": srs, "where": options.where}, base_path=os.getcwd()
        )
    else:
        seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)

    if not supports_tiled_access(mgr):
        print >> sys.stderr, "WARN: grids are incompatible. needs to scale/reproject tiles for export."

    md = dict(name="export", cache_name="cache", grid_name=options.grid, dest=options.dest)
    task = SeedTask(md, mgr, levels, None, seed_coverage)

    print format_export_task(task, custom_grid=custom_grid)

    logger = ProgressLog(verbose=True, silent=False)
    try:
        seed_task(task, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency)
Exemple #8
0
    tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]


    levels = parse_levels(options.levels)
    if levels[-1] >= tile_grid.levels:
        print >>sys.stderr, 'ERROR: destination grid only has %d levels' % tile_grid.levels
        sys.exit(2)

    if options.srs:
        srs = SRS(options.srs)
    else:
        srs = tile_grid.srs

    if options.coverage:
        seed_coverage = load_coverage(
            {'datasource': options.coverage, 'srs': srs, 'where': options.where},
            base_path=os.getcwd())
    else:
        seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)

    if not supports_tiled_access(mgr):
        print >>sys.stderr, 'WARN: grids are incompatible. needs to scale/reproject tiles for export.'

    md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
    task = SeedTask(md, mgr, levels, None, seed_coverage)

    print format_export_task(task, custom_grid=custom_grid)

    logger = ProgressLog(verbose=True, silent=False)
    try:
        seed_task(task, progress_logger=logger, dry_run=options.dry_run,