def transform_bbox(from_srs, to_srs, bbox):
"""
Transform list of `features`. Modifies `features` in-place.
"""
bbox_coverage = BBOXCoverage(bbox, SRS(from_srs))
coverage_transformed = bbox_coverage.transform_to(SRS(to_srs))
return coverage_transformed.bbox
def transform_bbox(from_srs, to_srs, bbox):
"""
Transform list of `features`. Modifies `features` in-place.
"""
bbox_coverage = BBOXCoverage(bbox, SRS(from_srs))
coverage_transformed = bbox_coverage.transform_to(SRS(to_srs))
return coverage_transformed.bbox
def test_intersection(self):
eq_(self.coverage.intersection((15, 15, 20, 20), SRS(4326)),
BBOXCoverage((15, 15, 20, 20), SRS(4326)))
eq_(self.coverage.intersection((15, 15, 80, 20), SRS(4326)),
BBOXCoverage((15, 15, 80, 20), SRS(4326)))
eq_(self.coverage.intersection((9, 10, 20, 20), SRS(4326)),
BBOXCoverage((9, 10, 20, 20), SRS(4326)))
eq_(self.coverage.intersection((-30, 10, -8, 70), SRS(4326)),
BBOXCoverage((-10, 10, -8, 70), SRS(4326)))
eq_(self.coverage.intersection((-30, 10, -11, 70), SRS(4326)), None)
eq_(self.coverage.intersection((0, 0, 1000, 1000), SRS(900913)), None)
assert bbox_equals(
self.coverage.intersection((0, 0, 1500000, 1500000),
SRS(900913)).bbox,
(0.0, 10, 13.47472926179282, 13.352207626707813))
def seed_tasks(self):
for grid_name in self.grids:
for cache_name, cache in iteritems(self.caches):
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage is False:
coverage = False
elif self.coverage:
coverage = self.coverage.transform_to(grid.srs)
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
try:
if coverage is not False:
coverage.extent.llbbox
except TransformationError:
raise SeedConfigurationError(
'%s: coverage transformation error' % self.name)
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(range(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
if self.refresh_timestamp:
# remove everything
self.refresh_timestamp = 0
md = dict(name=self.name,
cache_name=cache_name,
grid_name=grid_name)
yield SeedTask(md, tile_manager, levels,
self.refresh_timestamp, coverage)
def test_get_map_small_with_source_extent(self):
self.source.extent = BBOXCoverage([0, 0, 90, 45], SRS(4326)).extent
result = self.layer.get_map(MapQuery((-180, -90, 180, 90), (300, 150), SRS(4326), 'png'))
eq_(self.file_cache.stored_tiles, set([(1, 0, 1)]))
# source requests one tile (no meta-tiling configured) limited to source.extent
eq_(self.client.requested, [((0, 0, 90, 45), (128, 64), (SRS(4326)))])
eq_(result.size, (300, 150))
def test_get_map_small_with_source_extent(self, source, layer, mock_file_cache, mock_wms_client):
source.extent = BBOXCoverage([0, 0, 90, 45], SRS(4326)).extent
result = layer.get_map(MapQuery((-180, -90, 180, 90), (300, 150), SRS(4326), 'png'))
assert mock_file_cache.stored_tiles == set([(1, 0, 1)])
# source requests one tile (no meta-tiling configured) limited to source.extent
assert mock_wms_client.requested == [((0, 0, 90, 45), (128, 64), (SRS(4326)))]
assert result.size == (300, 150)
def seed_tasks(self):
for grid_name in self.grids:
for cache_name, cache in self.caches.iteritems():
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage:
if isinstance(
self.coverage,
GeomCoverage) and self.coverage.geom.is_empty:
continue
coverage = self.coverage.transform_to(grid.srs)
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(xrange(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
if self.refresh_timestamp:
# remove everything
self.refresh_timestamp = 0
md = dict(name=self.name,
cache_name=cache_name,
grid_name=grid_name)
yield SeedTask(md, tile_manager, levels,
self.refresh_timestamp, coverage)
def cleanup_tasks(self):
for grid_name in self.grids:
for cache_name, cache in self.caches.iteritems():
tile_manager = cache[grid_name]
grid = self.seeding_conf.grids[grid_name]
if self.coverage:
coverage = self.coverage.transform_to(grid.srs)
complete_extent = False
else:
coverage = BBOXCoverage(grid.bbox, grid.srs)
complete_extent = True
if self.levels:
levels = self.levels.for_grid(grid)
else:
levels = list(xrange(0, grid.levels))
if not tile_manager.cache.supports_timestamp:
# for caches without timestamp support (like MBTiles)
if self.remove_timestamp is self.init_time:
# remove everything
self.remove_timestamp = 0
else:
raise SeedConfigurationError(
"cleanup does not support remove_before for '%s'"
" because cache '%s' does not support timestamps" %
(self.name, cache_name))
md = dict(name=self.name,
cache_name=cache_name,
grid_name=grid_name)
yield CleanupTask(md,
tile_manager,
levels,
self.remove_timestamp,
coverage=coverage,
complete_extent=complete_extent)
def make_bbox_task(self, bbox, srs, levels):
md = dict(name="", cache_name="", grid_name="")
coverage = BBOXCoverage(bbox, srs)
return SeedTask(md,
self.tile_mgr,
levels,
refresh_timestamp=None,
coverage=coverage)
def layer(self, mock_file_cache, source, tile_locker):
grid = TileGrid(SRS(4326), bbox=[-180, -90, 180, 90])
image_opts = ImageOptions(resampling='nearest', format='png')
tile_mgr = TileManager(grid, mock_file_cache, [source], 'png',
meta_size=[1, 1], meta_buffer=0, image_opts=image_opts,
locker=tile_locker)
layer = CacheMapLayer(tile_mgr, image_opts=default_image_opts)
layer.extent = BBOXCoverage([0, 0, 90, 45], SRS(4326)).extent
return layer
def create_seed_task(tile_mgr, coverage, levels, update_tiles=None):
if not coverage:
coverage = BBOXCoverage(tile_mgr.grid.bbox, tile_mgr.grid.srs)
if update_tiles:
refresh_timestamp = time.time()
else:
refresh_timestamp = None
return SeedTask(md={}, tile_manager=tile_mgr, levels=levels,
refresh_timestamp=refresh_timestamp, coverage=coverage)
def setup(self):
self.file_cache = MockFileCache('/dev/null', 'png', lock_dir=tmp_lock_dir)
self.grid = TileGrid(SRS(4326), bbox=[-180, -90, 180, 90])
self.client = MockWMSClient()
self.source = WMSSource(self.client)
self.image_opts = ImageOptions(resampling='nearest', format='png')
self.tile_mgr = TileManager(self.grid, self.file_cache, [self.source], 'png',
meta_size=[1, 1], meta_buffer=0, image_opts=self.image_opts)
self.layer = CacheMapLayer(self.tile_mgr, image_opts=default_image_opts)
self.layer.extent = BBOXCoverage([0, 0, 90, 45], SRS(4326)).extent
def _init_tasks(self):
for cache_name, options in iteritems(self.conf['seeds']):
remove_before = None
if 'remove_before' in options:
remove_before = before_timestamp_from_options(
options['remove_before'])
try:
caches = self.mapproxy_conf.caches[cache_name].caches()
except KeyError:
print('error: cache %s not found. available caches: %s' %
(cache_name, ','.join(self.mapproxy_conf.caches.keys())),
file=sys.stderr)
return
caches = dict(
(grid, tile_mgr) for grid, extent, tile_mgr in caches)
for view in options['views']:
view_conf = self.conf['views'][view]
coverage = load_coverage(view_conf)
cache_srs = view_conf.get('srs', None)
if cache_srs is not None:
cache_srs = [SRS(s) for s in cache_srs]
level = view_conf.get('level', None)
assert len(level) == 2
for grid, tile_mgr in iteritems(caches):
if cache_srs and grid.srs not in cache_srs: continue
md = dict(name=view,
cache_name=cache_name,
grid_name=self.grids[grid])
levels = list(range(level[0], level[1] + 1))
if coverage:
if isinstance(coverage,
GeomCoverage) and coverage.geom.is_empty:
continue
seed_coverage = coverage.transform_to(grid.srs)
else:
seed_coverage = BBOXCoverage(grid.bbox, grid.srs)
self.seed_tasks.append(
SeedTask(md, tile_mgr, levels, remove_before,
seed_coverage))
if remove_before:
levels = list(range(grid.levels))
complete_extent = bool(coverage)
self.cleanup_tasks.append(
CleanupTask(md,
tile_mgr,
levels,
remove_before,
seed_coverage,
complete_extent=complete_extent))
levels = parse_levels(options.levels)
if levels[-1] >= tile_grid.levels:
print >>sys.stderr, 'ERROR: destination grid only has %d levels' % tile_grid.levels
sys.exit(2)
if options.srs:
srs = SRS(options.srs)
else:
srs = tile_grid.srs
if options.coverage:
seed_coverage = load_coverage(
{'datasource': options.coverage, 'srs': srs, 'where': options.where},
base_path=os.getcwd())
else:
seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
if not supports_tiled_access(mgr):
print >>sys.stderr, 'WARN: grids are incompatible. needs to scale/reproject tiles for export.'
md = dict(name='export', cache_name='cache', grid_name=options.grid, dest=options.dest)
task = SeedTask(md, mgr, levels, None, seed_coverage)
print format_export_task(task, custom_grid=custom_grid)
logger = ProgressLog(verbose=True, silent=False)
try:
seed_task(task, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency)
except KeyboardInterrupt:
print >>sys.stderr, 'stopping...'
if coverage.geom:
coverage_area = coverage.geom.area
else:
coverage_area = _area_from_bbox(coverage.bbox)
return coverage_area / grid_area
def estimate_tiles(grid, levels, coverage=None):
if coverage:
ratio = grid_coverage_ratio(grid.bbox, grid.srs, coverage)
else:
ratio = 1
tiles = 0
for level in levels:
grid_size = grid.grid_sizes[level]
level_tiles = grid_size[0] * grid_size[1]
level_tiles = int(math.ceil(level_tiles * ratio))
tiles += level_tiles
return tiles
if __name__ == '__main__':
from mapproxy.srs import SRS
from mapproxy.grid import tile_grid
from mapproxy.util.coverage import BBOXCoverage
print estimate_tiles(tile_grid(3857), levels=range(12), coverage=BBOXCoverage([5, 50, 10, 60], SRS(4326)))
def coverage(geom, srs):
if isinstance(geom, (list, tuple)):
return BBOXCoverage(geom, srs)
else:
return GeomCoverage(geom, srs)
def export_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration")
parser.add_option(
"-q",
"--quiet",
action="count",
dest="quiet",
default=0,
help=
"reduce number of messages to stdout, repeat to disable progress output"
)
parser.add_option("--source",
dest="source",
help="source to export (source or cache)")
parser.add_option(
"--grid",
help="grid for export. either the name of an existing grid or "
"the grid definition as a string")
parser.add_option("--dest",
help="destination of the export (directory or filename)")
parser.add_option("--type", help="type of the export format")
parser.add_option("--levels", help="levels to export: e.g 1,2,3 or 1..10")
parser.add_option(
"--fetch-missing-tiles",
dest="fetch_missing_tiles",
action='store_true',
default=False,
help="if missing tiles should be fetched from the sources")
parser.add_option(
"--force",
action='store_true',
default=False,
help="overwrite/append to existing --dest files/directories")
parser.add_option("-n",
"--dry-run",
action="store_true",
default=False,
help="do not export, just print output")
parser.add_option("-c",
"--concurrency",
type="int",
dest="concurrency",
default=1,
help="number of parallel export processes")
parser.add_option(
"--coverage",
help="the coverage for the export as a BBOX string, WKT file "
"or OGR datasource")
parser.add_option("--srs", help="the SRS of the coverage")
parser.add_option("--where", help="filter for OGR coverages")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
required_options = ['mapproxy_conf', 'grid', 'source', 'dest', 'levels']
for required in required_options:
if not getattr(options, required):
print('ERROR: missing required option --%s' %
required.replace('_', '-'),
file=sys.stderr)
parser.print_help()
sys.exit(1)
try:
conf = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ',
"%s: '%s'" % (e.strerror, e.filename),
file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
if '=' in options.grid:
try:
grid_conf = parse_grid_definition(options.grid)
except ValidationError as ex:
print('ERROR: invalid grid configuration', file=sys.stderr)
for error in ex.errors:
print(' ', error, file=sys.stderr)
sys.exit(2)
except ValueError:
print('ERROR: invalid grid configuration', file=sys.stderr)
sys.exit(2)
options.grid = 'tmp_mapproxy_export_grid'
grid_conf['name'] = options.grid
custom_grid = True
conf.grids[options.grid] = GridConfiguration(grid_conf, conf)
else:
custom_grid = False
if os.path.exists(options.dest) and not options.force:
print('ERROR: destination exists, remove first or use --force',
file=sys.stderr)
sys.exit(2)
cache_conf = {
'name': 'export',
'grids': [options.grid],
'sources': [options.source],
}
if options.type == 'mbtile':
cache_conf['cache'] = {
'type': 'mbtiles',
'filename': options.dest,
}
elif options.type == 'sqlite':
cache_conf['cache'] = {
'type': 'sqlite',
'directory': options.dest,
}
elif options.type == 'geopackage':
cache_conf['cache'] = {
'type': 'geopackage',
'filename': options.dest,
}
elif options.type == 'compact-v1':
cache_conf['cache'] = {
'type': 'compact',
'version': 1,
'directory': options.dest,
}
elif options.type == 'compact-v2':
cache_conf['cache'] = {
'type': 'compact',
'version': 2,
'directory': options.dest,
}
elif options.type in ('tc', 'mapproxy'):
cache_conf['cache'] = {
'type': 'file',
'directory': options.dest,
}
elif options.type == 'arcgis':
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'arcgis',
'directory': options.dest,
}
elif options.type in ('tms', None): # default
cache_conf['cache'] = {
'type': 'file',
'directory_layout': 'tms',
'directory': options.dest,
}
else:
print('ERROR: unsupported --type %s' % (options.type, ),
file=sys.stderr)
sys.exit(2)
if not options.fetch_missing_tiles:
for source in conf.sources.values():
source.conf['seed_only'] = True
tile_grid, extent, mgr = CacheConfiguration(cache_conf, conf).caches()[0]
levels = parse_levels(options.levels)
if levels[-1] >= tile_grid.levels:
print('ERROR: destination grid only has %d levels' % tile_grid.levels,
file=sys.stderr)
sys.exit(2)
if options.srs:
srs = SRS(options.srs)
else:
srs = tile_grid.srs
if options.coverage:
seed_coverage = load_coverage(
{
'datasource': options.coverage,
'srs': srs,
'where': options.where
},
base_path=os.getcwd())
else:
seed_coverage = BBOXCoverage(tile_grid.bbox, tile_grid.srs)
if not supports_tiled_access(mgr):
print(
'WARN: grids are incompatible. needs to scale/reproject tiles for export.',
file=sys.stderr)
md = dict(name='export',
cache_name='cache',
grid_name=options.grid,
dest=options.dest)
task = SeedTask(md, mgr, levels, 1, seed_coverage)
print(format_export_task(task, custom_grid=custom_grid))
logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2)
try:
seed_task(task,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency)
except KeyboardInterrupt:
print('stopping...', file=sys.stderr)
sys.exit(2)