def test_seed_refresh_remove_before_from_file(self):
# tile already there but too old, will be refreshed and removed
t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
# touch the seed_conf file and refresh everything
timestamp = time.time() - 60
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['refresh_from_file'])
seed(tasks, dry_run=False)
assert os.path.exists(t000)
assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
# now touch the seed_conf again and remove everything
os.utime(self.seed_conf_file, None)
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed_refresh_remove_before_from_file(self):
# tile already there but old
t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
# mtime is older than tile, no create of the tile
timestamp = time.time() - (60*60*30)
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['refresh_from_file'])
seed(tasks, dry_run=False)
# touch the seed_conf file and refresh everything
os.utime(self.seed_conf_file, None)
img_data = create_tmp_image((256, 256), format='png')
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
# touch the seed_conf file and refresh everything
timestamp = time.time() - 60
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['refresh_from_file'])
seed(tasks, dry_run=False)
assert os.path.exists(t000)
assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
# mtime is older than tile, no cleanup
timestamp = time.time() - 5
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert os.path.exists(t000)
# now touch the seed_conf again and remove everything
timestamp = time.time() + 5
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert not os.path.exists(t000)
def test_timeout(self):
# test concurrent seeding where seed concurrency is higher than the permitted
# concurrent_request value of the source and a lock times out
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['test'])
expected_req1 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
expected_req2 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=512&height=256&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
expected_req3 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=1024&height=512&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
with mock_httpd(('localhost', 42423), [expected_req1, expected_req2, expected_req3], unordered=True):
seed(tasks, dry_run=False, concurrency=3)
def test_cleanup_remove_all(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_all'])
self.make_tile((0, 0, 0))
self.make_tile((0, 0, 1))
self.make_tile((1, 0, 1))
self.make_tile((0, 1, 1))
self.make_tile((1, 1, 1))
self.make_tile((0, 0, 2))
self.make_tile((0, 0, 3))
eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
['00', '01', '02', '03'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert self.tile_exists((0, 0, 0))
assert not self.tile_exists((0, 0, 1))
assert not self.tile_exists((1, 0, 1))
assert not self.tile_exists((0, 1, 1))
assert not self.tile_exists((1, 1, 1))
assert not self.tile_exists((0, 0, 1))
assert self.tile_exists((0, 0, 2))
assert self.tile_exists((0, 0, 3))
# remove_all should remove the whole directory
eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
['00', '02', '03'])
def test_timeout(self):
# test concurrent seeding where seed concurrency is higher than the permitted
# concurrent_request value of the source and a lock times out
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['test'])
expected_req1 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
expected_req2 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=512&height=256&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
expected_req3 = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=1024&height=512&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}, 'duration': 0.1})
with mock_httpd(('localhost', 42423), [expected_req1, expected_req2, expected_req3], unordered=True):
seed(tasks, dry_run=False, concurrency=3)
def test_reseed_uptodate(self):
# tile already there.
self.make_tile((0, 0, 0))
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_remove_before(self):
# tile already there but too old
t000 = self.make_tile((0, 0, 0),
timestamp=time.time() - (60 * 60 * 25))
# old tile outside the seed view (should be removed)
t001 = self.make_tile((0, 0, 1),
timestamp=time.time() - (60 * 60 * 25))
assert os.path.exists(t000)
assert os.path.exists(t001)
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({
'path':
r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'
}, {
'body': img_data,
'headers': {
'content-type': 'image/png'
}
})
with mock_httpd(('localhost', 42423), [expected_req]):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert os.path.exists(t000)
assert os.path.getmtime(t000) - 5 < time.time(
) < os.path.getmtime(t000) + 5
assert not os.path.exists(t001)
def test_cleanup_remove_all(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_all'])
self.make_tile((0, 0, 0))
self.make_tile((0, 0, 1))
self.make_tile((1, 0, 1))
self.make_tile((0, 1, 1))
self.make_tile((1, 1, 1))
self.make_tile((0, 0, 2))
self.make_tile((0, 0, 3))
assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'),
['00', '01', '02', '03'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert self.tile_exists((0, 0, 0))
assert not self.tile_exists((0, 0, 1))
assert not self.tile_exists((1, 0, 1))
assert not self.tile_exists((0, 1, 1))
assert not self.tile_exists((1, 1, 1))
assert not self.tile_exists((0, 0, 1))
assert self.tile_exists((0, 0, 2))
assert self.tile_exists((0, 0, 3))
# remove_all should remove the whole directory
assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'),
['00', '02', '03'])
def test_cleanup_sqlite(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['sqlite_cache'])
cache = cleanup_tasks[0].tile_manager.cache
cache.store_tile(self.create_tile((0, 0, 2)))
cache.store_tile(self.create_tile((0, 0, 3)))
assert cache.is_cached(Tile((0, 0, 2)))
assert cache.is_cached(Tile((0, 0, 3)))
assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache',
'GLOBAL_GEODETIC'),
['2.mbtile', '3.mbtile'],
glob='*.mbtile')
cleanup(cleanup_tasks, verbose=False, dry_run=False)
# 3.mbtile file is still there
assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache',
'GLOBAL_GEODETIC'),
['2.mbtile', '3.mbtile'],
glob='*.mbtile')
assert cache.is_cached(Tile((0, 0, 2)))
assert not cache.is_cached(Tile((0, 0, 3)))
def test_cleanup_sqlite_remove_all(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all'])
cache = cleanup_tasks[0].tile_manager.cache
cache.store_tile(self.create_tile((0, 0, 2)))
cache.store_tile(self.create_tile((0, 0, 3)))
assert cache.is_cached(Tile((0, 0, 2)))
assert cache.is_cached(Tile((0, 0, 3)))
eq_(
sorted(
os.listdir(
os.path.join(self.dir, 'cache', 'sqlite_cache',
'GLOBAL_GEODETIC'))),
['2.mbtile', '3.mbtile'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
# 3.mbtile file should be removed completely
eq_(
sorted(
os.listdir(
os.path.join(self.dir, 'cache', 'sqlite_cache',
'GLOBAL_GEODETIC'))), ['3.mbtile'])
assert not cache.is_cached(Tile((0, 0, 2)))
assert cache.is_cached(Tile((0, 0, 3)))
def test_seed_dry_run(self):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one'
]), seed_conf.cleanups()
seed(tasks, dry_run=True)
cleanup(cleanup_tasks, verbose=False, dry_run=True)
def test_cleanup_mbtiles(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
cache = tasks[0].tile_manager.cache
cache.store_tile(self.create_tile())
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_cleanup_mbtiles(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
cache = tasks[0].tile_manager.cache
cache.store_tile(self.create_tile())
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_uptodate(self):
# tile already there.
self.make_tile((0, 0, 0))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed_mbtile(self):
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache']), seed_conf.cleanups(['cleanup_mbtile_cache'])
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed(self):
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_mbtiles_with_refresh(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
cache = tasks[0].tile_manager.cache
cache.store_tile(self.create_tile())
expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
# mbtiles does not support timestamps, refresh all tiles
seed(tasks, dry_run=False)
def test_cleanup_levels(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['cleanup'])
self.make_tile((0, 0, 0))
self.make_tile((0, 0, 1))
self.make_tile((0, 0, 2))
self.make_tile((0, 0, 3))
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert not self.tile_exists((0, 0, 0))
assert not self.tile_exists((0, 0, 1))
assert self.tile_exists((0, 0, 2))
assert not self.tile_exists((0, 0, 3))
def test_reseed_mbtiles_with_refresh(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
cache = tasks[0].tile_manager.cache
cache.store_tile(self.create_tile())
expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': tile_image, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
# mbtiles does not support timestamps, refresh all tiles
seed(tasks, dry_run=False)
def test_cleanup_levels(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['cleanup'])
self.make_tile((0, 0, 0))
self.make_tile((0, 0, 1))
self.make_tile((0, 0, 2))
self.make_tile((0, 0, 3))
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert not self.tile_exists((0, 0, 0))
assert not self.tile_exists((0, 0, 1))
assert self.tile_exists((0, 0, 2))
assert not self.tile_exists((0, 0, 3))
def test_cleanup_levels(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['cleanup'])
self.make_tile((0, 0, 0))
self.make_tile((0, 0, 1))
self.make_tile((0, 0, 2))
self.make_tile((0, 0, 3))
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert not self.tile_exists((0, 0, 0))
assert not self.tile_exists((0, 0, 1))
assert self.tile_exists((0, 0, 2))
assert not self.tile_exists((0, 0, 3))
eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
['02'])
def test_cleanup_sqlite_remove_all(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all'])
cache = cleanup_tasks[0].tile_manager.cache
cache.store_tile(self.create_tile((0, 0, 2)))
cache.store_tile(self.create_tile((0, 0, 3)))
assert cache.is_cached(Tile((0, 0, 2)))
assert cache.is_cached(Tile((0, 0, 3)))
eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
['2.mbtile', '3.mbtile'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
# 3.mbtile file should be removed completely
eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
['3.mbtile'])
assert not cache.is_cached(Tile((0, 0, 2)))
assert cache.is_cached(Tile((0, 0, 3)))
def test_seed_skip_uncached(self):
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
with local_base_config(self.mapproxy_conf.base_config):
expected_req = ({
'path':
r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'
}, {
'body': img_data,
'headers': {
'content-type': 'image/png'
}
})
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(
['one']), seed_conf.cleanups()
# tile not in cache => skipped by seeder
seed(tasks, dry_run=False, skip_uncached=True)
assert not self.tile_exists((0, 0, 0))
with mock_httpd(('localhost', 42423), [expected_req]):
# force tile generation in cache (via skip_uncached=False)
seed(tasks, dry_run=False, skip_uncached=False)
assert self.tile_exists((0, 0, 0))
# no refresh since tile is not older than 1 day (cf. config seed.yaml)
seed(tasks, dry_run=False, skip_uncached=True)
# create stale tile (older than 1 day)
self.make_tile((0, 0, 0),
timestamp=time.time() - (60 * 60 * 25))
with mock_httpd(('localhost', 42423), [expected_req]):
# check that old tile in cache is refreshed
seed(tasks, dry_run=False, skip_uncached=True)
assert self.tile_exists((0, 0, 0))
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_remove_before(self):
# tile already there but too old
t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
# old tile outside the seed view (should be removed)
t001 = self.make_tile((0, 0, 1), timestamp=time.time() - (60*60*25))
assert os.path.exists(t000)
assert os.path.exists(t001)
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
assert os.path.exists(t000)
assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
assert not os.path.exists(t001)
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict((key.lower(), value) for (key, value) in grids.iteritems())
if not grids.get(options.grid_name, False):
print 'grid not found: %s' % (options.grid_name,)
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
with local_base_config(proxy_configuration.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
except SeedConfigurationError, e:
print >>sys.stderr, 'ERROR: invalid configuration (see above)'
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print 'Old seed configuration format not supported'
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
print '--coverage and --seed-conf can only be used together'
sys.exit(1)
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option("-g", "--grid", dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option("--all", dest="show_all", action="store_true", default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage")
coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined")
coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
if options.show_all or options.grid_name:
grids = proxy_configuration.grids
else:
caches = proxy_configuration.caches
grids = {}
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict((key.lower(), value) for (key, value) in iteritems(grids))
if not grids.get(options.grid_name, False):
print('grid not found: %s' % (options.grid_name,))
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
try:
seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
except SeedConfigurationError as e:
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print('Old seed configuration format not supported')
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
print('--coverage and --seed-conf can only be used together')
sys.exit(1)
if options.list_grids:
display_grids_list(grids)
elif options.grid_name:
display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage)
else:
display_grids(grids, coverage=coverage)
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
if options.duration:
# calls with --duration are handled in call_with_duration
sys.exit(self.call_with_duration(options, args))
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
progress = None
if options.continue_seed or options.progress_file:
if not options.progress_file:
options.progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(options.progress_file,
continue_seed=options.continue_seed)
if options.reseed_file:
if not os.path.exists(options.reseed_file):
# create --reseed-file if missing
with open(options.reseed_file, 'w'):
pass
else:
if progress and not os.path.exists(options.progress_file):
# we have an existing --reseed-file but no --progress-file
# meaning the last seed call was completed
if options.reseed_interval and (
os.path.getmtime(options.reseed_file) >
(time.time() - options.reseed_interval)):
print("no need for re-seeding")
sys.exit(1)
os.utime(options.reseed_file, (time.time(), time.time()))
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels,
skip_uncached=options.skip_uncached)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def test_active_seed_tasks(self):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
assert len(seed_conf.seed_tasks_names()) == 5
assert len(seed_conf.seeds()) == 5
def test_active_seed_tasks(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file,
self.mapproxy_conf)
assert len(seed_conf.seed_tasks_names()) == 4
assert len(seed_conf.seeds()) == 3
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict(
(key.lower(), value) for (key, value) in grids.iteritems())
if not grids.get(options.grid_name, False):
print 'grid not found: %s' % (options.grid_name, )
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
with local_base_config(proxy_configuration.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_config,
proxy_configuration)
except SeedConfigurationError, e:
print >> sys.stderr, 'ERROR: invalid configuration (see above)'
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print 'Old seed configuration format not supported'
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage
and not options.seed_config) or (not options.coverage
and options.seed_config):
print '--coverage and --seed-conf can only be used together'
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
class SeedScript(object):
usage = "usage: %prog [options] seed_conf"
parser = OptionParser(usage)
parser.add_option("-q",
"--quiet",
action="count",
dest="quiet",
default=0,
help="don't print status messages to stdout")
parser.add_option("-s",
"--seed-conf",
dest="seed_file",
default=None,
help="seed configuration")
parser.add_option("-f",
"--proxy-conf",
dest="conf_file",
default=None,
help="proxy configuration")
parser.add_option("-c",
"--concurrency",
type="int",
dest="concurrency",
default=2,
help="number of parallel seed processes")
parser.add_option("-n",
"--dry-run",
action="store_true",
dest="dry_run",
default=False,
help="do not seed, just print output")
parser.add_option("-l",
"--skip-geoms-for-last-levels",
type="int",
dest="geom_levels",
default=0,
metavar="N",
help="do not check for intersections between tiles"
" and seed geometries on the last N levels")
parser.add_option("--summary",
action="store_true",
dest="summary",
default=False,
help="print summary with all seeding tasks and exit."
" does not seed anything.")
parser.add_option(
"-i",
"--interactive",
action="store_true",
dest="interactive",
default=False,
help="print each task description and ask if it should be seeded")
parser.add_option(
"--seed",
action="append",
dest="seed_names",
metavar='task1,task2,...',
help="seed only the named tasks. cleanup is disabled unless "
"--cleanup is used. use ALL to select all tasks")
parser.add_option(
"--cleanup",
action="append",
dest="cleanup_names",
metavar='task1,task2,...',
help="cleanup only the named tasks. seeding is disabled unless "
"--seed is used. use ALL to select all tasks")
parser.add_option(
"--use-cache-lock",
action="store_true",
default=False,
help="use locking to prevent multiple mapproxy-seed calls "
"to seed the same cache")
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging()
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError, ex:
print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
with local_base_config(mapproxy_conf.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError, ex:
print "error in configuration: " + '\n\t'.join(
str(ex).split('\n'))
sys.exit(2)
if options.summary:
print '========== Seeding tasks =========='
for task in seed_tasks:
print format_seed_task(task)
print '========== Cleanup tasks =========='
for task in cleanup_tasks:
print format_cleanup_task(task)
return 0
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print '========== Seeding tasks =========='
print 'Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print '========== Cleanup tasks =========='
print 'Start cleanup process (%d task%s)' % (len(
cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except KeyboardInterrupt:
print '\nexiting...'
return 2
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error('missing seed_conf file as last argument or --seed-conf option')
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print('Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
progress_store=progress)
seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency, cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' % (
len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
concurrency=options.concurrency, progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def grids_command(args=None):
parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
parser.add_option("-f",
"--mapproxy-conf",
dest="mapproxy_conf",
help="MapProxy configuration.")
parser.add_option(
"-g",
"--grid",
dest="grid_name",
help="Display only information about the specified grid.")
parser.add_option(
"--all",
dest="show_all",
action="store_true",
default=False,
help="Show also grids that are not referenced by any cache.")
parser.add_option(
"-l",
"--list",
dest="list_grids",
action="store_true",
default=False,
help="List names of configured grids, which are used by any cache")
coverage_group = parser.add_option_group(
"Approximate the number of tiles within a given coverage")
coverage_group.add_option(
"-s",
"--seed-conf",
dest="seed_config",
help="Seed configuration, where the coverage is defined")
coverage_group.add_option(
"-c",
"--coverage-name",
dest="coverage",
help="Calculate number of tiles when a coverage is given")
from mapproxy.script.util import setup_logging
import logging
setup_logging(logging.WARN)
if args:
args = args[1:] # remove script name
(options, args) = parser.parse_args(args)
if not options.mapproxy_conf:
if len(args) != 1:
parser.print_help()
sys.exit(1)
else:
options.mapproxy_conf = args[0]
try:
proxy_configuration = load_configuration(options.mapproxy_conf)
except IOError as e:
print('ERROR: ',
"%s: '%s'" % (e.strerror, e.filename),
file=sys.stderr)
sys.exit(2)
except ConfigurationError as e:
print(e, file=sys.stderr)
print('ERROR: invalid configuration (see above)', file=sys.stderr)
sys.exit(2)
with local_base_config(proxy_configuration.base_config):
if options.show_all or options.grid_name:
grids = proxy_configuration.grids
else:
caches = proxy_configuration.caches
grids = {}
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict(
(key.lower(), value) for (key, value) in iteritems(grids))
if not grids.get(options.grid_name, False):
print('grid not found: %s' % (options.grid_name, ))
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
try:
seed_conf = load_seed_tasks_conf(options.seed_config,
proxy_configuration)
except SeedConfigurationError as e:
print('ERROR: invalid configuration (see above)',
file=sys.stderr)
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print('Old seed configuration format not supported')
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage
and not options.seed_config) or (not options.coverage
and options.seed_config):
print('--coverage and --seed-conf can only be used together')
sys.exit(1)
if options.list_grids:
display_grids_list(grids)
elif options.grid_name:
display_grids({options.grid_name: grids[options.grid_name]},
coverage=coverage)
else:
display_grids(grids, coverage=coverage)
def test_active_seed_tasks(self):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
assert len(seed_conf.seed_tasks_names()) == 5
assert len(seed_conf.seeds()) == 5
def test_seed_dry_run(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=True)
cleanup(cleanup_tasks, verbose=False, dry_run=True)
def test_active_seed_tasks(self):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
assert len(seed_conf.seed_tasks_names()) == 4
assert len(seed_conf.seeds()) == 3
