def test_cleanup_remove_all(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_all']) self.make_tile((0, 0, 0)) self.make_tile((0, 0, 1)) self.make_tile((1, 0, 1)) self.make_tile((0, 1, 1)) self.make_tile((1, 1, 1)) self.make_tile((0, 0, 2)) self.make_tile((0, 0, 3)) eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))), ['00', '01', '02', '03']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert self.tile_exists((0, 0, 0)) assert not self.tile_exists((0, 0, 1)) assert not self.tile_exists((1, 0, 1)) assert not self.tile_exists((0, 1, 1)) assert not self.tile_exists((1, 1, 1)) assert not self.tile_exists((0, 0, 1)) assert self.tile_exists((0, 0, 2)) assert self.tile_exists((0, 0, 3)) # remove_all should remove the whole directory eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))), ['00', '02', '03'])
def test_seed_refresh_remove_before_from_file(self): # tile already there but too old, will be refreshed and removed t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25)) with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): # touch the seed_conf file and refresh everything timestamp = time.time() - 60 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5 # now touch the seed_conf again and remove everything os.utime(self.seed_conf_file, None) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_remove_before(self): # tile already there but too old t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60 * 60 * 25)) # old tile outside the seed view (should be removed) t001 = self.make_tile((0, 0, 1), timestamp=time.time() - (60 * 60 * 25)) assert os.path.exists(t000) assert os.path.exists(t001) with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({ 'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326' }, { 'body': img_data, 'headers': { 'content-type': 'image/png' } }) with mock_httpd(('localhost', 42423), [expected_req]): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time( ) < os.path.getmtime(t000) + 5 assert not os.path.exists(t001)
def test_reseed_uptodate(self): # tile already there. self.make_tile((0, 0, 0)) seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_cleanup_remove_all(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_all']) self.make_tile((0, 0, 0)) self.make_tile((0, 0, 1)) self.make_tile((1, 0, 1)) self.make_tile((0, 1, 1)) self.make_tile((1, 1, 1)) self.make_tile((0, 0, 2)) self.make_tile((0, 0, 3)) assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'), ['00', '01', '02', '03']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert self.tile_exists((0, 0, 0)) assert not self.tile_exists((0, 0, 1)) assert not self.tile_exists((1, 0, 1)) assert not self.tile_exists((0, 1, 1)) assert not self.tile_exists((1, 1, 1)) assert not self.tile_exists((0, 0, 1)) assert self.tile_exists((0, 0, 2)) assert self.tile_exists((0, 0, 3)) # remove_all should remove the whole directory assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'), ['00', '02', '03'])
def test_cleanup_sqlite_remove_all(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all']) cache = cleanup_tasks[0].tile_manager.cache cache.store_tile(self.create_tile((0, 0, 2))) cache.store_tile(self.create_tile((0, 0, 3))) assert cache.is_cached(Tile((0, 0, 2))) assert cache.is_cached(Tile((0, 0, 3))) eq_( sorted( os.listdir( os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))), ['2.mbtile', '3.mbtile']) cleanup(cleanup_tasks, verbose=False, dry_run=False) # 3.mbtile file should be removed completely eq_( sorted( os.listdir( os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))), ['3.mbtile']) assert not cache.is_cached(Tile((0, 0, 2))) assert cache.is_cached(Tile((0, 0, 3)))
def test_cleanup_sqlite(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['sqlite_cache']) cache = cleanup_tasks[0].tile_manager.cache cache.store_tile(self.create_tile((0, 0, 2))) cache.store_tile(self.create_tile((0, 0, 3))) assert cache.is_cached(Tile((0, 0, 2))) assert cache.is_cached(Tile((0, 0, 3))) assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'), ['2.mbtile', '3.mbtile'], glob='*.mbtile') cleanup(cleanup_tasks, verbose=False, dry_run=False) # 3.mbtile file is still there assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'), ['2.mbtile', '3.mbtile'], glob='*.mbtile') assert cache.is_cached(Tile((0, 0, 2))) assert not cache.is_cached(Tile((0, 0, 3)))
def test_cleanup_mbtiles(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache']) cache = tasks[0].tile_manager.cache cache.store_tile(self.create_tile()) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_uptodate(self): # tile already there. self.make_tile((0, 0, 0)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed_dry_run(self): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one' ]), seed_conf.cleanups() seed(tasks, dry_run=True) cleanup(cleanup_tasks, verbose=False, dry_run=True)
def test_seed(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed_mbtile(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache']), seed_conf.cleanups(['cleanup_mbtile_cache']) seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_cleanup_levels(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['cleanup']) self.make_tile((0, 0, 0)) self.make_tile((0, 0, 1)) self.make_tile((0, 0, 2)) self.make_tile((0, 0, 3)) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert not self.tile_exists((0, 0, 0)) assert not self.tile_exists((0, 0, 1)) assert self.tile_exists((0, 0, 2)) assert not self.tile_exists((0, 0, 3))
def test_seed_refresh_remove_before_from_file(self): # tile already there but old t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25)) # mtime is older than tile, no create of the tile timestamp = time.time() - (60*60*30) os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) # touch the seed_conf file and refresh everything os.utime(self.seed_conf_file, None) img_data = create_tmp_image((256, 256), format='png') expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): # touch the seed_conf file and refresh everything timestamp = time.time() - 60 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5 # mtime is older than tile, no cleanup timestamp = time.time() - 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert os.path.exists(t000) # now touch the seed_conf again and remove everything timestamp = time.time() + 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert not os.path.exists(t000)
def test_cleanup_levels(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['cleanup']) self.make_tile((0, 0, 0)) self.make_tile((0, 0, 1)) self.make_tile((0, 0, 2)) self.make_tile((0, 0, 3)) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert not self.tile_exists((0, 0, 0)) assert not self.tile_exists((0, 0, 1)) assert self.tile_exists((0, 0, 2)) assert not self.tile_exists((0, 0, 3)) eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))), ['02'])
def test_cleanup_sqlite_remove_all(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all']) cache = cleanup_tasks[0].tile_manager.cache cache.store_tile(self.create_tile((0, 0, 2))) cache.store_tile(self.create_tile((0, 0, 3))) assert cache.is_cached(Tile((0, 0, 2))) assert cache.is_cached(Tile((0, 0, 3))) eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))), ['2.mbtile', '3.mbtile']) cleanup(cleanup_tasks, verbose=False, dry_run=False) # 3.mbtile file should be removed completely eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))), ['3.mbtile']) assert not cache.is_cached(Tile((0, 0, 2))) assert cache.is_cached(Tile((0, 0, 3)))
def test_seed_skip_uncached(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() with local_base_config(self.mapproxy_conf.base_config): expected_req = ({ 'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326' }, { 'body': img_data, 'headers': { 'content-type': 'image/png' } }) seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds( ['one']), seed_conf.cleanups() # tile not in cache => skipped by seeder seed(tasks, dry_run=False, skip_uncached=True) assert not self.tile_exists((0, 0, 0)) with mock_httpd(('localhost', 42423), [expected_req]): # force tile generation in cache (via skip_uncached=False) seed(tasks, dry_run=False, skip_uncached=False) assert self.tile_exists((0, 0, 0)) # no refresh since tile is not older than 1 day (cf. config seed.yaml) seed(tasks, dry_run=False, skip_uncached=True) # create stale tile (older than 1 day) self.make_tile((0, 0, 0), timestamp=time.time() - (60 * 60 * 25)) with mock_httpd(('localhost', 42423), [expected_req]): # check that old tile in cache is refreshed seed(tasks, dry_run=False, skip_uncached=True) assert self.tile_exists((0, 0, 0)) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_reseed_remove_before(self): # tile already there but too old t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25)) # old tile outside the seed view (should be removed) t001 = self.make_tile((0, 0, 1), timestamp=time.time() - (60*60*25)) assert os.path.exists(t000) assert os.path.exists(t001) with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5 assert not os.path.exists(t001)
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) if options.duration: # calls with --duration are handled in call_with_duration sys.exit(self.call_with_duration(options, args)) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 progress = None if options.continue_seed or options.progress_file: if not options.progress_file: options.progress_file = '.mapproxy_seed_progress' progress = ProgressStore(options.progress_file, continue_seed=options.continue_seed) if options.reseed_file: if not os.path.exists(options.reseed_file): # create --reseed-file if missing with open(options.reseed_file, 'w'): pass else: if progress and not os.path.exists(options.progress_file): # we have an existing --reseed-file but no --progress-file # meaning the last seed call was completed if options.reseed_interval and ( os.path.getmtime(options.reseed_file) > (time.time() - options.reseed_interval)): print("no need for re-seeding") sys.exit(1) os.utime(options.reseed_file, (time.time(), time.time())) with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels, skip_uncached=options.skip_uncached) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 progress = None if options.continue_seed or options.progress_file: if options.progress_file: progress_file = options.progress_file else: progress_file = '.mapproxy_seed_progress' progress = ProgressStore(progress_file, continue_seed=options.continue_seed) try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
class SeedScript(object): usage = "usage: %prog [options] seed_conf" parser = OptionParser(usage) parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, help="don't print status messages to stdout") parser.add_option("-s", "--seed-conf", dest="seed_file", default=None, help="seed configuration") parser.add_option("-f", "--proxy-conf", dest="conf_file", default=None, help="proxy configuration") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=2, help="number of parallel seed processes") parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, help="do not seed, just print output") parser.add_option("-l", "--skip-geoms-for-last-levels", type="int", dest="geom_levels", default=0, metavar="N", help="do not check for intersections between tiles" " and seed geometries on the last N levels") parser.add_option("--summary", action="store_true", dest="summary", default=False, help="print summary with all seeding tasks and exit." " does not seed anything.") parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False, help="print each task description and ask if it should be seeded") parser.add_option( "--seed", action="append", dest="seed_names", metavar='task1,task2,...', help="seed only the named tasks. cleanup is disabled unless " "--cleanup is used. use ALL to select all tasks") parser.add_option( "--cleanup", action="append", dest="cleanup_names", metavar='task1,task2,...', help="cleanup only the named tasks. seeding is disabled unless " "--seed is used. use ALL to select all tasks") parser.add_option( "--use-cache-lock", action="store_true", default=False, help="use locking to prevent multiple mapproxy-seed calls " "to seed the same cache") def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging() try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError, ex: print "ERROR: " + '\n\t'.join(str(ex).split('\n')) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None with local_base_config(mapproxy_conf.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError, ex: print "error in configuration: " + '\n\t'.join( str(ex).split('\n')) sys.exit(2) if options.summary: print '========== Seeding tasks ==========' for task in seed_tasks: print format_seed_task(task) print '========== Cleanup tasks ==========' for task in cleanup_tasks: print format_cleanup_task(task) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print '========== Seeding tasks ==========' print 'Start seeding process (%d task%s)' % ( len(seed_tasks), 's' if len(seed_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print '========== Cleanup tasks ==========' print 'Start cleanup process (%d task%s)' % (len( cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except KeyboardInterrupt: print '\nexiting...' return 2
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error('missing seed_conf file as last argument or --seed-conf option') else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 progress = None if options.continue_seed or options.progress_file: if options.progress_file: progress_file = options.progress_file else: progress_file = '.mapproxy_seed_progress' progress = ProgressStore(progress_file, continue_seed=options.continue_seed) try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print('Start seeding process (%d task%s)' % ( len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % ( len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2) cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def test_seed_dry_run(self): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=True) cleanup(cleanup_tasks, verbose=False, dry_run=True)