def test_load_broken(self):
with TempFile(no_create=True) as tmp:
with open(tmp, 'w') as f:
f.write('##invaliddata')
f.write(pickle.dumps({("view", "cache", "grid"): [(0, 1), (2, 4)]}))
store = ProgressStore(tmp)
assert store.status == {}
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error('missing seed_conf file as last argument or --seed-conf option')
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print('Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
progress_store=progress)
seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency, cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' % (
len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
concurrency=options.concurrency, progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def get_progress_store(gpkg):
progress_file = os.path.join(os.path.dirname(gpkg), ".progress_logger")
return ProgressStore(filename=progress_file, continue_seed=True)
def test_load_store(self):
with TempFile(no_create=True) as tmp:
with open(tmp, "wb") as f:
f.write(
pickle.dumps({("view", "cache", "grid"): [(0, 1),
(2, 4)]}))
store = ProgressStore(tmp)
assert store.get(("view", "cache", "grid")) == [(0, 1), (2, 4)]
assert store.get(("view", "cache", "grid2")) == None
store.add(("view", "cache", "grid"), [])
store.add(("view", "cache", "grid2"), [(0, 1)])
store.write()
store = ProgressStore(tmp)
assert store.get(("view", "cache", "grid")) == []
assert store.get(("view", "cache", "grid2")) == [(0, 1)]
def test_load_empty(self):
store = ProgressStore("doesnotexist.no_realy.txt")
store.load()
assert store.get(("foo", "bar", "baz")) == None
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
def test_load_empty(self):
store = ProgressStore('doesnotexist.no_realy.txt')
store.load()
assert store.get(('foo', 'bar', 'baz')) == None
def test_load_store(self):
with TempFile(no_create=True) as tmp:
with open(tmp, 'w') as f:
f.write(pickle.dumps({("view", "cache", "grid"): [(0, 1), (2, 4)]}))
store = ProgressStore(tmp)
assert store.get(('view', 'cache', 'grid')) == [(0, 1), (2, 4)]
assert store.get(('view', 'cache', 'grid2')) == None
store.add(('view', 'cache', 'grid'), [])
store.add(('view', 'cache', 'grid2'), [(0, 1)])
store.write()
store = ProgressStore(tmp)
assert store.get(('view', 'cache', 'grid')) == []
assert store.get(('view', 'cache', 'grid2')) == [(0, 1)]
def test_load_empty(self):
store = ProgressStore('doesnotexist.no_realy.txt')
store.load()
assert store.get(('foo', 'bar', 'baz')) == None
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging(options.logging_conf)
if options.duration:
# calls with --duration are handled in call_with_duration
sys.exit(self.call_with_duration(options, args))
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError as ex:
print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
if not sys.stdout.isatty() and options.quiet == 0:
# disable verbose output for non-ttys
options.quiet = 1
progress = None
if options.continue_seed or options.progress_file:
if not options.progress_file:
options.progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(options.progress_file,
continue_seed=options.continue_seed)
if options.reseed_file:
if not os.path.exists(options.reseed_file):
# create --reseed-file if missing
with open(options.reseed_file, 'w'):
pass
else:
if progress and not os.path.exists(options.progress_file):
# we have an existing --reseed-file but no --progress-file
# meaning the last seed call was completed
if options.reseed_interval and (
os.path.getmtime(options.reseed_file) >
(time.time() - options.reseed_interval)):
print("no need for re-seeding")
sys.exit(1)
os.utime(options.reseed_file, (time.time(), time.time()))
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError as ex:
print("error in configuration: " +
'\n\t'.join(str(ex).split('\n')))
sys.exit(2)
if options.summary:
print('========== Seeding tasks ==========')
for task in seed_tasks:
print(format_seed_task(task))
print('========== Cleanup tasks ==========')
for task in cleanup_tasks:
print(format_cleanup_task(task))
return 0
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print('========== Seeding tasks ==========')
print(
'Start seeding process (%d task%s)' %
(len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels,
skip_uncached=options.skip_uncached)
if cleanup_tasks:
print('========== Cleanup tasks ==========')
print('Start cleanup process (%d task%s)' %
(len(cleanup_tasks),
's' if len(cleanup_tasks) > 1 else ''))
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2,
progress_store=progress)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except SeedInterrupted:
print('\ninterrupted...')
return 3
except KeyboardInterrupt:
print('\nexiting...')
return 2
if progress:
progress.remove()
class SeedScript(object):
usage = "usage: %prog [options] seed_conf"
parser = OptionParser(usage)
parser.add_option("-q", "--quiet",
action="count", dest="quiet", default=0,
help="don't print status messages to stdout")
parser.add_option("-s", "--seed-conf",
dest="seed_file", default=None,
help="seed configuration")
parser.add_option("-f", "--proxy-conf",
dest="conf_file", default=None,
help="proxy configuration")
parser.add_option("-c", "--concurrency", type="int",
dest="concurrency", default=2,
help="number of parallel seed processes")
parser.add_option("-n", "--dry-run",
action="store_true", dest="dry_run", default=False,
help="do not seed, just print output")
parser.add_option("-l", "--skip-geoms-for-last-levels",
type="int", dest="geom_levels", default=0,
metavar="N",
help="do not check for intersections between tiles"
" and seed geometries on the last N levels")
parser.add_option("--summary",
action="store_true", dest="summary", default=False,
help="print summary with all seeding tasks and exit."
" does not seed anything.")
parser.add_option("-i", "--interactive",
action="store_true", dest="interactive", default=False,
help="print each task description and ask if it should be seeded")
parser.add_option("--seed",
action="append", dest="seed_names", metavar='task1,task2,...',
help="seed only the named tasks. cleanup is disabled unless "
"--cleanup is used. use ALL to select all tasks")
parser.add_option("--cleanup",
action="append", dest="cleanup_names", metavar='task1,task2,...',
help="cleanup only the named tasks. seeding is disabled unless "
"--seed is used. use ALL to select all tasks")
parser.add_option("--use-cache-lock",
action="store_true", default=False,
help="use locking to prevent multiple mapproxy-seed calls "
"to seed the same cache")
parser.add_option("--continue", dest='continue_seed',
action="store_true", default=False,
help="continue an aborted seed progress")
parser.add_option("--progress-file", dest='progress_file',
default=None,
help="filename for storing the seed progress (for --continue option)")
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error('missing seed_conf file as last argument or --seed-conf option')
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging()
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError, ex:
print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
with mapproxy_conf:
try:
seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError, ex:
print "error in configuration: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
if options.summary:
print '========== Seeding tasks =========='
for task in seed_tasks:
print format_seed_task(task)
print '========== Cleanup tasks =========='
for task in cleanup_tasks:
print format_cleanup_task(task)
return 0
progress = None
if options.continue_seed or options.progress_file:
if options.progress_file:
progress_file = options.progress_file
else:
progress_file = '.mapproxy_seed_progress'
progress = ProgressStore(progress_file,
continue_seed=options.continue_seed)
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)
if seed_tasks:
print '========== Seeding tasks =========='
print 'Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
progress_store=progress)
seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
concurrency=options.concurrency, cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print '========== Cleanup tasks =========='
print 'Start cleanup process (%d task%s)' % (
len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
concurrency=options.concurrency, progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except KeyboardInterrupt:
print '\nexiting...'
return 2
if progress:
progress.remove()