def test_seed_refresh_remove_before_from_file(self):
# tile already there but too old, will be refreshed and removed
t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
# touch the seed_conf file and refresh everything
timestamp = time.time() - 60
os.utime(self.seed_conf_file, (timestamp, timestamp))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks = seed_conf.seeds(['refresh_from_file'])
seed(tasks, dry_run=False)
assert os.path.exists(t000)
assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
# now touch the seed_conf again and remove everything
os.utime(self.seed_conf_file, None)
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def __call__(self, environ, start_response):
resp = None
req = Request(environ)
with local_base_config(self.base_config):
match = self.handler_path_re.match(req.path)
if match:
handler_name = match.group(1)
if handler_name in self.handlers:
try:
resp = self.handlers[handler_name].handle(req)
except Exception, ex:
if self.base_config.debug_mode:
raise
else:
log_wsgiapp.fatal('fatal error in %s for %s %s',
handler_name,
environ.get('PATH_INFO'),
environ.get('QUERY_STRING'),
exc_info=True)
import traceback
traceback.print_exc(file=environ['wsgi.errors'])
resp = Response('internal error', status=500)
if resp is None:
if req.path in ('', '/'):
resp = self.welcome_response(req.script_url)
else:
resp = Response('not found',
mimetype='text/plain',
status=404)
return resp(environ, start_response)
def __call__(self, environ, start_response):
resp = None
req = Request(environ)
with local_base_config(self.base_config):
match = self.handler_path_re.match(req.path)
if match:
handler_name = match.group(1)
if handler_name in self.handlers:
try:
resp = self.handlers[handler_name].handle(req)
except Exception, ex:
if self.base_config.debug_mode:
raise
else:
log_wsgiapp.fatal('fatal error in %s for %s %s',
handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True)
import traceback
traceback.print_exc(file=environ['wsgi.errors'])
resp = Response('internal error', status=500)
if resp is None:
if req.path in ('', '/'):
resp = self.welcome_response(req.script_url)
else:
resp = Response('not found', mimetype='text/plain', status=404)
return resp(environ, start_response)
def test_reseed_uptodate(self):
# tile already there.
self.make_tile((0, 0, 0))
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def call(*args):
with local_base_config(self.base_config):
try:
return func(*args)
except Exception:
if use_result_objects:
return sys.exc_info()
else:
raise
def call(*args):
with local_base_config(self.base_config):
try:
return func(*args)
except Exception:
if use_result_objects:
return sys.exc_info()
else:
raise
def test_seed(self):
with tmp_image((256, 256), format='png') as img:
img_data = img.read()
expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
'&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
'&width=256&height=128&srs=EPSG:4326'},
{'body': img_data, 'headers': {'content-type': 'image/png'}})
with mock_httpd(('localhost', 42423), [expected_req]):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=False)
cleanup(cleanup_tasks, verbose=False, dry_run=False)
def run(self):
with local_base_config(self.base_config):
while True:
task = self.task_queue.get()
if task is None:
self.task_queue.task_done()
break
exec_id, func, args = task
try:
result = func(*args)
except Exception:
result = sys.exc_info()
self.result_queue.put((exec_id, result))
self.task_queue.task_done()
def run(self):
with local_base_config(self.base_config):
while True:
task = self.task_queue.get()
if task is None:
self.task_queue.task_done()
break
exec_id, func, args = task
try:
result = func(*args)
except Exception:
result = sys.exc_info()
self.result_queue.put((exec_id, result))
self.task_queue.task_done()
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict(
(key.lower(), value) for (key, value) in grids.iteritems())
if not grids.get(options.grid_name, False):
print 'grid not found: %s' % (options.grid_name, )
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
with local_base_config(proxy_configuration.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_config,
proxy_configuration)
except SeedConfigurationError, e:
print >> sys.stderr, 'ERROR: invalid configuration (see above)'
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print 'Old seed configuration format not supported'
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage
class SeedScript(object):
usage = "usage: %prog [options] seed_conf"
parser = OptionParser(usage)
parser.add_option("-q",
"--quiet",
action="count",
dest="quiet",
default=0,
help="don't print status messages to stdout")
parser.add_option("-s",
"--seed-conf",
dest="seed_file",
default=None,
help="seed configuration")
parser.add_option("-f",
"--proxy-conf",
dest="conf_file",
default=None,
help="proxy configuration")
parser.add_option("-c",
"--concurrency",
type="int",
dest="concurrency",
default=2,
help="number of parallel seed processes")
parser.add_option("-n",
"--dry-run",
action="store_true",
dest="dry_run",
default=False,
help="do not seed, just print output")
parser.add_option("-l",
"--skip-geoms-for-last-levels",
type="int",
dest="geom_levels",
default=0,
metavar="N",
help="do not check for intersections between tiles"
" and seed geometries on the last N levels")
parser.add_option("--summary",
action="store_true",
dest="summary",
default=False,
help="print summary with all seeding tasks and exit."
" does not seed anything.")
parser.add_option(
"-i",
"--interactive",
action="store_true",
dest="interactive",
default=False,
help="print each task description and ask if it should be seeded")
parser.add_option(
"--seed",
action="append",
dest="seed_names",
metavar='task1,task2,...',
help="seed only the named tasks. cleanup is disabled unless "
"--cleanup is used. use ALL to select all tasks")
parser.add_option(
"--cleanup",
action="append",
dest="cleanup_names",
metavar='task1,task2,...',
help="cleanup only the named tasks. seeding is disabled unless "
"--seed is used. use ALL to select all tasks")
parser.add_option(
"--use-cache-lock",
action="store_true",
default=False,
help="use locking to prevent multiple mapproxy-seed calls "
"to seed the same cache")
def __call__(self):
(options, args) = self.parser.parse_args()
if len(args) != 1 and not options.seed_file:
self.parser.print_help()
sys.exit(1)
if not options.seed_file:
if len(args) != 1:
self.parser.error(
'missing seed_conf file as last argument or --seed-conf option'
)
else:
options.seed_file = args[0]
if not options.conf_file:
self.parser.error('missing mapproxy configuration -f/--proxy-conf')
setup_logging()
try:
mapproxy_conf = load_configuration(options.conf_file, seed=True)
except ConfigurationError, ex:
print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
sys.exit(2)
if options.use_cache_lock:
cache_locker = CacheLocker('.mapproxy_seed.lck')
else:
cache_locker = None
with local_base_config(mapproxy_conf.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_file,
mapproxy_conf)
seed_names, cleanup_names = self.task_names(seed_conf, options)
seed_tasks = seed_conf.seeds(seed_names)
cleanup_tasks = seed_conf.cleanups(cleanup_names)
except ConfigurationError, ex:
print "error in configuration: " + '\n\t'.join(
str(ex).split('\n'))
sys.exit(2)
if options.summary:
print '========== Seeding tasks =========='
for task in seed_tasks:
print format_seed_task(task)
print '========== Cleanup tasks =========='
for task in cleanup_tasks:
print format_cleanup_task(task)
return 0
try:
if options.interactive:
seed_tasks, cleanup_tasks = self.interactive(
seed_tasks, cleanup_tasks)
if seed_tasks:
print '========== Seeding tasks =========='
print 'Start seeding process (%d task%s)' % (
len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
seed(seed_tasks,
progress_logger=logger,
dry_run=options.dry_run,
concurrency=options.concurrency,
cache_locker=cache_locker,
skip_geoms_for_last_levels=options.geom_levels)
if cleanup_tasks:
print '========== Cleanup tasks =========='
print 'Start cleanup process (%d task%s)' % (len(
cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
logger = ProgressLog(verbose=options.quiet == 0,
silent=options.quiet >= 2)
cleanup(cleanup_tasks,
verbose=options.quiet == 0,
dry_run=options.dry_run,
concurrency=options.concurrency,
progress_logger=logger,
skip_geoms_for_last_levels=options.geom_levels)
except KeyboardInterrupt:
print '\nexiting...'
return 2
def test2():
with local_base_config(conf2):
pool2 = EventletPool(5)
list(pool2.imap(check2, range(200)))
def test1():
with local_base_config(conf1):
pool1 = EventletPool(5)
list(pool1.imap(check1, range(200)))
def run(self):
with local_base_config(self.conf):
try:
self.work_loop()
except KeyboardInterrupt:
return
def run(self):
with local_base_config(self.conf):
try:
self.work_loop()
except KeyboardInterrupt:
return
def test_seed_dry_run(self):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
seed(tasks, dry_run=True)
cleanup(cleanup_tasks, verbose=False, dry_run=True)
grids = {}
for cache in caches.values():
grids.update(cache.grid_confs())
grids = dict(grids)
if options.grid_name:
options.grid_name = options.grid_name.lower()
# ignore case for keys
grids = dict((key.lower(), value) for (key, value) in grids.iteritems())
if not grids.get(options.grid_name, False):
print "grid not found: %s" % (options.grid_name,)
sys.exit(1)
coverage = None
if options.coverage and options.seed_config:
with local_base_config(proxy_configuration.base_config):
try:
seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
except SeedConfigurationError, e:
print >> sys.stderr, "ERROR: invalid configuration (see above)"
sys.exit(2)
if not isinstance(seed_conf, SeedingConfiguration):
print "Old seed configuration format not supported"
sys.exit(1)
coverage = seed_conf.coverage(options.coverage)
coverage.name = options.coverage
elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
print "--coverage and --seed-conf can only be used together"
def test_active_seed_tasks(self):
with local_base_config(self.mapproxy_conf.base_config):
seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
assert len(seed_conf.seed_tasks_names()) == 5
assert len(seed_conf.seeds()) == 4
def test2():
with local_base_config(conf2):
pool2 = EventletPool(5)
list(pool2.imap(check2, range(200)))
def test1():
with local_base_config(conf1):
pool1 = EventletPool(5)
list(pool1.imap(check1, range(200)))
