def test_seed_refresh_remove_before_from_file(self): # tile already there but too old, will be refreshed and removed t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25)) with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): # touch the seed_conf file and refresh everything timestamp = time.time() - 60 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5 # now touch the seed_conf again and remove everything os.utime(self.seed_conf_file, None) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def __call__(self, environ, start_response): resp = None req = Request(environ) with local_base_config(self.base_config): match = self.handler_path_re.match(req.path) if match: handler_name = match.group(1) if handler_name in self.handlers: try: resp = self.handlers[handler_name].handle(req) except Exception, ex: if self.base_config.debug_mode: raise else: log_wsgiapp.fatal('fatal error in %s for %s %s', handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True) import traceback traceback.print_exc(file=environ['wsgi.errors']) resp = Response('internal error', status=500) if resp is None: if req.path in ('', '/'): resp = self.welcome_response(req.script_url) else: resp = Response('not found', mimetype='text/plain', status=404) return resp(environ, start_response)
def test_reseed_uptodate(self): # tile already there. self.make_tile((0, 0, 0)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def call(*args): with local_base_config(self.base_config): try: return func(*args) except Exception: if use_result_objects: return sys.exc_info() else: raise
def test_seed(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def run(self): with local_base_config(self.base_config): while True: task = self.task_queue.get() if task is None: self.task_queue.task_done() break exec_id, func, args = task try: result = func(*args) except Exception: result = sys.exc_info() self.result_queue.put((exec_id, result)) self.task_queue.task_done()
for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict( (key.lower(), value) for (key, value) in grids.iteritems()) if not grids.get(options.grid_name, False): print 'grid not found: %s' % (options.grid_name, ) sys.exit(1) coverage = None if options.coverage and options.seed_config: with local_base_config(proxy_configuration.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError, e: print >> sys.stderr, 'ERROR: invalid configuration (see above)' sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print 'Old seed configuration format not supported' sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage
class SeedScript(object): usage = "usage: %prog [options] seed_conf" parser = OptionParser(usage) parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, help="don't print status messages to stdout") parser.add_option("-s", "--seed-conf", dest="seed_file", default=None, help="seed configuration") parser.add_option("-f", "--proxy-conf", dest="conf_file", default=None, help="proxy configuration") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=2, help="number of parallel seed processes") parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, help="do not seed, just print output") parser.add_option("-l", "--skip-geoms-for-last-levels", type="int", dest="geom_levels", default=0, metavar="N", help="do not check for intersections between tiles" " and seed geometries on the last N levels") parser.add_option("--summary", action="store_true", dest="summary", default=False, help="print summary with all seeding tasks and exit." " does not seed anything.") parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False, help="print each task description and ask if it should be seeded") parser.add_option( "--seed", action="append", dest="seed_names", metavar='task1,task2,...', help="seed only the named tasks. cleanup is disabled unless " "--cleanup is used. use ALL to select all tasks") parser.add_option( "--cleanup", action="append", dest="cleanup_names", metavar='task1,task2,...', help="cleanup only the named tasks. seeding is disabled unless " "--seed is used. use ALL to select all tasks") parser.add_option( "--use-cache-lock", action="store_true", default=False, help="use locking to prevent multiple mapproxy-seed calls " "to seed the same cache") def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging() try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError, ex: print "ERROR: " + '\n\t'.join(str(ex).split('\n')) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None with local_base_config(mapproxy_conf.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError, ex: print "error in configuration: " + '\n\t'.join( str(ex).split('\n')) sys.exit(2) if options.summary: print '========== Seeding tasks ==========' for task in seed_tasks: print format_seed_task(task) print '========== Cleanup tasks ==========' for task in cleanup_tasks: print format_cleanup_task(task) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print '========== Seeding tasks ==========' print 'Start seeding process (%d task%s)' % ( len(seed_tasks), 's' if len(seed_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print '========== Cleanup tasks ==========' print 'Start cleanup process (%d task%s)' % (len( cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except KeyboardInterrupt: print '\nexiting...' return 2
def test2(): with local_base_config(conf2): pool2 = EventletPool(5) list(pool2.imap(check2, range(200)))
def test1(): with local_base_config(conf1): pool1 = EventletPool(5) list(pool1.imap(check1, range(200)))
def run(self): with local_base_config(self.conf): try: self.work_loop() except KeyboardInterrupt: return
def test_seed_dry_run(self): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=True) cleanup(cleanup_tasks, verbose=False, dry_run=True)
grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict((key.lower(), value) for (key, value) in grids.iteritems()) if not grids.get(options.grid_name, False): print "grid not found: %s" % (options.grid_name,) sys.exit(1) coverage = None if options.coverage and options.seed_config: with local_base_config(proxy_configuration.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError, e: print >> sys.stderr, "ERROR: invalid configuration (see above)" sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print "Old seed configuration format not supported" sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print "--coverage and --seed-conf can only be used together"
def test_active_seed_tasks(self): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) assert len(seed_conf.seed_tasks_names()) == 5 assert len(seed_conf.seeds()) == 4