def test_seed_refresh_remove_before_from_file(self): # tile already there but old t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25)) # mtime is older than tile, no create of the tile timestamp = time.time() - (60*60*30) os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) # touch the seed_conf file and refresh everything os.utime(self.seed_conf_file, None) img_data = create_tmp_image((256, 256), format='png') expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): # touch the seed_conf file and refresh everything timestamp = time.time() - 60 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5 # mtime is older than tile, no cleanup timestamp = time.time() - 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert os.path.exists(t000) # now touch the seed_conf again and remove everything timestamp = time.time() + 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert not os.path.exists(t000)
def __call__(self, environ, start_response): resp = None req = Request(environ) if self.cors_origin: orig_start_response = start_response def start_response(status, headers, exc_info=None): headers.append(('Access-control-allow-origin', self.cors_origin)) return orig_start_response(status, headers, exc_info) with local_base_config(self.base_config): match = self.handler_path_re.match(req.path) if match: handler_name = match.group(1) if handler_name in self.handlers: try: resp = self.handlers[handler_name].handle(req) except Exception: if self.base_config.debug_mode: raise else: log_wsgiapp.fatal('fatal error in %s for %s?%s', handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True) import traceback traceback.print_exc(file=environ['wsgi.errors']) resp = Response('internal error', status=500) if resp is None: if req.path in ('', '/'): resp = self.welcome_response(req.script_url) else: resp = Response('not found', mimetype='text/plain', status=404) return resp(environ, start_response)
def test_reseed_uptodate(self): # tile already there. self.make_tile((0, 0, 0)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def run(self): with local_base_config(self.conf): try: self.work_loop() except KeyboardInterrupt: return except BackoffError: return
def call(*args): with local_base_config(self.base_config): try: return func(*args) except Exception: if use_result_objects: return sys.exc_info() else: raise
def call(func, *args): with local_base_config(self.base_config): try: return func(*args) except Exception: if use_result_objects: return sys.exc_info() else: raise
def test_reseed_uptodate(self): # tile already there. self.make_tile((0, 0, 0)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one' ]), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def test_seed(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326'}, {'body': img_data, 'headers': {'content-type': 'image/png'}}) with mock_httpd(('localhost', 42423), [expected_req]): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=False) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def run(self): with local_base_config(self.base_config): while True: task = self.task_queue.get() if task is None: self.task_queue.task_done() break exec_id, func, args = task try: result = func(*args) except Exception: result = sys.exc_info() self.result_queue.put((exec_id, result)) self.task_queue.task_done()
def test_seed_skip_uncached(self): with tmp_image((256, 256), format='png') as img: img_data = img.read() with local_base_config(self.mapproxy_conf.base_config): expected_req = ({ 'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326' }, { 'body': img_data, 'headers': { 'content-type': 'image/png' } }) seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds( ['one']), seed_conf.cleanups() # tile not in cache => skipped by seeder seed(tasks, dry_run=False, skip_uncached=True) assert not self.tile_exists((0, 0, 0)) with mock_httpd(('localhost', 42423), [expected_req]): # force tile generation in cache (via skip_uncached=False) seed(tasks, dry_run=False, skip_uncached=False) assert self.tile_exists((0, 0, 0)) # no refresh since tile is not older than 1 day (cf. config seed.yaml) seed(tasks, dry_run=False, skip_uncached=True) # create stale tile (older than 1 day) self.make_tile((0, 0, 0), timestamp=time.time() - (60 * 60 * 25)) with mock_httpd(('localhost', 42423), [expected_req]): # check that old tile in cache is refreshed seed(tasks, dry_run=False, skip_uncached=True) assert self.tile_exists((0, 0, 0)) cleanup(cleanup_tasks, verbose=False, dry_run=False)
def __call__(self, environ, start_response): resp = None req = Request(environ) if self.cors_origin: orig_start_response = start_response def start_response(status, headers, exc_info=None): headers.append( ('Access-control-allow-origin', self.cors_origin)) return orig_start_response(status, headers, exc_info) with local_base_config(self.base_config): match = self.handler_path_re.match(req.path) if match: handler_name = match.group(1) if handler_name in self.handlers: try: resp = self.handlers[handler_name].handle(req) except Exception: if self.base_config.debug_mode: raise else: log_wsgiapp.fatal('fatal error in %s for %s?%s', handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True) import traceback traceback.print_exc(file=environ['wsgi.errors']) resp = Response('internal error', status=500) if resp is None: if req.path in ('', '/'): resp = self.welcome_response(req.script_url) else: resp = Response('not found', mimetype='text/plain', status=404) return resp(environ, start_response)
def test_seed_refresh_remove_before_from_file(self): # tile already there but old t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60 * 60 * 25)) # mtime is older than tile, no create of the tile timestamp = time.time() - (60 * 60 * 30) os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) # touch the seed_conf file and refresh everything os.utime(self.seed_conf_file, None) img_data = create_tmp_image((256, 256), format='png') expected_req = ({ 'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng' '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0' '&width=256&height=128&srs=EPSG:4326' }, { 'body': img_data, 'headers': { 'content-type': 'image/png' } }) with mock_httpd(('localhost', 42423), [expected_req]): # touch the seed_conf file and refresh everything timestamp = time.time() - 60 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks = seed_conf.seeds(['refresh_from_file']) seed(tasks, dry_run=False) assert os.path.exists(t000) assert os.path.getmtime(t000) - 5 < time.time( ) < os.path.getmtime(t000) + 5 # mtime is older than tile, no cleanup timestamp = time.time() - 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert os.path.exists(t000) # now touch the seed_conf again and remove everything timestamp = time.time() + 5 os.utime(self.seed_conf_file, (timestamp, timestamp)) with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) cleanup_tasks = seed_conf.cleanups(['remove_from_file']) cleanup(cleanup_tasks, verbose=False, dry_run=False) assert not os.path.exists(t000)
def test_active_seed_tasks(self): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) assert len(seed_conf.seed_tasks_names()) == 5 assert len(seed_conf.seeds()) == 5
def defrag_command(args=None): parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option("--min-percent", type=float, default=10.0, help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)") parser.add_option("--min-mb", type=float, default=1.0, help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)") parser.add_option("--dry-run", "-n", action="store_true", help="Do not de-fragment, only print output") parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...', help="only defragment the named caches") from mapproxy.script.util import setup_logging import logging setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s") if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: parser.print_help() sys.exit(1) try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): available_caches = OrderedDict() for name, cache_conf in proxy_configuration.caches.items(): for grid, extent, tile_mgr in cache_conf.caches(): if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)): available_caches.setdefault(name, []).append(tile_mgr.cache) if options.cache_names: defrag_caches = options.cache_names.split(',') missing = set(defrag_caches).difference(available_caches.keys()) if missing: print('unknown caches: %s' % (', '.join(missing), )) print('available compact caches: %s' % (', '.join(available_caches.keys()), )) sys.exit(1) else: defrag_caches = None for name, caches in available_caches.items(): if defrag_caches and name not in defrag_caches: continue for cache in caches: logger = DefragLog(name) defrag_compact_cache(cache, min_percent=options.min_percent/100, min_bytes=options.min_mb*1024*1024, dry_run=options.dry_run, log_progress=logger, )
def defrag_command(args=None): parser = optparse.OptionParser( "%prog defrag-compact [options] -f mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option( "--min-percent", type=float, default=10.0, help= "Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)" ) parser.add_option( "--min-mb", type=float, default=1.0, help= "Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)" ) parser.add_option("--dry-run", "-n", action="store_true", help="Do not de-fragment, only print output") parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...', help="only defragment the named caches") from mapproxy.script.util import setup_logging import logging setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s") if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: parser.print_help() sys.exit(1) try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): available_caches = OrderedDict() for name, cache_conf in proxy_configuration.caches.items(): for grid, extent, tile_mgr in cache_conf.caches(): if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)): available_caches.setdefault(name, []).append(tile_mgr.cache) if options.cache_names: defrag_caches = options.cache_names.split(',') missing = set(defrag_caches).difference(available_caches.keys()) if missing: print('unknown caches: %s' % (', '.join(missing), )) print('available compact caches: %s' % (', '.join(available_caches.keys()), )) sys.exit(1) else: defrag_caches = None for name, caches in available_caches.items(): if defrag_caches and name not in defrag_caches: continue for cache in caches: logger = DefragLog(name) defrag_compact_cache( cache, min_percent=options.min_percent / 100, min_bytes=options.min_mb * 1024 * 1024, dry_run=options.dry_run, log_progress=logger, )
def run(self): with local_base_config(self.conf): try: self.work_loop() except KeyboardInterrupt: return
grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict((key.lower(), value) for (key, value) in grids.iteritems()) if not grids.get(options.grid_name, False): print 'grid not found: %s' % (options.grid_name,) sys.exit(1) coverage = None if options.coverage and options.seed_config: with local_base_config(proxy_configuration.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError, e: print >>sys.stderr, 'ERROR: invalid configuration (see above)' sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print 'Old seed configuration format not supported' sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print '--coverage and --seed-conf can only be used together'
def test1(): with local_base_config(conf1): pool1 = EventletPool(5) list(pool1.imap(check1, list(range(200))))
def test2(): with local_base_config(conf2): pool2 = EventletPool(5) list(pool2.imap(check2, range(200)))
def test1(): with local_base_config(conf1): pool1 = EventletPool(5) list(pool1.imap(check1, range(200)))
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option( "-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option( "--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option( "-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group( "Approximate the number of tiles within a given coverage") coverage_group.add_option( "-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option( "-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): if options.show_all or options.grid_name: grids = proxy_configuration.grids else: caches = proxy_configuration.caches grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict( (key.lower(), value) for (key, value) in iteritems(grids)) if not grids.get(options.grid_name, False): print('grid not found: %s' % (options.grid_name, )) sys.exit(1) coverage = None if options.coverage and options.seed_config: try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError as e: print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print('Old seed configuration format not supported') sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print('--coverage and --seed-conf can only be used together') sys.exit(1) if options.list_grids: display_grids_list(grids) elif options.grid_name: display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage) else: display_grids(grids, coverage=coverage)
def test1(): with local_base_config(conf1): pool1 = ThreadPool(5) list(pool1.imap(check1, range(200)))
def test2(): with local_base_config(conf2): pool2 = ThreadPool(5) list(pool2.imap(check2, list(range(200))))
def test1(): with local_base_config(conf1): pool1 = ThreadPool(5) list(pool1.imap(check1, list(range(200))))
def test2(): with local_base_config(conf2): pool2 = ThreadPool(5) list(pool2.imap(check2, range(200)))
def grids_command(args=None): parser = optparse.OptionParser("%prog grids [options] mapproxy_conf") parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf", help="MapProxy configuration.") parser.add_option("-g", "--grid", dest="grid_name", help="Display only information about the specified grid.") parser.add_option("--all", dest="show_all", action="store_true", default=False, help="Show also grids that are not referenced by any cache.") parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache") coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage") coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined") coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given") from mapproxy.script.util import setup_logging import logging setup_logging(logging.WARN) if args: args = args[1:] # remove script name (options, args) = parser.parse_args(args) if not options.mapproxy_conf: if len(args) != 1: parser.print_help() sys.exit(1) else: options.mapproxy_conf = args[0] try: proxy_configuration = load_configuration(options.mapproxy_conf) except IOError as e: print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr) sys.exit(2) except ConfigurationError as e: print(e, file=sys.stderr) print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) with local_base_config(proxy_configuration.base_config): if options.show_all or options.grid_name: grids = proxy_configuration.grids else: caches = proxy_configuration.caches grids = {} for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict((key.lower(), value) for (key, value) in iteritems(grids)) if not grids.get(options.grid_name, False): print('grid not found: %s' % (options.grid_name,)) sys.exit(1) coverage = None if options.coverage and options.seed_config: try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError as e: print('ERROR: invalid configuration (see above)', file=sys.stderr) sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print('Old seed configuration format not supported') sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config): print('--coverage and --seed-conf can only be used together') sys.exit(1) if options.list_grids: display_grids_list(grids) elif options.grid_name: display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage) else: display_grids(grids, coverage=coverage)
def test2(): with local_base_config(conf2): pool2 = EventletPool(5) list(pool2.imap(check2, list(range(200))))
for cache in caches.values(): grids.update(cache.grid_confs()) grids = dict(grids) if options.grid_name: options.grid_name = options.grid_name.lower() # ignore case for keys grids = dict( (key.lower(), value) for (key, value) in grids.iteritems()) if not grids.get(options.grid_name, False): print 'grid not found: %s' % (options.grid_name, ) sys.exit(1) coverage = None if options.coverage and options.seed_config: with local_base_config(proxy_configuration.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration) except SeedConfigurationError, e: print >> sys.stderr, 'ERROR: invalid configuration (see above)' sys.exit(2) if not isinstance(seed_conf, SeedingConfiguration): print 'Old seed configuration format not supported' sys.exit(1) coverage = seed_conf.coverage(options.coverage) coverage.name = options.coverage elif (options.coverage
def test_seed_dry_run(self): with local_base_config(self.mapproxy_conf.base_config): seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf) tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups() seed(tasks, dry_run=True) cleanup(cleanup_tasks, verbose=False, dry_run=True)