Beispiel #1
0
    def test_seed_refresh_remove_before_from_file(self):
        # tile already there but old
        t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))

        # mtime is older than tile, no create of the tile
        timestamp = time.time() - (60*60*30)
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            tasks = seed_conf.seeds(['refresh_from_file'])
            seed(tasks, dry_run=False)


        # touch the seed_conf file and refresh everything
        os.utime(self.seed_conf_file, None)
        img_data = create_tmp_image((256, 256), format='png')
        expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                              '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                              '&width=256&height=128&srs=EPSG:4326'},
                        {'body': img_data, 'headers': {'content-type': 'image/png'}})
        with mock_httpd(('localhost', 42423), [expected_req]):
            # touch the seed_conf file and refresh everything
            timestamp = time.time() - 60
            os.utime(self.seed_conf_file, (timestamp, timestamp))

            with local_base_config(self.mapproxy_conf.base_config):
                seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                tasks = seed_conf.seeds(['refresh_from_file'])
                seed(tasks, dry_run=False)

                assert os.path.exists(t000)
                assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5

        # mtime is older than tile, no cleanup
        timestamp = time.time() - 5
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
            cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert os.path.exists(t000)

        # now touch the seed_conf again and remove everything
        timestamp = time.time() + 5
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
            cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert not os.path.exists(t000)
Beispiel #2
0
    def __call__(self, environ, start_response):
        resp = None
        req = Request(environ)

        if self.cors_origin:
            orig_start_response = start_response
            def start_response(status, headers, exc_info=None):
                headers.append(('Access-control-allow-origin', self.cors_origin))
                return orig_start_response(status, headers, exc_info)

        with local_base_config(self.base_config):
            match = self.handler_path_re.match(req.path)
            if match:
                handler_name = match.group(1)
                if handler_name in self.handlers:
                    try:
                        resp = self.handlers[handler_name].handle(req)
                    except Exception:
                        if self.base_config.debug_mode:
                            raise
                        else:
                            log_wsgiapp.fatal('fatal error in %s for %s?%s',
                                handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True)
                            import traceback
                            traceback.print_exc(file=environ['wsgi.errors'])
                            resp = Response('internal error', status=500)
            if resp is None:
                if req.path in ('', '/'):
                    resp = self.welcome_response(req.script_url)
                else:
                    resp = Response('not found', mimetype='text/plain', status=404)
            return resp(environ, start_response)
Beispiel #3
0
 def test_reseed_uptodate(self):
     # tile already there.
     self.make_tile((0, 0, 0))
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
         seed(tasks, dry_run=False)
         cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #4
0
 def run(self):
     with local_base_config(self.conf):
         try:
             self.work_loop()
         except KeyboardInterrupt:
             return
         except BackoffError:
             return
Beispiel #5
0
 def call(*args):
     with local_base_config(self.base_config):
         try:
             return func(*args)
         except Exception:
             if use_result_objects:
                 return sys.exc_info()
             else:
                 raise
Beispiel #6
0
 def test_seed(self):
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             with local_base_config(self.mapproxy_conf.base_config):
                 seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                 tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
                 seed(tasks, dry_run=False)
                 cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #7
0
 def run(self):
     with local_base_config(self.base_config):
         while True:
             task = self.task_queue.get()
             if task is None:
                 self.task_queue.task_done()
                 break
             exec_id, func, args = task
             try:
                 result = func(*args)
             except Exception:
                 result = sys.exc_info()
             self.result_queue.put((exec_id, result))
             self.task_queue.task_done()
Beispiel #8
0
        grids = {}
        for cache in caches.values():
            grids.update(cache.grid_confs())
        grids = dict(grids)

    if options.grid_name:
        options.grid_name = options.grid_name.lower()
        # ignore case for keys
        grids = dict((key.lower(), value) for (key, value) in grids.iteritems())
        if not grids.get(options.grid_name, False):
            print 'grid not found: %s' % (options.grid_name,)
            sys.exit(1)

    coverage = None
    if options.coverage and options.seed_config:
        with local_base_config(proxy_configuration.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
            except SeedConfigurationError, e:
                print >>sys.stderr, 'ERROR: invalid configuration (see above)'
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print 'Old seed configuration format not supported'
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

    elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
        print '--coverage and --seed-conf can only be used together'
Beispiel #9
0
def grids_command(args=None):
    parser = optparse.OptionParser("%prog grids [options] mapproxy_conf")
    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
        help="MapProxy configuration.")
    parser.add_option("-g", "--grid", dest="grid_name",
        help="Display only information about the specified grid.")
    parser.add_option("--all", dest="show_all", action="store_true", default=False,
        help="Show also grids that are not referenced by any cache.")
    parser.add_option("-l", "--list", dest="list_grids", action="store_true", default=False, help="List names of configured grids, which are used by any cache")
    coverage_group = parser.add_option_group("Approximate the number of tiles within a given coverage")
    coverage_group.add_option("-s", "--seed-conf", dest="seed_config", help="Seed configuration, where the coverage is defined")
    coverage_group.add_option("-c", "--coverage-name", dest="coverage", help="Calculate number of tiles when a coverage is given")

    from mapproxy.script.util import setup_logging
    import logging
    setup_logging(logging.WARN)

    if args:
        args = args[1:] # remove script name

    (options, args) = parser.parse_args(args)
    if not options.mapproxy_conf:
        if len(args) != 1:
            parser.print_help()
            sys.exit(1)
        else:
            options.mapproxy_conf = args[0]
    try:
        proxy_configuration = load_configuration(options.mapproxy_conf)
    except IOError as e:
        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
        sys.exit(2)
    except ConfigurationError as e:
        print(e, file=sys.stderr)
        print('ERROR: invalid configuration (see above)', file=sys.stderr)
        sys.exit(2)

    with local_base_config(proxy_configuration.base_config):
        if options.show_all or options.grid_name:
            grids = proxy_configuration.grids
        else:
            caches = proxy_configuration.caches
            grids = {}
            for cache in caches.values():
                grids.update(cache.grid_confs())
            grids = dict(grids)

        if options.grid_name:
            options.grid_name = options.grid_name.lower()
            # ignore case for keys
            grids = dict((key.lower(), value) for (key, value) in iteritems(grids))
            if not grids.get(options.grid_name, False):
                print('grid not found: %s' % (options.grid_name,))
                sys.exit(1)

        coverage = None
        if options.coverage and options.seed_config:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
            except SeedConfigurationError as e:
                print('ERROR: invalid configuration (see above)', file=sys.stderr)
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print('Old seed configuration format not supported')
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

        elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
            print('--coverage and --seed-conf can only be used together')
            sys.exit(1)

        if options.list_grids:
            display_grids_list(grids)
        elif options.grid_name:
            display_grids({options.grid_name: grids[options.grid_name]}, coverage=coverage)
        else:
            display_grids(grids, coverage=coverage)
Beispiel #10
0
 def test2():
     with local_base_config(conf2):
         pool2 = EventletPool(5)
         list(pool2.imap(check2, range(200)))
Beispiel #11
0
 def test1():
     with local_base_config(conf1):
         pool1 = EventletPool(5)
         list(pool1.imap(check1, range(200)))
Beispiel #12
0
 def test2():
     with local_base_config(conf2):
         pool2 = ThreadPool(5)
         list(pool2.imap(check2, list(range(200))))
Beispiel #13
0
 def test1():
     with local_base_config(conf1):
         pool1 = ThreadPool(5)
         list(pool1.imap(check1, list(range(200))))
Beispiel #14
0
 def test_seed_dry_run(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
         seed(tasks, dry_run=True)
         cleanup(cleanup_tasks, verbose=False, dry_run=True)
Beispiel #15
0
 def test_active_seed_tasks(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         assert len(seed_conf.seed_tasks_names()) == 5
         assert len(seed_conf.seeds()) == 5
Beispiel #16
0
def defrag_command(args=None):
    parser = optparse.OptionParser("%prog defrag-compact [options] -f mapproxy_conf")
    parser.add_option("-f", "--mapproxy-conf", dest="mapproxy_conf",
        help="MapProxy configuration.")

    parser.add_option("--min-percent", type=float, default=10.0,
        help="Only defrag if fragmentation is larger (10 means at least 10% of the file does not have to be used)")

    parser.add_option("--min-mb", type=float, default=1.0,
        help="Only defrag if fragmentation is larger (2 means at least 2MB the file does not have to be used)")

    parser.add_option("--dry-run", "-n", action="store_true",
        help="Do not de-fragment, only print output")

    parser.add_option("--caches", dest="cache_names", metavar='cache1,cache2,...',
        help="only defragment the named caches")


    from mapproxy.script.util import setup_logging
    import logging
    setup_logging(logging.INFO, format="[%(asctime)s] %(msg)s")

    if args:
        args = args[1:] # remove script name

    (options, args) = parser.parse_args(args)
    if not options.mapproxy_conf:
        parser.print_help()
        sys.exit(1)

    try:
        proxy_configuration = load_configuration(options.mapproxy_conf)
    except IOError as e:
        print('ERROR: ', "%s: '%s'" % (e.strerror, e.filename), file=sys.stderr)
        sys.exit(2)
    except ConfigurationError as e:
        print(e, file=sys.stderr)
        print('ERROR: invalid configuration (see above)', file=sys.stderr)
        sys.exit(2)


    with local_base_config(proxy_configuration.base_config):
        available_caches = OrderedDict()
        for name, cache_conf in proxy_configuration.caches.items():
            for grid, extent, tile_mgr in cache_conf.caches():
                if isinstance(tile_mgr.cache, (CompactCacheV1, CompactCacheV2)):
                    available_caches.setdefault(name, []).append(tile_mgr.cache)

        if options.cache_names:
            defrag_caches = options.cache_names.split(',')
            missing = set(defrag_caches).difference(available_caches.keys())
            if missing:
                print('unknown caches: %s' % (', '.join(missing), ))
                print('available compact caches: %s' %
                      (', '.join(available_caches.keys()), ))
                sys.exit(1)
        else:
            defrag_caches = None

        for name, caches in available_caches.items():
            if defrag_caches and name not in defrag_caches:
                continue
            for cache in caches:
                logger = DefragLog(name)
                defrag_compact_cache(cache,
                    min_percent=options.min_percent/100,
                    min_bytes=options.min_mb*1024*1024,
                    dry_run=options.dry_run,
                    log_progress=logger,
                )