Ejemplo n.º 1
0
    def test_seed_refresh_remove_before_from_file(self):
        # tile already there but too old, will be refreshed and removed
        t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
        with tmp_image((256, 256), format='png') as img:
            img_data = img.read()
            expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                                  '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                                  '&width=256&height=128&srs=EPSG:4326'},
                            {'body': img_data, 'headers': {'content-type': 'image/png'}})
            with mock_httpd(('localhost', 42423), [expected_req]):
                # touch the seed_conf file and refresh everything
                timestamp = time.time() - 60
                os.utime(self.seed_conf_file, (timestamp, timestamp))
                
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    tasks = seed_conf.seeds(['refresh_from_file'])

                    seed(tasks, dry_run=False)

                assert os.path.exists(t000)
                assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
        
                # now touch the seed_conf again and remove everything
                os.utime(self.seed_conf_file, None)
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
                    cleanup(cleanup_tasks, verbose=False, dry_run=False)
Ejemplo n.º 2
0
    def __call__(self, environ, start_response):
        resp = None
        req = Request(environ)

        with local_base_config(self.base_config):
            match = self.handler_path_re.match(req.path)
            if match:
                handler_name = match.group(1)
                if handler_name in self.handlers:
                    try:
                        resp = self.handlers[handler_name].handle(req)
                    except Exception, ex:
                        if self.base_config.debug_mode:
                            raise
                        else:
                            log_wsgiapp.fatal('fatal error in %s for %s %s',
                                              handler_name,
                                              environ.get('PATH_INFO'),
                                              environ.get('QUERY_STRING'),
                                              exc_info=True)
                            import traceback
                            traceback.print_exc(file=environ['wsgi.errors'])
                            resp = Response('internal error', status=500)
            if resp is None:
                if req.path in ('', '/'):
                    resp = self.welcome_response(req.script_url)
                else:
                    resp = Response('not found',
                                    mimetype='text/plain',
                                    status=404)
            return resp(environ, start_response)
Ejemplo n.º 3
0
    def __call__(self, environ, start_response):
        resp = None
        req = Request(environ)

        with local_base_config(self.base_config):
            match = self.handler_path_re.match(req.path)
            if match:
                handler_name = match.group(1)
                if handler_name in self.handlers:
                    try:
                        resp = self.handlers[handler_name].handle(req)
                    except Exception, ex:
                        if self.base_config.debug_mode:
                            raise
                        else:
                            log_wsgiapp.fatal('fatal error in %s for %s %s',
                                handler_name, environ.get('PATH_INFO'), environ.get('QUERY_STRING'), exc_info=True)
                            import traceback
                            traceback.print_exc(file=environ['wsgi.errors'])
                            resp = Response('internal error', status=500)
            if resp is None:
                if req.path in ('', '/'):
                    resp = self.welcome_response(req.script_url)
                else:
                    resp = Response('not found', mimetype='text/plain', status=404)
            return resp(environ, start_response)
Ejemplo n.º 4
0
 def test_reseed_uptodate(self):
     # tile already there.
     self.make_tile((0, 0, 0))
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
         seed(tasks, dry_run=False)
         cleanup(cleanup_tasks, verbose=False, dry_run=False)
Ejemplo n.º 5
0
 def call(*args):
     with local_base_config(self.base_config):
         try:
             return func(*args)
         except Exception:
             if use_result_objects:
                 return sys.exc_info()
             else:
                 raise
Ejemplo n.º 6
0
 def call(*args):
     with local_base_config(self.base_config):
         try:
             return func(*args)
         except Exception:
             if use_result_objects:
                 return sys.exc_info()
             else:
                 raise
Ejemplo n.º 7
0
 def test_seed(self):
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             with local_base_config(self.mapproxy_conf.base_config):
                 seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                 tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
                 seed(tasks, dry_run=False)
                 cleanup(cleanup_tasks, verbose=False, dry_run=False)
Ejemplo n.º 8
0
 def run(self):
     with local_base_config(self.base_config):
         while True:
             task = self.task_queue.get()
             if task is None:
                 self.task_queue.task_done()
                 break
             exec_id, func, args = task
             try:
                 result = func(*args)
             except Exception:
                 result = sys.exc_info()
             self.result_queue.put((exec_id, result))
             self.task_queue.task_done()
Ejemplo n.º 9
0
 def run(self):
     with local_base_config(self.base_config):
         while True:
             task = self.task_queue.get()
             if task is None:
                 self.task_queue.task_done()
                 break
             exec_id, func, args = task
             try:
                 result = func(*args)
             except Exception:
                 result = sys.exc_info()
             self.result_queue.put((exec_id, result))
             self.task_queue.task_done()
Ejemplo n.º 10
0
        for cache in caches.values():
            grids.update(cache.grid_confs())
        grids = dict(grids)

    if options.grid_name:
        options.grid_name = options.grid_name.lower()
        # ignore case for keys
        grids = dict(
            (key.lower(), value) for (key, value) in grids.iteritems())
        if not grids.get(options.grid_name, False):
            print 'grid not found: %s' % (options.grid_name, )
            sys.exit(1)

    coverage = None
    if options.coverage and options.seed_config:
        with local_base_config(proxy_configuration.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config,
                                                 proxy_configuration)
            except SeedConfigurationError, e:
                print >> sys.stderr, 'ERROR: invalid configuration (see above)'
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print 'Old seed configuration format not supported'
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

    elif (options.coverage
Ejemplo n.º 11
0
class SeedScript(object):
    usage = "usage: %prog [options] seed_conf"
    parser = OptionParser(usage)
    parser.add_option("-q",
                      "--quiet",
                      action="count",
                      dest="quiet",
                      default=0,
                      help="don't print status messages to stdout")
    parser.add_option("-s",
                      "--seed-conf",
                      dest="seed_file",
                      default=None,
                      help="seed configuration")
    parser.add_option("-f",
                      "--proxy-conf",
                      dest="conf_file",
                      default=None,
                      help="proxy configuration")
    parser.add_option("-c",
                      "--concurrency",
                      type="int",
                      dest="concurrency",
                      default=2,
                      help="number of parallel seed processes")
    parser.add_option("-n",
                      "--dry-run",
                      action="store_true",
                      dest="dry_run",
                      default=False,
                      help="do not seed, just print output")
    parser.add_option("-l",
                      "--skip-geoms-for-last-levels",
                      type="int",
                      dest="geom_levels",
                      default=0,
                      metavar="N",
                      help="do not check for intersections between tiles"
                      " and seed geometries on the last N levels")
    parser.add_option("--summary",
                      action="store_true",
                      dest="summary",
                      default=False,
                      help="print summary with all seeding tasks and exit."
                      " does not seed anything.")
    parser.add_option(
        "-i",
        "--interactive",
        action="store_true",
        dest="interactive",
        default=False,
        help="print each task description and ask if it should be seeded")

    parser.add_option(
        "--seed",
        action="append",
        dest="seed_names",
        metavar='task1,task2,...',
        help="seed only the named tasks. cleanup is disabled unless "
        "--cleanup is used. use ALL to select all tasks")

    parser.add_option(
        "--cleanup",
        action="append",
        dest="cleanup_names",
        metavar='task1,task2,...',
        help="cleanup only the named tasks. seeding is disabled unless "
        "--seed is used. use ALL to select all tasks")

    parser.add_option(
        "--use-cache-lock",
        action="store_true",
        default=False,
        help="use locking to prevent multiple mapproxy-seed calls "
        "to seed the same cache")

    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging()

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError, ex:
            print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        with local_base_config(mapproxy_conf.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError, ex:
                print "error in configuration: " + '\n\t'.join(
                    str(ex).split('\n'))
                sys.exit(2)

            if options.summary:
                print '========== Seeding tasks =========='
                for task in seed_tasks:
                    print format_seed_task(task)
                print '========== Cleanup tasks =========='
                for task in cleanup_tasks:
                    print format_cleanup_task(task)
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print '========== Seeding tasks =========='
                    print 'Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print '========== Cleanup tasks =========='
                    print 'Start cleanup process (%d task%s)' % (len(
                        cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except KeyboardInterrupt:
                print '\nexiting...'
                return 2
Ejemplo n.º 12
0
 def test2():
     with local_base_config(conf2):
         pool2 = EventletPool(5)
         list(pool2.imap(check2, range(200)))
Ejemplo n.º 13
0
 def test1():
     with local_base_config(conf1):
         pool1 = EventletPool(5)
         list(pool1.imap(check1, range(200)))
Ejemplo n.º 14
0
 def run(self):
     with local_base_config(self.conf):
         try:
             self.work_loop()
         except KeyboardInterrupt:
             return
Ejemplo n.º 15
0
 def run(self):
     with local_base_config(self.conf):
         try:
             self.work_loop()
         except KeyboardInterrupt:
             return
Ejemplo n.º 16
0
 def test_seed_dry_run(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
         seed(tasks, dry_run=True)
         cleanup(cleanup_tasks, verbose=False, dry_run=True)
Ejemplo n.º 17
0
        grids = {}
        for cache in caches.values():
            grids.update(cache.grid_confs())
        grids = dict(grids)

    if options.grid_name:
        options.grid_name = options.grid_name.lower()
        # ignore case for keys
        grids = dict((key.lower(), value) for (key, value) in grids.iteritems())
        if not grids.get(options.grid_name, False):
            print "grid not found: %s" % (options.grid_name,)
            sys.exit(1)

    coverage = None
    if options.coverage and options.seed_config:
        with local_base_config(proxy_configuration.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_config, proxy_configuration)
            except SeedConfigurationError, e:
                print >> sys.stderr, "ERROR: invalid configuration (see above)"
                sys.exit(2)

            if not isinstance(seed_conf, SeedingConfiguration):
                print "Old seed configuration format not supported"
                sys.exit(1)

            coverage = seed_conf.coverage(options.coverage)
            coverage.name = options.coverage

    elif (options.coverage and not options.seed_config) or (not options.coverage and options.seed_config):
        print "--coverage and --seed-conf can only be used together"
Ejemplo n.º 18
0
 def test_active_seed_tasks(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         assert len(seed_conf.seed_tasks_names()) == 5
         assert len(seed_conf.seeds()) == 4
Ejemplo n.º 19
0
 def test2():
     with local_base_config(conf2):
         pool2 = EventletPool(5)
         list(pool2.imap(check2, range(200)))
Ejemplo n.º 20
0
 def test1():
     with local_base_config(conf1):
         pool1 = EventletPool(5)
         list(pool1.imap(check1, range(200)))