Beispiel #1
0
    def test_cleanup_remove_all(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['remove_all'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((1, 0, 1))
        self.make_tile((0, 1, 1))
        self.make_tile((1, 1, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['00', '01', '02', '03'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert not self.tile_exists((1, 0, 1))
        assert not self.tile_exists((0, 1, 1))
        assert not self.tile_exists((1, 1, 1))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert self.tile_exists((0, 0, 3))

        # remove_all should remove the whole directory
        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['00', '02', '03'])
Beispiel #2
0
    def test_seed_refresh_remove_before_from_file(self):
        # tile already there but too old, will be refreshed and removed
        t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
        with tmp_image((256, 256), format='png') as img:
            img_data = img.read()
            expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                                  '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                                  '&width=256&height=128&srs=EPSG:4326'},
                            {'body': img_data, 'headers': {'content-type': 'image/png'}})
            with mock_httpd(('localhost', 42423), [expected_req]):
                # touch the seed_conf file and refresh everything
                timestamp = time.time() - 60
                os.utime(self.seed_conf_file, (timestamp, timestamp))
                
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    tasks = seed_conf.seeds(['refresh_from_file'])

                    seed(tasks, dry_run=False)

                assert os.path.exists(t000)
                assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
        
                # now touch the seed_conf again and remove everything
                os.utime(self.seed_conf_file, None)
                with local_base_config(self.mapproxy_conf.base_config):
                    seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                    cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
                    cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #3
0
    def test_reseed_remove_before(self):
        # tile already there but too old
        t000 = self.make_tile((0, 0, 0),
                              timestamp=time.time() - (60 * 60 * 25))
        # old tile outside the seed view (should be removed)
        t001 = self.make_tile((0, 0, 1),
                              timestamp=time.time() - (60 * 60 * 25))
        assert os.path.exists(t000)
        assert os.path.exists(t001)
        with tmp_image((256, 256), format='png') as img:
            img_data = img.read()
            expected_req = ({
                'path':
                r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                '&width=256&height=128&srs=EPSG:4326'
            }, {
                'body': img_data,
                'headers': {
                    'content-type': 'image/png'
                }
            })
            with mock_httpd(('localhost', 42423), [expected_req]):
                seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                                 self.mapproxy_conf)
                tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups()
                seed(tasks, dry_run=False)
                cleanup(cleanup_tasks, verbose=False, dry_run=False)

        assert os.path.exists(t000)
        assert os.path.getmtime(t000) - 5 < time.time(
        ) < os.path.getmtime(t000) + 5
        assert not os.path.exists(t001)
Beispiel #4
0
 def test_reseed_uptodate(self):
     # tile already there.
     self.make_tile((0, 0, 0))
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
     seed(tasks, dry_run=False)
     cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #5
0
    def test_cleanup_remove_all(self):
        seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                         self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['remove_all'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((1, 0, 1))
        self.make_tile((0, 1, 1))
        self.make_tile((1, 1, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'),
                            ['00', '01', '02', '03'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert not self.tile_exists((1, 0, 1))
        assert not self.tile_exists((0, 1, 1))
        assert not self.tile_exists((1, 1, 1))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert self.tile_exists((0, 0, 3))

        # remove_all should remove the whole directory
        assert_files_in_dir(os.path.join(self.dir, 'cache', 'one_EPSG4326'),
                            ['00', '02', '03'])
Beispiel #6
0
    def test_cleanup_sqlite_remove_all(self):
        seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                         self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all'])

        cache = cleanup_tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile((0, 0, 2)))
        cache.store_tile(self.create_tile((0, 0, 3)))
        assert cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))

        eq_(
            sorted(
                os.listdir(
                    os.path.join(self.dir, 'cache', 'sqlite_cache',
                                 'GLOBAL_GEODETIC'))),
            ['2.mbtile', '3.mbtile'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)

        # 3.mbtile file should be removed completely
        eq_(
            sorted(
                os.listdir(
                    os.path.join(self.dir, 'cache', 'sqlite_cache',
                                 'GLOBAL_GEODETIC'))), ['3.mbtile'])
        assert not cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))
Beispiel #7
0
    def test_cleanup_sqlite(self):
        seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                         self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['sqlite_cache'])

        cache = cleanup_tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile((0, 0, 2)))
        cache.store_tile(self.create_tile((0, 0, 3)))
        assert cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))

        assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache',
                                         'GLOBAL_GEODETIC'),
                            ['2.mbtile', '3.mbtile'],
                            glob='*.mbtile')

        cleanup(cleanup_tasks, verbose=False, dry_run=False)

        # 3.mbtile file is still there
        assert_files_in_dir(os.path.join(self.dir, 'cache', 'sqlite_cache',
                                         'GLOBAL_GEODETIC'),
                            ['2.mbtile', '3.mbtile'],
                            glob='*.mbtile')
        assert cache.is_cached(Tile((0, 0, 2)))
        assert not cache.is_cached(Tile((0, 0, 3)))
Beispiel #8
0
    def test_cleanup_mbtiles(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])

        cache = tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile())

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #9
0
 def test_reseed_uptodate(self):
     # tile already there.
     self.make_tile((0, 0, 0))
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
         seed(tasks, dry_run=False)
         cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #10
0
 def test_seed_dry_run(self):
     with local_base_config(self.mapproxy_conf.base_config):
         seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                          self.mapproxy_conf)
         tasks, cleanup_tasks = seed_conf.seeds(['one'
                                                 ]), seed_conf.cleanups()
         seed(tasks, dry_run=True)
         cleanup(cleanup_tasks, verbose=False, dry_run=True)
Beispiel #11
0
    def test_cleanup_mbtiles(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache_refresh']), seed_conf.cleanups(['cleanup_mbtile_cache'])
        
        cache = tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile())

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #12
0
 def test_seed(self):
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
             tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
             seed(tasks, dry_run=False)
             cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #13
0
 def test_seed_mbtile(self):
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=bar&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
             tasks, cleanup_tasks = seed_conf.seeds(['mbtile_cache']), seed_conf.cleanups(['cleanup_mbtile_cache'])
             seed(tasks, dry_run=False)
             cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #14
0
 def test_cleanup_levels(self):
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     cleanup_tasks = seed_conf.cleanups(['cleanup'])
     
     self.make_tile((0, 0, 0))
     self.make_tile((0, 0, 1))
     self.make_tile((0, 0, 2))
     self.make_tile((0, 0, 3))
     
     cleanup(cleanup_tasks, verbose=False, dry_run=False)
     assert not self.tile_exists((0, 0, 0))
     assert not self.tile_exists((0, 0, 1))
     assert self.tile_exists((0, 0, 2))
     assert not self.tile_exists((0, 0, 3))
Beispiel #15
0
    def test_seed_refresh_remove_before_from_file(self):
        # tile already there but old
        t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))

        # mtime is older than tile, no create of the tile
        timestamp = time.time() - (60*60*30)
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            tasks = seed_conf.seeds(['refresh_from_file'])
            seed(tasks, dry_run=False)


        # touch the seed_conf file and refresh everything
        os.utime(self.seed_conf_file, None)
        img_data = create_tmp_image((256, 256), format='png')
        expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                              '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                              '&width=256&height=128&srs=EPSG:4326'},
                        {'body': img_data, 'headers': {'content-type': 'image/png'}})
        with mock_httpd(('localhost', 42423), [expected_req]):
            # touch the seed_conf file and refresh everything
            timestamp = time.time() - 60
            os.utime(self.seed_conf_file, (timestamp, timestamp))

            with local_base_config(self.mapproxy_conf.base_config):
                seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
                tasks = seed_conf.seeds(['refresh_from_file'])
                seed(tasks, dry_run=False)

                assert os.path.exists(t000)
                assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5

        # mtime is older than tile, no cleanup
        timestamp = time.time() - 5
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
            cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert os.path.exists(t000)

        # now touch the seed_conf again and remove everything
        timestamp = time.time() + 5
        os.utime(self.seed_conf_file, (timestamp, timestamp))
        with local_base_config(self.mapproxy_conf.base_config):
            seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
            cleanup_tasks = seed_conf.cleanups(['remove_from_file'])
            cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert not os.path.exists(t000)
Beispiel #16
0
    def test_cleanup_levels(self):
        seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                         self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['cleanup'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert not self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert not self.tile_exists((0, 0, 3))
Beispiel #17
0
    def test_cleanup_levels(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['cleanup'])

        self.make_tile((0, 0, 0))
        self.make_tile((0, 0, 1))
        self.make_tile((0, 0, 2))
        self.make_tile((0, 0, 3))

        cleanup(cleanup_tasks, verbose=False, dry_run=False)
        assert not self.tile_exists((0, 0, 0))
        assert not self.tile_exists((0, 0, 1))
        assert self.tile_exists((0, 0, 2))
        assert not self.tile_exists((0, 0, 3))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'one_EPSG4326'))),
            ['02'])
Beispiel #18
0
    def test_cleanup_sqlite_remove_all(self):
        seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
        cleanup_tasks = seed_conf.cleanups(['sqlite_cache_remove_all'])

        cache = cleanup_tasks[0].tile_manager.cache
        cache.store_tile(self.create_tile((0, 0, 2)))
        cache.store_tile(self.create_tile((0, 0, 3)))
        assert cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))

        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
            ['2.mbtile', '3.mbtile'])

        cleanup(cleanup_tasks, verbose=False, dry_run=False)

        # 3.mbtile file should be removed completely
        eq_(sorted(os.listdir(os.path.join(self.dir, 'cache', 'sqlite_cache', 'GLOBAL_GEODETIC'))),
            ['3.mbtile'])
        assert not cache.is_cached(Tile((0, 0, 2)))
        assert cache.is_cached(Tile((0, 0, 3)))
Beispiel #19
0
    def test_seed_skip_uncached(self):
        with tmp_image((256, 256), format='png') as img:
            img_data = img.read()
            with local_base_config(self.mapproxy_conf.base_config):
                expected_req = ({
                    'path':
                    r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                    '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                    '&width=256&height=128&srs=EPSG:4326'
                }, {
                    'body': img_data,
                    'headers': {
                        'content-type': 'image/png'
                    }
                })
                seed_conf = load_seed_tasks_conf(self.seed_conf_file,
                                                 self.mapproxy_conf)
                tasks, cleanup_tasks = seed_conf.seeds(
                    ['one']), seed_conf.cleanups()

                # tile not in cache => skipped by seeder
                seed(tasks, dry_run=False, skip_uncached=True)
                assert not self.tile_exists((0, 0, 0))

                with mock_httpd(('localhost', 42423), [expected_req]):
                    # force tile generation in cache (via skip_uncached=False)
                    seed(tasks, dry_run=False, skip_uncached=False)
                assert self.tile_exists((0, 0, 0))

                # no refresh since tile is not older than 1 day (cf. config seed.yaml)
                seed(tasks, dry_run=False, skip_uncached=True)

                # create stale tile (older than 1 day)
                self.make_tile((0, 0, 0),
                               timestamp=time.time() - (60 * 60 * 25))
                with mock_httpd(('localhost', 42423), [expected_req]):
                    # check that old tile in cache is refreshed
                    seed(tasks, dry_run=False, skip_uncached=True)
                assert self.tile_exists((0, 0, 0))
                cleanup(cleanup_tasks, verbose=False, dry_run=False)
Beispiel #20
0
 def test_reseed_remove_before(self):
     # tile already there but too old
     t000 = self.make_tile((0, 0, 0), timestamp=time.time() - (60*60*25))
     # old tile outside the seed view (should be removed)
     t001 = self.make_tile((0, 0, 1), timestamp=time.time() - (60*60*25))
     assert os.path.exists(t000)
     assert os.path.exists(t001)
     with tmp_image((256, 256), format='png') as img:
         img_data = img.read()
         expected_req = ({'path': r'/service?LAYERS=foo&SERVICE=WMS&FORMAT=image%2Fpng'
                               '&REQUEST=GetMap&VERSION=1.1.1&bbox=-180.0,-90.0,180.0,90.0'
                               '&width=256&height=128&srs=EPSG:4326'},
                         {'body': img_data, 'headers': {'content-type': 'image/png'}})
         with mock_httpd(('localhost', 42423), [expected_req]):
             seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
             tasks, cleanup_tasks = seed_conf.seeds(), seed_conf.cleanups()
             seed(tasks, dry_run=False)
             cleanup(cleanup_tasks, verbose=False, dry_run=False)
     
     assert os.path.exists(t000)
     assert os.path.getmtime(t000) - 5 < time.time() < os.path.getmtime(t000) + 5
     assert not os.path.exists(t001)
Beispiel #21
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        if options.duration:
            # calls with --duration are handled in call_with_duration
            sys.exit(self.call_with_duration(options, args))

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        progress = None
        if options.continue_seed or options.progress_file:
            if not options.progress_file:
                options.progress_file = '.mapproxy_seed_progress'
            progress = ProgressStore(options.progress_file,
                                     continue_seed=options.continue_seed)

        if options.reseed_file:
            if not os.path.exists(options.reseed_file):
                # create --reseed-file if missing
                with open(options.reseed_file, 'w'):
                    pass
            else:
                if progress and not os.path.exists(options.progress_file):
                    # we have an existing --reseed-file but no --progress-file
                    # meaning the last seed call was completed
                    if options.reseed_interval and (
                            os.path.getmtime(options.reseed_file) >
                        (time.time() - options.reseed_interval)):
                        print("no need for re-seeding")
                        sys.exit(1)
                    os.utime(options.reseed_file, (time.time(), time.time()))

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels,
                         skip_uncached=options.skip_uncached)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
Beispiel #22
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                                         continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
Beispiel #23
0
class SeedScript(object):
    usage = "usage: %prog [options] seed_conf"
    parser = OptionParser(usage)
    parser.add_option("-q",
                      "--quiet",
                      action="count",
                      dest="quiet",
                      default=0,
                      help="don't print status messages to stdout")
    parser.add_option("-s",
                      "--seed-conf",
                      dest="seed_file",
                      default=None,
                      help="seed configuration")
    parser.add_option("-f",
                      "--proxy-conf",
                      dest="conf_file",
                      default=None,
                      help="proxy configuration")
    parser.add_option("-c",
                      "--concurrency",
                      type="int",
                      dest="concurrency",
                      default=2,
                      help="number of parallel seed processes")
    parser.add_option("-n",
                      "--dry-run",
                      action="store_true",
                      dest="dry_run",
                      default=False,
                      help="do not seed, just print output")
    parser.add_option("-l",
                      "--skip-geoms-for-last-levels",
                      type="int",
                      dest="geom_levels",
                      default=0,
                      metavar="N",
                      help="do not check for intersections between tiles"
                      " and seed geometries on the last N levels")
    parser.add_option("--summary",
                      action="store_true",
                      dest="summary",
                      default=False,
                      help="print summary with all seeding tasks and exit."
                      " does not seed anything.")
    parser.add_option(
        "-i",
        "--interactive",
        action="store_true",
        dest="interactive",
        default=False,
        help="print each task description and ask if it should be seeded")

    parser.add_option(
        "--seed",
        action="append",
        dest="seed_names",
        metavar='task1,task2,...',
        help="seed only the named tasks. cleanup is disabled unless "
        "--cleanup is used. use ALL to select all tasks")

    parser.add_option(
        "--cleanup",
        action="append",
        dest="cleanup_names",
        metavar='task1,task2,...',
        help="cleanup only the named tasks. seeding is disabled unless "
        "--seed is used. use ALL to select all tasks")

    parser.add_option(
        "--use-cache-lock",
        action="store_true",
        default=False,
        help="use locking to prevent multiple mapproxy-seed calls "
        "to seed the same cache")

    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging()

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError, ex:
            print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        with local_base_config(mapproxy_conf.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError, ex:
                print "error in configuration: " + '\n\t'.join(
                    str(ex).split('\n'))
                sys.exit(2)

            if options.summary:
                print '========== Seeding tasks =========='
                for task in seed_tasks:
                    print format_seed_task(task)
                print '========== Cleanup tasks =========='
                for task in cleanup_tasks:
                    print format_cleanup_task(task)
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print '========== Seeding tasks =========='
                    print 'Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print '========== Cleanup tasks =========='
                    print 'Start cleanup process (%d task%s)' % (len(
                        cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except KeyboardInterrupt:
                print '\nexiting...'
                return 2
Beispiel #24
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error('missing seed_conf file as last argument or --seed-conf option')
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                    continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print('Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
                        progress_store=progress)
                    seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
                         concurrency=options.concurrency, cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' % (
                        len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
                    cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
                            concurrency=options.concurrency, progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
Beispiel #25
0
 def test_seed_dry_run(self):
     seed_conf  = load_seed_tasks_conf(self.seed_conf_file, self.mapproxy_conf)
     tasks, cleanup_tasks = seed_conf.seeds(['one']), seed_conf.cleanups()
     seed(tasks, dry_run=True)
     cleanup(cleanup_tasks, verbose=False, dry_run=True)