Exemple #1
0
def cleanup(tasks,
            concurrency=2,
            dry_run=False,
            skip_geoms_for_last_levels=0,
            verbose=True,
            progress_logger=None):
    for task in tasks:
        print format_cleanup_task(task)

        if task.complete_extent:
            if hasattr(task.tile_manager.cache, 'level_location'):
                simple_cleanup(task,
                               dry_run=dry_run,
                               progress_logger=progress_logger)
                continue
            elif hasattr(task.tile_manager.cache, 'remove_level_tiles_before'):
                cache_cleanup(task,
                              dry_run=dry_run,
                              progress_logger=progress_logger)
                continue

        tilewalker_cleanup(
            task,
            dry_run=dry_run,
            concurrency=concurrency,
            skip_geoms_for_last_levels=skip_geoms_for_last_levels,
            progress_logger=progress_logger)
Exemple #2
0
def cleanup(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
               verbose=True, progress_logger=None):
    for task in tasks:
        print format_cleanup_task(task)

        if task.complete_extent:
            if hasattr(task.tile_manager.cache, 'level_location'):
                simple_cleanup(task, dry_run=dry_run, progress_logger=progress_logger)
                continue
            elif hasattr(task.tile_manager.cache, 'remove_level_tiles_before'):
                cache_cleanup(task, dry_run=dry_run, progress_logger=progress_logger)
                continue

        tilewalker_cleanup(task, dry_run=dry_run, concurrency=concurrency,
                         skip_geoms_for_last_levels=skip_geoms_for_last_levels,
                         progress_logger=progress_logger)
Exemple #3
0
def cleanup(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
               verbose=True, progress_logger=None):
    for task in tasks:
        print(format_cleanup_task(task))

        if task.coverage is False:
            continue

        # seed_progress for tilewalker cleanup
        seed_progress = None
        # cleanup_progress for os.walk based cleanup
        cleanup_progress = None
        if progress_logger and progress_logger.progress_store:
            progress_logger.current_task_id = task.id
            start_progress = progress_logger.progress_store.get(task.id)
            seed_progress = SeedProgress(old_progress_identifier=start_progress)
            cleanup_progress = DirectoryCleanupProgress(old_dir=start_progress)

        if task.complete_extent:
            if callable(getattr(task.tile_manager.cache, 'level_location', None)):
                simple_cleanup(task, dry_run=dry_run, progress_logger=progress_logger,
                    cleanup_progress=cleanup_progress)
                continue
            elif callable(getattr(task.tile_manager.cache, 'remove_level_tiles_before', None)):
                cache_cleanup(task, dry_run=dry_run, progress_logger=progress_logger)
                continue

        tilewalker_cleanup(task, dry_run=dry_run, concurrency=concurrency,
                         skip_geoms_for_last_levels=skip_geoms_for_last_levels,
                         progress_logger=progress_logger,
                         seed_progress=seed_progress,
        )
Exemple #4
0
    def interactive(self, seed_tasks, cleanup_tasks):
        selected_seed_tasks = []
        print '========== Select seeding tasks =========='
        for task in seed_tasks:
            print format_seed_task(task)
            if ask_yes_no_question('    Seed this task (y/n)? '):
                selected_seed_tasks.append(task)
        seed_tasks = selected_seed_tasks

        selected_cleanup_tasks = []
        print '========== Select cleanup tasks =========='
        for task in cleanup_tasks:
            print format_cleanup_task(task)
            if ask_yes_no_question('    Cleanup this task (y/n)? '):
                selected_cleanup_tasks.append(task)
        cleanup_tasks = selected_cleanup_tasks
        return seed_tasks, cleanup_tasks
Exemple #5
0
    def interactive(self, seed_tasks, cleanup_tasks):
        selected_seed_tasks = []
        print '========== Select seeding tasks =========='
        for task in seed_tasks:
            print format_seed_task(task)
            if ask_yes_no_question('    Seed this task (y/n)? '):
                selected_seed_tasks.append(task)
        seed_tasks = selected_seed_tasks

        selected_cleanup_tasks = []
        print '========== Select cleanup tasks =========='
        for task in cleanup_tasks:
            print format_cleanup_task(task)
            if ask_yes_no_question('    Cleanup this task (y/n)? '):
                selected_cleanup_tasks.append(task)
        cleanup_tasks = selected_cleanup_tasks
        return seed_tasks, cleanup_tasks
Exemple #6
0
def cleanup(tasks,
            concurrency=2,
            dry_run=False,
            skip_geoms_for_last_levels=0,
            verbose=True,
            progress_logger=None):
    for task in tasks:
        print(format_cleanup_task(task))

        if task.coverage is False:
            continue

        # seed_progress for tilewalker cleanup
        seed_progress = None
        # cleanup_progress for os.walk based cleanup
        cleanup_progress = None
        if progress_logger and progress_logger.progress_store:
            progress_logger.current_task_id = task.id
            start_progress = progress_logger.progress_store.get(task.id)
            seed_progress = SeedProgress(
                old_progress_identifier=start_progress)
            cleanup_progress = DirectoryCleanupProgress(old_dir=start_progress)

        if task.complete_extent:
            if callable(
                    getattr(task.tile_manager.cache, 'level_location', None)):
                simple_cleanup(task,
                               dry_run=dry_run,
                               progress_logger=progress_logger,
                               cleanup_progress=cleanup_progress)
                continue
            elif callable(
                    getattr(task.tile_manager.cache,
                            'remove_level_tiles_before', None)):
                cache_cleanup(task,
                              dry_run=dry_run,
                              progress_logger=progress_logger)
                continue

        tilewalker_cleanup(
            task,
            dry_run=dry_run,
            concurrency=concurrency,
            skip_geoms_for_last_levels=skip_geoms_for_last_levels,
            progress_logger=progress_logger,
            seed_progress=seed_progress,
        )
Exemple #7
0
class SeedScript(object):
    usage = "usage: %prog [options] seed_conf"
    parser = OptionParser(usage)
    parser.add_option("-q",
                      "--quiet",
                      action="count",
                      dest="quiet",
                      default=0,
                      help="don't print status messages to stdout")
    parser.add_option("-s",
                      "--seed-conf",
                      dest="seed_file",
                      default=None,
                      help="seed configuration")
    parser.add_option("-f",
                      "--proxy-conf",
                      dest="conf_file",
                      default=None,
                      help="proxy configuration")
    parser.add_option("-c",
                      "--concurrency",
                      type="int",
                      dest="concurrency",
                      default=2,
                      help="number of parallel seed processes")
    parser.add_option("-n",
                      "--dry-run",
                      action="store_true",
                      dest="dry_run",
                      default=False,
                      help="do not seed, just print output")
    parser.add_option("-l",
                      "--skip-geoms-for-last-levels",
                      type="int",
                      dest="geom_levels",
                      default=0,
                      metavar="N",
                      help="do not check for intersections between tiles"
                      " and seed geometries on the last N levels")
    parser.add_option("--summary",
                      action="store_true",
                      dest="summary",
                      default=False,
                      help="print summary with all seeding tasks and exit."
                      " does not seed anything.")
    parser.add_option(
        "-i",
        "--interactive",
        action="store_true",
        dest="interactive",
        default=False,
        help="print each task description and ask if it should be seeded")

    parser.add_option(
        "--seed",
        action="append",
        dest="seed_names",
        metavar='task1,task2,...',
        help="seed only the named tasks. cleanup is disabled unless "
        "--cleanup is used. use ALL to select all tasks")

    parser.add_option(
        "--cleanup",
        action="append",
        dest="cleanup_names",
        metavar='task1,task2,...',
        help="cleanup only the named tasks. seeding is disabled unless "
        "--seed is used. use ALL to select all tasks")

    parser.add_option(
        "--use-cache-lock",
        action="store_true",
        default=False,
        help="use locking to prevent multiple mapproxy-seed calls "
        "to seed the same cache")

    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging()

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError, ex:
            print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        with local_base_config(mapproxy_conf.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError, ex:
                print "error in configuration: " + '\n\t'.join(
                    str(ex).split('\n'))
                sys.exit(2)

            if options.summary:
                print '========== Seeding tasks =========='
                for task in seed_tasks:
                    print format_seed_task(task)
                print '========== Cleanup tasks =========='
                for task in cleanup_tasks:
                    print format_cleanup_task(task)
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print '========== Seeding tasks =========='
                    print 'Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print '========== Cleanup tasks =========='
                    print 'Start cleanup process (%d task%s)' % (len(
                        cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except KeyboardInterrupt:
                print '\nexiting...'
                return 2
Exemple #8
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error('missing seed_conf file as last argument or --seed-conf option')
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                    continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print('Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
                        progress_store=progress)
                    seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
                         concurrency=options.concurrency, cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' % (
                        len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
                    cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
                            concurrency=options.concurrency, progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
Exemple #9
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                                         continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
Exemple #10
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        if options.duration:
            # calls with --duration are handled in call_with_duration
            sys.exit(self.call_with_duration(options, args))

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        progress = None
        if options.continue_seed or options.progress_file:
            if not options.progress_file:
                options.progress_file = '.mapproxy_seed_progress'
            progress = ProgressStore(options.progress_file,
                                     continue_seed=options.continue_seed)

        if options.reseed_file:
            if not os.path.exists(options.reseed_file):
                # create --reseed-file if missing
                with open(options.reseed_file, 'w'):
                    pass
            else:
                if progress and not os.path.exists(options.progress_file):
                    # we have an existing --reseed-file but no --progress-file
                    # meaning the last seed call was completed
                    if options.reseed_interval and (
                            os.path.getmtime(options.reseed_file) >
                        (time.time() - options.reseed_interval)):
                        print("no need for re-seeding")
                        sys.exit(1)
                    os.utime(options.reseed_file, (time.time(), time.time()))

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels,
                         skip_uncached=options.skip_uncached)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()