示例#1
0
def seed(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
    progress_logger=None, cache_locker=None):
    if cache_locker is None:
        cache_locker = DummyCacheLocker()

    active_tasks = tasks[::-1]
    while active_tasks:
        task = active_tasks[-1]
        print format_seed_task(task)

        wait = len(active_tasks) == 1
        try:
            with cache_locker.lock(task.md['cache_name'], no_block=not wait):
                if progress_logger and progress_logger.progress_store:
                    progress_logger.current_task_id = task.id
                    start_progress = progress_logger.progress_store.get(task.id)
                else:
                    start_progress = None
                seed_progress = SeedProgress(old_progress_identifier=start_progress)
                seed_task(task, concurrency, dry_run, skip_geoms_for_last_levels, progress_logger,
                    seed_progress=seed_progress)
        except CacheLockedError:
            print '    ...cache is locked, skipping'
            active_tasks = [task] + active_tasks[:-1]
        else:
            active_tasks.pop()
示例#2
0
def seed(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
    progress_logger=None, cache_locker=None):
    if cache_locker is None:
        cache_locker = DummyCacheLocker()

    active_tasks = tasks[::-1]
    while active_tasks:
        task = active_tasks[-1]
        print(format_seed_task(task))

        wait = len(active_tasks) == 1
        try:
            with cache_locker.lock(task.md['cache_name'], no_block=not wait):
                if progress_logger and progress_logger.progress_store:
                    progress_logger.current_task_id = task.id
                    start_progress = progress_logger.progress_store.get(task.id)
                else:
                    start_progress = None
                seed_progress = SeedProgress(old_progress_identifier=start_progress)
                seed_task(task, concurrency, dry_run, skip_geoms_for_last_levels, progress_logger,
                    seed_progress=seed_progress)
        except CacheLockedError:
            print('    ...cache is locked, skipping')
            active_tasks = [task] + active_tasks[:-1]
        else:
            active_tasks.pop()
示例#3
0
    def interactive(self, seed_tasks, cleanup_tasks):
        selected_seed_tasks = []
        print '========== Select seeding tasks =========='
        for task in seed_tasks:
            print format_seed_task(task)
            if ask_yes_no_question('    Seed this task (y/n)? '):
                selected_seed_tasks.append(task)
        seed_tasks = selected_seed_tasks

        selected_cleanup_tasks = []
        print '========== Select cleanup tasks =========='
        for task in cleanup_tasks:
            print format_cleanup_task(task)
            if ask_yes_no_question('    Cleanup this task (y/n)? '):
                selected_cleanup_tasks.append(task)
        cleanup_tasks = selected_cleanup_tasks
        return seed_tasks, cleanup_tasks
示例#4
0
    def interactive(self, seed_tasks, cleanup_tasks):
        selected_seed_tasks = []
        print '========== Select seeding tasks =========='
        for task in seed_tasks:
            print format_seed_task(task)
            if ask_yes_no_question('    Seed this task (y/n)? '):
                selected_seed_tasks.append(task)
        seed_tasks = selected_seed_tasks

        selected_cleanup_tasks = []
        print '========== Select cleanup tasks =========='
        for task in cleanup_tasks:
            print format_cleanup_task(task)
            if ask_yes_no_question('    Cleanup this task (y/n)? '):
                selected_cleanup_tasks.append(task)
        cleanup_tasks = selected_cleanup_tasks
        return seed_tasks, cleanup_tasks
示例#5
0
def seed(tasks, concurrency=2, dry_run=False, skip_geoms_for_last_levels=0,
    progress_logger=None, cache_locker=None):
    if cache_locker is None:
        cache_locker = DummyCacheLocker()
    
    active_tasks = tasks[::-1]
    while active_tasks:
        task = active_tasks[-1]
        print format_seed_task(task)
        
        wait = len(active_tasks) == 1
        try:
            with cache_locker.lock(task.md['cache_name'], no_block=not wait):
                _seed_task(task, concurrency, dry_run, skip_geoms_for_last_levels, progress_logger)
        except CacheLockedError:
            print '    ...cache is locked, skipping'
            active_tasks = [task] + active_tasks[:-1]
        else:
            active_tasks.pop()
示例#6
0
def seed(tasks,
         concurrency=2,
         dry_run=False,
         skip_geoms_for_last_levels=0,
         progress_logger=None,
         cache_locker=None):
    if cache_locker is None:
        cache_locker = DummyCacheLocker()

    active_tasks = tasks[::-1]
    while active_tasks:
        task = active_tasks[-1]
        print format_seed_task(task)

        wait = len(active_tasks) == 1
        try:
            with cache_locker.lock(task.md['cache_name'], no_block=not wait):
                _seed_task(task, concurrency, dry_run,
                           skip_geoms_for_last_levels, progress_logger)
        except CacheLockedError:
            print '    ...cache is locked, skipping'
            active_tasks = [task] + active_tasks[:-1]
        else:
            active_tasks.pop()
示例#7
0
class SeedScript(object):
    usage = "usage: %prog [options] seed_conf"
    parser = OptionParser(usage)
    parser.add_option("-q",
                      "--quiet",
                      action="count",
                      dest="quiet",
                      default=0,
                      help="don't print status messages to stdout")
    parser.add_option("-s",
                      "--seed-conf",
                      dest="seed_file",
                      default=None,
                      help="seed configuration")
    parser.add_option("-f",
                      "--proxy-conf",
                      dest="conf_file",
                      default=None,
                      help="proxy configuration")
    parser.add_option("-c",
                      "--concurrency",
                      type="int",
                      dest="concurrency",
                      default=2,
                      help="number of parallel seed processes")
    parser.add_option("-n",
                      "--dry-run",
                      action="store_true",
                      dest="dry_run",
                      default=False,
                      help="do not seed, just print output")
    parser.add_option("-l",
                      "--skip-geoms-for-last-levels",
                      type="int",
                      dest="geom_levels",
                      default=0,
                      metavar="N",
                      help="do not check for intersections between tiles"
                      " and seed geometries on the last N levels")
    parser.add_option("--summary",
                      action="store_true",
                      dest="summary",
                      default=False,
                      help="print summary with all seeding tasks and exit."
                      " does not seed anything.")
    parser.add_option(
        "-i",
        "--interactive",
        action="store_true",
        dest="interactive",
        default=False,
        help="print each task description and ask if it should be seeded")

    parser.add_option(
        "--seed",
        action="append",
        dest="seed_names",
        metavar='task1,task2,...',
        help="seed only the named tasks. cleanup is disabled unless "
        "--cleanup is used. use ALL to select all tasks")

    parser.add_option(
        "--cleanup",
        action="append",
        dest="cleanup_names",
        metavar='task1,task2,...',
        help="cleanup only the named tasks. seeding is disabled unless "
        "--seed is used. use ALL to select all tasks")

    parser.add_option(
        "--use-cache-lock",
        action="store_true",
        default=False,
        help="use locking to prevent multiple mapproxy-seed calls "
        "to seed the same cache")

    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging()

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError, ex:
            print "ERROR: " + '\n\t'.join(str(ex).split('\n'))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        with local_base_config(mapproxy_conf.base_config):
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError, ex:
                print "error in configuration: " + '\n\t'.join(
                    str(ex).split('\n'))
                sys.exit(2)

            if options.summary:
                print '========== Seeding tasks =========='
                for task in seed_tasks:
                    print format_seed_task(task)
                print '========== Cleanup tasks =========='
                for task in cleanup_tasks:
                    print format_cleanup_task(task)
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print '========== Seeding tasks =========='
                    print 'Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print '========== Cleanup tasks =========='
                    print 'Start cleanup process (%d task%s)' % (len(
                        cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except KeyboardInterrupt:
                print '\nexiting...'
                return 2
示例#8
0
文件: script.py 项目: TNRIS/mapproxy
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error('missing seed_conf file as last argument or --seed-conf option')
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " + '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                    continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print('Start seeding process (%d task%s)' % (
                        len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2,
                        progress_store=progress)
                    seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run,
                         concurrency=options.concurrency, cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' % (
                        len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet==0, silent=options.quiet>=2)
                    cleanup(cleanup_tasks, verbose=options.quiet==0, dry_run=options.dry_run,
                            concurrency=options.concurrency, progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
示例#9
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            progress = None
            if options.continue_seed or options.progress_file:
                if options.progress_file:
                    progress_file = options.progress_file
                else:
                    progress_file = '.mapproxy_seed_progress'
                progress = ProgressStore(progress_file,
                                         continue_seed=options.continue_seed)

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()
示例#10
0
    def __call__(self):
        (options, args) = self.parser.parse_args()

        if len(args) != 1 and not options.seed_file:
            self.parser.print_help()
            sys.exit(1)

        if not options.seed_file:
            if len(args) != 1:
                self.parser.error(
                    'missing seed_conf file as last argument or --seed-conf option'
                )
            else:
                options.seed_file = args[0]

        if not options.conf_file:
            self.parser.error('missing mapproxy configuration -f/--proxy-conf')

        setup_logging(options.logging_conf)

        if options.duration:
            # calls with --duration are handled in call_with_duration
            sys.exit(self.call_with_duration(options, args))

        try:
            mapproxy_conf = load_configuration(options.conf_file, seed=True)
        except ConfigurationError as ex:
            print("ERROR: " + '\n\t'.join(str(ex).split('\n')))
            sys.exit(2)

        if options.use_cache_lock:
            cache_locker = CacheLocker('.mapproxy_seed.lck')
        else:
            cache_locker = None

        if not sys.stdout.isatty() and options.quiet == 0:
            # disable verbose output for non-ttys
            options.quiet = 1

        progress = None
        if options.continue_seed or options.progress_file:
            if not options.progress_file:
                options.progress_file = '.mapproxy_seed_progress'
            progress = ProgressStore(options.progress_file,
                                     continue_seed=options.continue_seed)

        if options.reseed_file:
            if not os.path.exists(options.reseed_file):
                # create --reseed-file if missing
                with open(options.reseed_file, 'w'):
                    pass
            else:
                if progress and not os.path.exists(options.progress_file):
                    # we have an existing --reseed-file but no --progress-file
                    # meaning the last seed call was completed
                    if options.reseed_interval and (
                            os.path.getmtime(options.reseed_file) >
                        (time.time() - options.reseed_interval)):
                        print("no need for re-seeding")
                        sys.exit(1)
                    os.utime(options.reseed_file, (time.time(), time.time()))

        with mapproxy_conf:
            try:
                seed_conf = load_seed_tasks_conf(options.seed_file,
                                                 mapproxy_conf)
                seed_names, cleanup_names = self.task_names(seed_conf, options)
                seed_tasks = seed_conf.seeds(seed_names)
                cleanup_tasks = seed_conf.cleanups(cleanup_names)
            except ConfigurationError as ex:
                print("error in configuration: " +
                      '\n\t'.join(str(ex).split('\n')))
                sys.exit(2)

            if options.summary:
                print('========== Seeding tasks ==========')
                for task in seed_tasks:
                    print(format_seed_task(task))
                print('========== Cleanup tasks ==========')
                for task in cleanup_tasks:
                    print(format_cleanup_task(task))
                return 0

            try:
                if options.interactive:
                    seed_tasks, cleanup_tasks = self.interactive(
                        seed_tasks, cleanup_tasks)

                if seed_tasks:
                    print('========== Seeding tasks ==========')
                    print(
                        'Start seeding process (%d task%s)' %
                        (len(seed_tasks), 's' if len(seed_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    seed(seed_tasks,
                         progress_logger=logger,
                         dry_run=options.dry_run,
                         concurrency=options.concurrency,
                         cache_locker=cache_locker,
                         skip_geoms_for_last_levels=options.geom_levels,
                         skip_uncached=options.skip_uncached)
                if cleanup_tasks:
                    print('========== Cleanup tasks ==========')
                    print('Start cleanup process (%d task%s)' %
                          (len(cleanup_tasks),
                           's' if len(cleanup_tasks) > 1 else ''))
                    logger = ProgressLog(verbose=options.quiet == 0,
                                         silent=options.quiet >= 2,
                                         progress_store=progress)
                    cleanup(cleanup_tasks,
                            verbose=options.quiet == 0,
                            dry_run=options.dry_run,
                            concurrency=options.concurrency,
                            progress_logger=logger,
                            skip_geoms_for_last_levels=options.geom_levels)
            except SeedInterrupted:
                print('\ninterrupted...')
                return 3
            except KeyboardInterrupt:
                print('\nexiting...')
                return 2

            if progress:
                progress.remove()