def test_locked_by_process_no_block(self): proc_is_locked = multiprocessing.Event() def lock(): locker = CacheLocker(self.lock_file) with locker.lock('foo'): proc_is_locked.set() time.sleep(10) p = multiprocessing.Process(target=lock) p.start() # wait for process to start proc_is_locked.wait() locker = CacheLocker(self.lock_file) # test unlocked bar with locker.lock('bar', no_block=True): assert True # test locked foo try: with locker.lock('foo', no_block=True): assert False except CacheLockedError: pass finally: p.terminate() p.join()
def test_locked_by_process_waiting(self): proc_is_locked = multiprocessing.Event() def lock(): locker = CacheLocker(self.lock_file) with locker.lock('foo'): proc_is_locked.set() time.sleep(.1) p = multiprocessing.Process(target=lock) start_time = time.time() p.start() # wait for process to start proc_is_locked.wait() locker = CacheLocker(self.lock_file, polltime=0.02) try: with locker.lock('foo', no_block=False): diff = time.time() - start_time assert diff > 0.1 finally: p.terminate() p.join()
class SeedScript(object): usage = "usage: %prog [options] seed_conf" parser = OptionParser(usage) parser.add_option("-q", "--quiet", action="count", dest="quiet", default=0, help="don't print status messages to stdout") parser.add_option("-s", "--seed-conf", dest="seed_file", default=None, help="seed configuration") parser.add_option("-f", "--proxy-conf", dest="conf_file", default=None, help="proxy configuration") parser.add_option("-c", "--concurrency", type="int", dest="concurrency", default=2, help="number of parallel seed processes") parser.add_option("-n", "--dry-run", action="store_true", dest="dry_run", default=False, help="do not seed, just print output") parser.add_option("-l", "--skip-geoms-for-last-levels", type="int", dest="geom_levels", default=0, metavar="N", help="do not check for intersections between tiles" " and seed geometries on the last N levels") parser.add_option("--summary", action="store_true", dest="summary", default=False, help="print summary with all seeding tasks and exit." " does not seed anything.") parser.add_option( "-i", "--interactive", action="store_true", dest="interactive", default=False, help="print each task description and ask if it should be seeded") parser.add_option( "--seed", action="append", dest="seed_names", metavar='task1,task2,...', help="seed only the named tasks. cleanup is disabled unless " "--cleanup is used. use ALL to select all tasks") parser.add_option( "--cleanup", action="append", dest="cleanup_names", metavar='task1,task2,...', help="cleanup only the named tasks. seeding is disabled unless " "--seed is used. use ALL to select all tasks") parser.add_option( "--use-cache-lock", action="store_true", default=False, help="use locking to prevent multiple mapproxy-seed calls " "to seed the same cache") def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging() try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError, ex: print "ERROR: " + '\n\t'.join(str(ex).split('\n')) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None with local_base_config(mapproxy_conf.base_config): try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError, ex: print "error in configuration: " + '\n\t'.join( str(ex).split('\n')) sys.exit(2) if options.summary: print '========== Seeding tasks ==========' for task in seed_tasks: print format_seed_task(task) print '========== Cleanup tasks ==========' for task in cleanup_tasks: print format_cleanup_task(task) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print '========== Seeding tasks ==========' print 'Start seeding process (%d task%s)' % ( len(seed_tasks), 's' if len(seed_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print '========== Cleanup tasks ==========' print 'Start cleanup process (%d task%s)' % (len( cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '') logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except KeyboardInterrupt: print '\nexiting...' return 2
def lock(): locker = CacheLocker(self.lock_file) with locker.lock('foo'): proc_is_locked.set() time.sleep(.1)
def test_free_lock(self): locker = CacheLocker(self.lock_file) with locker.lock('foo'): assert True
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 progress = None if options.continue_seed or options.progress_file: if options.progress_file: progress_file = options.progress_file else: progress_file = '.mapproxy_seed_progress' progress = ProgressStore(progress_file, continue_seed=options.continue_seed) try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()
def lock(): locker = CacheLocker(lock_file) with locker.lock("foo"): proc_is_locked.set() time.sleep(.1)
def test_free_lock(self, lock_file): locker = CacheLocker(lock_file) with locker.lock("foo"): assert True
def __call__(self): (options, args) = self.parser.parse_args() if len(args) != 1 and not options.seed_file: self.parser.print_help() sys.exit(1) if not options.seed_file: if len(args) != 1: self.parser.error( 'missing seed_conf file as last argument or --seed-conf option' ) else: options.seed_file = args[0] if not options.conf_file: self.parser.error('missing mapproxy configuration -f/--proxy-conf') setup_logging(options.logging_conf) if options.duration: # calls with --duration are handled in call_with_duration sys.exit(self.call_with_duration(options, args)) try: mapproxy_conf = load_configuration(options.conf_file, seed=True) except ConfigurationError as ex: print("ERROR: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.use_cache_lock: cache_locker = CacheLocker('.mapproxy_seed.lck') else: cache_locker = None if not sys.stdout.isatty() and options.quiet == 0: # disable verbose output for non-ttys options.quiet = 1 progress = None if options.continue_seed or options.progress_file: if not options.progress_file: options.progress_file = '.mapproxy_seed_progress' progress = ProgressStore(options.progress_file, continue_seed=options.continue_seed) if options.reseed_file: if not os.path.exists(options.reseed_file): # create --reseed-file if missing with open(options.reseed_file, 'w'): pass else: if progress and not os.path.exists(options.progress_file): # we have an existing --reseed-file but no --progress-file # meaning the last seed call was completed if options.reseed_interval and ( os.path.getmtime(options.reseed_file) > (time.time() - options.reseed_interval)): print("no need for re-seeding") sys.exit(1) os.utime(options.reseed_file, (time.time(), time.time())) with mapproxy_conf: try: seed_conf = load_seed_tasks_conf(options.seed_file, mapproxy_conf) seed_names, cleanup_names = self.task_names(seed_conf, options) seed_tasks = seed_conf.seeds(seed_names) cleanup_tasks = seed_conf.cleanups(cleanup_names) except ConfigurationError as ex: print("error in configuration: " + '\n\t'.join(str(ex).split('\n'))) sys.exit(2) if options.summary: print('========== Seeding tasks ==========') for task in seed_tasks: print(format_seed_task(task)) print('========== Cleanup tasks ==========') for task in cleanup_tasks: print(format_cleanup_task(task)) return 0 try: if options.interactive: seed_tasks, cleanup_tasks = self.interactive( seed_tasks, cleanup_tasks) if seed_tasks: print('========== Seeding tasks ==========') print( 'Start seeding process (%d task%s)' % (len(seed_tasks), 's' if len(seed_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) seed(seed_tasks, progress_logger=logger, dry_run=options.dry_run, concurrency=options.concurrency, cache_locker=cache_locker, skip_geoms_for_last_levels=options.geom_levels, skip_uncached=options.skip_uncached) if cleanup_tasks: print('========== Cleanup tasks ==========') print('Start cleanup process (%d task%s)' % (len(cleanup_tasks), 's' if len(cleanup_tasks) > 1 else '')) logger = ProgressLog(verbose=options.quiet == 0, silent=options.quiet >= 2, progress_store=progress) cleanup(cleanup_tasks, verbose=options.quiet == 0, dry_run=options.dry_run, concurrency=options.concurrency, progress_logger=logger, skip_geoms_for_last_levels=options.geom_levels) except SeedInterrupted: print('\ninterrupted...') return 3 except KeyboardInterrupt: print('\nexiting...') return 2 if progress: progress.remove()