def test_process_pool_close_stopped(self): """Process Pool Spawn is stopped after close.""" pool = ProcessPool() pool.schedule(function, args=[1]) pool.close() pool.join() self.assertFalse(pool.active)
def test_process_pool_close_stopped(self): """Process Pool Fork is stopped after close.""" pool = ProcessPool(max_workers=1) pool.schedule(function, args=[1]) pool.close() pool.join() self.assertFalse(pool.active)
def test_process_pool_close_stopped(self): """Process Pool Forkserver is stopped after close.""" pool = ProcessPool(max_workers=1) pool.schedule(function, args=[1]) pool.close() pool.join() self.assertFalse(pool.active)
def test_process_pool_join_futures_timeout(self): """Process Pool Fork TimeoutError is raised if join on long futures.""" pool = ProcessPool(max_workers=1) for _ in range(2): pool.schedule(long_function) pool.close() self.assertRaises(TimeoutError, pool.join, 0.4) pool.stop() pool.join()
def test_process_pool_close_futures(self): """Process Pool Fork all futures are performed on close.""" futures = [] pool = ProcessPool(max_workers=1) for index in range(10): futures.append(pool.schedule(function, args=[index])) pool.close() pool.join() map(self.assertTrue, [f.done() for f in futures])
def test_process_pool_join_futures_timeout(self): """Process Pool Spawn TimeoutError is raised if join on long tasks.""" pool = ProcessPool() for _ in range(2): pool.schedule(long_function) pool.close() self.assertRaises(TimeoutError, pool.join, 0.4) pool.stop() pool.join()
def test_process_pool_close_futures(self): """Process Pool Spawn all futures are performed on close.""" futures = [] pool = ProcessPool() for index in range(10): futures.append(pool.schedule(function, args=[index])) pool.close() pool.join() map(self.assertTrue, [f.done() for f in futures])
class CounterDaemon(object): def __init__(self, workers=1, poll_interval=None, max_tasks=100, task_timeout=0.1, task_default_sleep=0.01, task_sleep_rand_range=(1, 20)): self.workers = workers self.poll_interval = poll_interval self.max_tasks = max_tasks self.task_timeout = task_timeout self.task_default_sleep = task_default_sleep self.task_sleep_rand_range = task_sleep_rand_range self.pool = ProcessPool(max_workers=self.workers, max_tasks=self.max_tasks) self.pool._start_pool() def done_callback(self, bucket_id, future): pid = os.getpid() try: result = future.result() logger.debug('Result: {}\tpid: {}\tbucket: {}'.format(result, pid, bucket_id)) except futures.TimeoutError as e: logger.warning('TimeoutError\tpid: {}\tbucket: {}'.format(pid, bucket_id)) except futures.CancelledError: return except Exception as e: logger.exception('TaskError\t pid: {}\tbucket: {}\tError: {}'.format(pid, bucket_id, e)) def run_once(self): for bucket_id in random.sample(BUCKET_RANGE, self.workers): sleep = self.task_default_sleep * random.randint(*self.task_sleep_rand_range) future = self.pool.schedule( incr, args=(bucket_id,), kwargs={'sleep': sleep}, timeout=self.task_timeout ) future.add_done_callback(functools.partial(self.done_callback, bucket_id)) def run_forever(self): while True: try: self.run_once() except Exception as e: logger.exception('RunOnceError: {}'.format(e)) time.sleep(self.poll_interval or (3 * self.task_timeout)) def start(self): try: self.run_forever() except Exception as e: logger.exception('Error during running daemon: {}'.format(e)) self.pool.close() time.sleep(10) self.pool.stop() finally: self.pool.join()
def find_tlds(self): dom_list = [self.known_domain + '.' + tld for tld in self.tld_list] try: pool = ThreadPool(max_workers=self.max_workers, max_tasks=self.max_tasks) results = pool.map(self.check_tld, dom_list, timeout=self.timeout) pool.close() pool.join() print(results) except Exception as e: print(repr(e)) pass
def parallel_checks() -> None: """ Do all the checks that don't change code and can run in parallel. """ chores = [ do_mypy, do_detect_secrets, do_git_secrets, vulture, do_compile_py, do_lint, do_flake8, do_dodgy, do_bandit, do_python_taint, do_mccabe, do_check_manifest, do_liccheck, ] if IS_GITLAB: # other tasks assume there will be a LOC file by now. do_count_lines_of_code() for chore in chores: print(chore()) return # can't do pyroma because that needs a package, which might not exist yet. pool = ProcessPool(12) # max_workers=len(chores)) # cpu_count()) # log_to_stderr(logging.DEBUG) tasks = [] for chore in chores: tasks.append(pool.schedule(chore, args=())) print("close & join") pool.close() pool.join() for current_task in tasks: # pylint: disable=broad-except try: result = current_task.result() exception = current_task.exception() if exception: print(current_task.exception()) print(result) if "Abnormal" in str(result): print("One or more parallel tasks failed.") sys.exit(-1) except Exception as ex: print(ex) sys.exit(-1)
def process_urls(paths, n_processes, prefix=COMMON_CRAWL_URL, max_failures=100, num_progress_reports=50): print(f"Using {n_processes} parallel processes") failed_paths = [] bios = [] time0 = time.time() path_name = (paths[0] + '///').split('/')[1] num_progress_reports = max( 1, min(num_progress_reports, len(paths) // n_processes)) done = 0 pool = ProcessPool(n_processes) for i, paths_chunk in enumerate(chunks(paths, num_progress_reports)): ans = pool.map(bios_from_wet_url, [prefix + path for path in paths_chunk], timeout=1200) iterator = ans.result() for p in paths_chunk + ["done"]: try: a = next(iterator) assert p != "done" if a is not None: bios += [dict(path=p, **b) for b in a] continue except StopIteration: assert p == "done" break except Exception as error: print("--------------------\n" * 10 + f"function raised {error}") failed_paths.append(p) done += len(paths_chunk) pct = (i + 1) / num_progress_reports eta = (time.time() - time0) * (1 / pct - 1) / 60 / 60 print( f"{eta:.1f} hours left, {done:,}/{len(paths):,} done ({pct:.0%}),", f"{int(len(bios)/pct):,} estimated bios, {path_name}") if len(failed_paths) > 0: print(f" {len(failed_paths):,} failed paths") if len(failed_paths) > max_failures: break pool.close() return dedup_exact(bios), failed_paths # dedup_exact is new!
def handle(self, *args, **options): trials = options['trials'] bucket_id = options['bucket'] pool = ProcessPool(max_workers=2) pool._start_pool() bucket, _ = Counter.objects.get_or_create(bucket=bucket_id) bucket.count = 0 bucket.save() future_1 = pool.schedule(run_atomic_transactions, args=('T1', bucket_id, trials)) future_2 = pool.schedule(run_savepoints, args=('T2', bucket_id, trials)) pool.close() pool.join()
class PebbleExecutor(concurrent.futures.Executor): def __init__(self, max_workers, timeout=None): self.pool = ProcessPool(max_workers=max_workers) self.timeout = timeout def submit(self, fn, *args, **kwargs): return self.pool.schedule(fn, args=args, timeout=self.timeout) def map(self, func, *iterables, timeout=None, chunksize=1): raise NotImplementedError("This wrapper does not support `map`.") def shutdown(self, wait=True): if wait: log.info("Closing workers...") self.pool.close() else: log.info("Ending workers...") self.pool.stop() self.pool.join() log.info("Workers joined.")
class Wintermute(discord.Client): __channels = None __bot_prelude = None __timeout = None __parser = None __pool = None def __init__(self, channels={}, bot_prelude='[bot] ', timeout=10, multiprocessing=1, loglevel=logging.INFO): super().__init__() self.__channels = channels self.__bot_prelude = bot_prelude self.__timeout = timeout logging.basicConfig(level=loglevel) self.__parser = BotGram(prelude=bot_prelude) self.__pool = ProcessPool(max_workers=multiprocessing, initializer=seed) def __del__(self): self.__pool.close() self.__pool.join() async def on_ready(self): logging.info("Online as " + str(self.user.name)) game = discord.Game() game.name = "Manipulating humanity" await self.change_presence(game=game) logging.info('Setup done') async def on_message(self, mess): if mess.channel.is_private: logging.info("new private message from " + str(mess.author)) else: logging.info("new message on server " + str(mess.server) + " and channel " + str(mess.channel) + " from " + str(mess.author)) logging.info("len: " + str(len(mess.content))) logging.info("content: " + str(mess.content)) if mess.channel.is_private: return if mess.author == self.user: return if (mess.channel.permissions_for(mess.channel.server.me).send_messages and mess.channel.name == self.__channels[mess.server.name]): resp = self.__parser.parse(mess) if resp is not None: task = self.__pool.schedule(str, args=(resp, ), timeout=self.__timeout) asyncio.ensure_future(self.collect_response( mess.channel, mess.author.mention, task), loop=self.loop) async def collect_response(self, channel, mention, task): try: while not task.done(): await asyncio.sleep(0.1) result = task.result() await self.send_message(channel, result) except TimeoutError: await self.send_message( channel, self.__bot_prelude + mention + ' Your request timed out') task.cancel() except HTTPException as e: if e.response.status == 400: await self.send_message( channel, self.__bot_prelude + mention + ' Error: Request reply was probably too long')