def test_process_pool_close_stopped(self):
"""Process Pool Spawn is stopped after close."""
pool = ProcessPool()
pool.schedule(function, args=[1])
pool.close()
pool.join()
self.assertFalse(pool.active)
def test_process_pool_close_stopped(self):
"""Process Pool Fork is stopped after close."""
pool = ProcessPool(max_workers=1)
pool.schedule(function, args=[1])
pool.close()
pool.join()
self.assertFalse(pool.active)
def test_process_pool_close_stopped(self):
"""Process Pool Forkserver is stopped after close."""
pool = ProcessPool(max_workers=1)
pool.schedule(function, args=[1])
pool.close()
pool.join()
self.assertFalse(pool.active)
def test_process_pool_join_futures_timeout(self):
"""Process Pool Fork TimeoutError is raised if join on long futures."""
pool = ProcessPool(max_workers=1)
for _ in range(2):
pool.schedule(long_function)
pool.close()
self.assertRaises(TimeoutError, pool.join, 0.4)
pool.stop()
pool.join()
def test_process_pool_close_futures(self):
"""Process Pool Fork all futures are performed on close."""
futures = []
pool = ProcessPool(max_workers=1)
for index in range(10):
futures.append(pool.schedule(function, args=[index]))
pool.close()
pool.join()
map(self.assertTrue, [f.done() for f in futures])
def test_process_pool_join_futures_timeout(self):
"""Process Pool Spawn TimeoutError is raised if join on long tasks."""
pool = ProcessPool()
for _ in range(2):
pool.schedule(long_function)
pool.close()
self.assertRaises(TimeoutError, pool.join, 0.4)
pool.stop()
pool.join()
def test_process_pool_close_futures(self):
"""Process Pool Spawn all futures are performed on close."""
futures = []
pool = ProcessPool()
for index in range(10):
futures.append(pool.schedule(function, args=[index]))
pool.close()
pool.join()
map(self.assertTrue, [f.done() for f in futures])
class CounterDaemon(object):
def __init__(self, workers=1, poll_interval=None, max_tasks=100, task_timeout=0.1,
task_default_sleep=0.01, task_sleep_rand_range=(1, 20)):
self.workers = workers
self.poll_interval = poll_interval
self.max_tasks = max_tasks
self.task_timeout = task_timeout
self.task_default_sleep = task_default_sleep
self.task_sleep_rand_range = task_sleep_rand_range
self.pool = ProcessPool(max_workers=self.workers, max_tasks=self.max_tasks)
self.pool._start_pool()
def done_callback(self, bucket_id, future):
pid = os.getpid()
try:
result = future.result()
logger.debug('Result: {}\tpid: {}\tbucket: {}'.format(result, pid, bucket_id))
except futures.TimeoutError as e:
logger.warning('TimeoutError\tpid: {}\tbucket: {}'.format(pid, bucket_id))
except futures.CancelledError:
return
except Exception as e:
logger.exception('TaskError\t pid: {}\tbucket: {}\tError: {}'.format(pid, bucket_id, e))
def run_once(self):
for bucket_id in random.sample(BUCKET_RANGE, self.workers):
sleep = self.task_default_sleep * random.randint(*self.task_sleep_rand_range)
future = self.pool.schedule(
incr,
args=(bucket_id,),
kwargs={'sleep': sleep},
timeout=self.task_timeout
)
future.add_done_callback(functools.partial(self.done_callback, bucket_id))
def run_forever(self):
while True:
try:
self.run_once()
except Exception as e:
logger.exception('RunOnceError: {}'.format(e))
time.sleep(self.poll_interval or (3 * self.task_timeout))
def start(self):
try:
self.run_forever()
except Exception as e:
logger.exception('Error during running daemon: {}'.format(e))
self.pool.close()
time.sleep(10)
self.pool.stop()
finally:
self.pool.join()
def find_tlds(self):
dom_list = [self.known_domain + '.' + tld for tld in self.tld_list]
try:
pool = ThreadPool(max_workers=self.max_workers,
max_tasks=self.max_tasks)
results = pool.map(self.check_tld, dom_list, timeout=self.timeout)
pool.close()
pool.join()
print(results)
except Exception as e:
print(repr(e))
pass
def parallel_checks() -> None:
"""
Do all the checks that don't change code and can run in parallel.
"""
chores = [
do_mypy,
do_detect_secrets,
do_git_secrets,
vulture,
do_compile_py,
do_lint,
do_flake8,
do_dodgy,
do_bandit,
do_python_taint,
do_mccabe,
do_check_manifest,
do_liccheck,
]
if IS_GITLAB:
# other tasks assume there will be a LOC file by now.
do_count_lines_of_code()
for chore in chores:
print(chore())
return
# can't do pyroma because that needs a package, which might not exist yet.
pool = ProcessPool(12) # max_workers=len(chores)) # cpu_count())
# log_to_stderr(logging.DEBUG)
tasks = []
for chore in chores:
tasks.append(pool.schedule(chore, args=()))
print("close & join")
pool.close()
pool.join()
for current_task in tasks:
# pylint: disable=broad-except
try:
result = current_task.result()
exception = current_task.exception()
if exception:
print(current_task.exception())
print(result)
if "Abnormal" in str(result):
print("One or more parallel tasks failed.")
sys.exit(-1)
except Exception as ex:
print(ex)
sys.exit(-1)
def process_urls(paths,
n_processes,
prefix=COMMON_CRAWL_URL,
max_failures=100,
num_progress_reports=50):
print(f"Using {n_processes} parallel processes")
failed_paths = []
bios = []
time0 = time.time()
path_name = (paths[0] + '///').split('/')[1]
num_progress_reports = max(
1, min(num_progress_reports,
len(paths) // n_processes))
done = 0
pool = ProcessPool(n_processes)
for i, paths_chunk in enumerate(chunks(paths, num_progress_reports)):
ans = pool.map(bios_from_wet_url,
[prefix + path for path in paths_chunk],
timeout=1200)
iterator = ans.result()
for p in paths_chunk + ["done"]:
try:
a = next(iterator)
assert p != "done"
if a is not None:
bios += [dict(path=p, **b) for b in a]
continue
except StopIteration:
assert p == "done"
break
except Exception as error:
print("--------------------\n" * 10 +
f"function raised {error}")
failed_paths.append(p)
done += len(paths_chunk)
pct = (i + 1) / num_progress_reports
eta = (time.time() - time0) * (1 / pct - 1) / 60 / 60
print(
f"{eta:.1f} hours left, {done:,}/{len(paths):,} done ({pct:.0%}),",
f"{int(len(bios)/pct):,} estimated bios, {path_name}")
if len(failed_paths) > 0:
print(f" {len(failed_paths):,} failed paths")
if len(failed_paths) > max_failures:
break
pool.close()
return dedup_exact(bios), failed_paths # dedup_exact is new!
def handle(self, *args, **options):
trials = options['trials']
bucket_id = options['bucket']
pool = ProcessPool(max_workers=2)
pool._start_pool()
bucket, _ = Counter.objects.get_or_create(bucket=bucket_id)
bucket.count = 0
bucket.save()
future_1 = pool.schedule(run_atomic_transactions,
args=('T1', bucket_id, trials))
future_2 = pool.schedule(run_savepoints,
args=('T2', bucket_id, trials))
pool.close()
pool.join()
class PebbleExecutor(concurrent.futures.Executor):
def __init__(self, max_workers, timeout=None):
self.pool = ProcessPool(max_workers=max_workers)
self.timeout = timeout
def submit(self, fn, *args, **kwargs):
return self.pool.schedule(fn, args=args, timeout=self.timeout)
def map(self, func, *iterables, timeout=None, chunksize=1):
raise NotImplementedError("This wrapper does not support `map`.")
def shutdown(self, wait=True):
if wait:
log.info("Closing workers...")
self.pool.close()
else:
log.info("Ending workers...")
self.pool.stop()
self.pool.join()
log.info("Workers joined.")
class Wintermute(discord.Client):
__channels = None
__bot_prelude = None
__timeout = None
__parser = None
__pool = None
def __init__(self,
channels={},
bot_prelude='[bot] ',
timeout=10,
multiprocessing=1,
loglevel=logging.INFO):
super().__init__()
self.__channels = channels
self.__bot_prelude = bot_prelude
self.__timeout = timeout
logging.basicConfig(level=loglevel)
self.__parser = BotGram(prelude=bot_prelude)
self.__pool = ProcessPool(max_workers=multiprocessing,
initializer=seed)
def __del__(self):
self.__pool.close()
self.__pool.join()
async def on_ready(self):
logging.info("Online as " + str(self.user.name))
game = discord.Game()
game.name = "Manipulating humanity"
await self.change_presence(game=game)
logging.info('Setup done')
async def on_message(self, mess):
if mess.channel.is_private:
logging.info("new private message from " + str(mess.author))
else:
logging.info("new message on server " + str(mess.server) +
" and channel " + str(mess.channel) + " from " +
str(mess.author))
logging.info("len: " + str(len(mess.content)))
logging.info("content: " + str(mess.content))
if mess.channel.is_private:
return
if mess.author == self.user:
return
if (mess.channel.permissions_for(mess.channel.server.me).send_messages
and mess.channel.name == self.__channels[mess.server.name]):
resp = self.__parser.parse(mess)
if resp is not None:
task = self.__pool.schedule(str,
args=(resp, ),
timeout=self.__timeout)
asyncio.ensure_future(self.collect_response(
mess.channel, mess.author.mention, task),
loop=self.loop)
async def collect_response(self, channel, mention, task):
try:
while not task.done():
await asyncio.sleep(0.1)
result = task.result()
await self.send_message(channel, result)
except TimeoutError:
await self.send_message(
channel,
self.__bot_prelude + mention + ' Your request timed out')
task.cancel()
except HTTPException as e:
if e.response.status == 400:
await self.send_message(
channel, self.__bot_prelude + mention +
' Error: Request reply was probably too long')
