async def func(work=work):
while True:
while (
isinstance(work.input_queue, tuple)
and all(not q for q in work.input_queue)
) or not work.input_queue:
await trio.sleep(self.sleep_time)
if self._has_to_stop:
return
t_start = time.time()
log_memory_usage(
f"{time.time() - self.t_start:.2f} s. Launch work "
+ work.name_no_space
+ f" (?). mem usage"
)
work.func_or_cls(work.input_queue, work.output_queue)
if self._has_to_stop:
return
await trio.sleep(self.sleep_time)
logger.info(
f"work {work.name_no_space} "
f"done in {time.time() - t_start:.3f} s"
)
await trio.sleep(self.sleep_time)
def _init_compute_log(self):
log_memory_usage(time_as_str(2) + ": starting execution. mem usage")
logger.info(f" topology: {str_short(type(self.topology))}")
logger.info(f" executor: {str_short(type(self))}")
logger.info(f" nb_cpus_allowed = {nb_cores}")
logger.info(f" nb_max_workers = {self.nb_max_workers}")
logger.info(f" path_dir_result = {self.path_dir_result}")
async def async_run_work_cpu(self, work):
"""Executes the work on an item (key, obj), and add the result on
work.output_queue.
Parameters
----------
work :
A work from the topology
key : hashable
The key of the dictionnary item to be process
obj : object
The value of the dictionnary item to be process
"""
self.nb_working_workers_cpu += 1
try:
key, obj = work.input_queue.pop_first_item()
except KeyError:
self.nb_working_workers_cpu -= 1
return
if work.check_exception(key, obj):
self.nb_working_workers_cpu -= 1
return
t_start = time.time()
log_memory_usage(f"{time.time() - self.t_start:.2f} s. Launch work " +
work.name_no_space + f" ({key}). mem usage")
# pylint: disable=W0703
try:
# here we do something very bad from the async point of view:
# we launch a potentially long blocking function:
ret = work.func_or_cls(obj)
except Exception as error:
self.log_exception(error, work.name_no_space, key)
if self.stop_if_error:
raise
ret = error
else:
logger.info(f"work {work.name_no_space} ({key}) "
f"done in {time.time() - t_start:.3f} s")
if work.output_queue is not None:
work.output_queue[key] = ret
self.nb_working_workers_cpu -= 1
async def async_run_work_cpu(self, work):
"""Is destined to be started with a "trio.start_soon".
Executes the work on an item (key, obj), and add the result on
work.output_queue.
Parameters
----------
work :
A work from the topology
"""
self.nb_working_workers_cpu += 1
try:
key, obj = work.input_queue.pop_first_item()
except KeyError:
self.nb_working_workers_cpu -= 1
return
if work.check_exception(key, obj):
self.nb_working_workers_cpu -= 1
return
t_start = time.time()
log_memory_usage(
f"{time.time() - self.t_start:.2f} s. Launch work "
+ work.name_no_space
+ f" ({key}). mem usage"
)
# pylint: disable=W0703
try:
ret = await trio.run_sync_in_worker_thread(work.func_or_cls, obj)
except Exception as error:
self.log_exception(error, work.name_no_space, key)
if self.stop_if_error:
raise
ret = error
else:
logger.info(
f"work {work.name_no_space} ({key}) "
f"done in {time.time() - t_start:.3f} s"
)
if work.output_queue is not None:
work.output_queue[key] = ret
self.nb_working_workers_cpu -= 1
def _run_works(self):
while not all([len(queue) == 0 for queue in self.topology.queues]):
for work in self.works:
# global functions
if work.kind is not None and "global" in work.kind:
if len(work.output_queue) > self.nb_items_queue_max:
continue
work.func_or_cls(work.input_queue, work.output_queue)
else:
if not work.input_queue:
continue
key, obj = work.input_queue.pop_first_item()
if work.check_exception(key, obj):
continue
t_start = time.time()
log_memory_usage(
f"{time.time() - self.t_start:.2f} s. Launch work " +
work.name_no_space + f" ({key}). mem usage")
# pylint: disable=W0703
try:
ret = work.func_or_cls(obj)
except Exception as error:
self.log_exception(error, work.name_no_space, key)
if self.stop_if_error:
raise
ret = error
else:
logger.info(f"work {work.name_no_space} ({key}) "
f"done in {time.time() - t_start:.3f} s")
if work.output_queue is not None:
work.output_queue[key] = ret
def _finalize_compute(self):
log_memory_usage(time_as_str(2) + ": end of `compute`. mem usage")
self.topology.print_at_exit(time() - self.t_start)
self._reset_std_as_default()
async def async_run_work_cpu(self, work):
"""Is destined to be started with a "trio.start_soon".
Executes the work on an item (key, obj), and add the result on
work.output_queue.
Parameters
----------
work :
A work from the topology
"""
self.nb_working_workers_cpu += 1
try:
key, obj = work.input_queue.pop_first_item()
except KeyError:
self.nb_working_workers_cpu -= 1
if work.check_exception(key, obj):
self.nb_working_workers_cpu -= 1
return
t_start = time.time()
log_memory_usage(
f"{time.time() - self.t_start:.2f} s. Launch work "
+ work.name_no_space
+ f" ({key}). mem usage"
)
def exec_work_and_comm(func, obj, child_conn, event):
# log_debug(f"process ({key}) started")
event.set()
# pylint: disable=W0703
try:
result = func(obj)
except Exception as error:
result = error
# log_debug(f"in process, send result ({key}): {result}")
child_conn.send(result)
parent_conn, child_conn = Pipe()
event = Event()
def run_process():
# we do this complicate thing because there may be a strange bug
def start_process_and_check(index_attempt):
process = Process(
target=exec_work_and_comm,
args=(work.func_or_cls, obj, child_conn, event),
)
process.daemon = True
process.start()
# check whether the process has really started (possible bug!)
if not event.wait(1):
log_debug(
f"problem: process {work.name_no_space} ({key}) "
f"has not really started... (attempt {index_attempt})"
)
process.terminate()
return False
return process
really_started = False
for index_attempt in range(10):
process = start_process_and_check(index_attempt)
if process:
really_started = True
break
if not really_started:
raise Exception(
f"A process {work.name_no_space} ({key}) "
"has not started after 10 attempts"
)
# todo: use parent_conn.poll to implement a timeout
# log_debug(f"waiting for result ({key})")
result = parent_conn.recv()
# log_debug(f"result ({key}) received")
process.join(10 * self.sleep_time)
if process.exitcode != 0:
logger.info(f"process.exitcode: {process.exitcode}")
process.terminate()
return result
ret = await trio.run_sync_in_worker_thread(run_process)
if isinstance(ret, Exception):
self.log_exception(ret, work.name_no_space, key)
if self.stop_if_error:
raise ret
else:
logger.info(
f"work {work.name_no_space} ({key}) "
f"done in {time.time() - t_start:.3f} s"
)
if work.output_queue is not None:
work.output_queue[key] = ret
self.nb_working_workers_cpu -= 1
def compute(self, sequential=None, has_to_exit=True):
"""Compute (run all works to be done).
Parameters
----------
sequential : None
If bool(sequential) is True, the computations are run in sequential
(useful for debugging).
has_to_exit : True
If bool(has_to_exit) is True and if the computation has to stop
because of a signal 12 (cluster), a signal 99 is sent at exit.
"""
if hasattr(self, "path_output"):
logger.info("path results:\n" + str(self.path_output))
if hasattr(self, "params"):
tmp_path_params = str(
self.path_output /
("params_" + time_as_str() + f"_{os.getpid()}"))
if not os.path.exists(tmp_path_params + ".xml"):
path_params = tmp_path_params + ".xml"
else:
i = 1
while os.path.exists(tmp_path_params + "_" + str(i) +
".xml"):
i += 1
path_params = tmp_path_params + "_" + str(i) + ".xml"
self.params._save_as_xml(path_params)
self.t_start = time()
log_memory_usage(time_as_str(2) + ": starting execution. mem usage")
self.nb_workers_cpu = 0
self.nb_workers_io = 0
workers = []
class CheckWorksThread(threading.Thread):
cls_to_be_updated = threading.Thread
def __init__(self):
self.has_to_stop = False
super().__init__()
self.exitcode = None
self.daemon = True
def in_time_loop(self):
t_tmp = time()
for worker in workers:
if (isinstance(worker, self.cls_to_be_updated)
and worker.fill_destination()):
workers.remove(worker)
t_tmp = time() - t_tmp
if t_tmp > 0.2:
logger.info("update list of workers with fill_destination "
"done in {:.3f} s".format(t_tmp))
sleep(dt_update)
def run(self):
try:
while not self.has_to_stop:
self.in_time_loop()
except Exception as e:
print("Exception in UpdateThread")
self.exitcode = 1
self.exception = e
class CheckWorksProcess(CheckWorksThread):
cls_to_be_updated = Process
def in_time_loop(self):
# weird bug subprocessing py3
for worker in workers:
if not worker.really_started:
# print('check if worker has really started.' +
# worker.key)
try:
worker.really_started = (
worker.comm_started.get_nowait())
except queue.Empty:
pass
if (not worker.really_started
and time() - worker.t_start > 10):
# bug! The worker does not work. We kill it! :-)
logger.error(
cstring(
"Mysterious bug multiprocessing: "
"a launched worker has not started. "
"We kill it! ({}, key: {}).".format(
worker.work_name, worker.key),
color="FAIL",
))
# the case of this worker has been
worker.really_started = True
worker.terminate()
super().in_time_loop()
self.thread_check_works_t = CheckWorksThread()
self.thread_check_works_t.start()
self.thread_check_works_p = CheckWorksProcess()
self.thread_check_works_p.start()
while not self._has_to_stop and (any(
[not q.is_empty() for q in self.queues]) or len(workers) > 0):
# debug
# if logger.level == 10 and \
# all([q.is_empty() for q in self.queues]) and len(workers) == 1:
# for worker in workers:
# try:
# is_alive = worker.is_alive()
# except AttributeError:
# is_alive = None
# logger.debug(
# str((worker, worker.key, worker.exitcode, is_alive)))
# if time() - worker.t_start > 60:
# from fluiddyn import ipydebug
# ipydebug()
self.nb_workers = len(workers)
# slow down this loop...
sleep(dt_small)
if self.nb_workers_cpu >= nb_max_workers:
logger.debug(
cstring(
("The workers are saturated: "
"{}, sleep {} s").format(self.nb_workers_cpu, dt),
color="WARNING",
))
sleep(dt)
for q in self.queues:
if not q.is_empty():
logger.debug(q)
logger.debug("check_and_act for work: " + repr(q.work))
try:
new_workers = q.check_and_act(sequential=sequential)
except OSError:
logger.error(
cstring(
"Memory full: to free some memory, no more "
"computing job will be launched while the last "
"(saving) waiting queue is not empty.",
color="FAIL",
))
log_memory_usage(color="FAIL", mode="error")
self._clear_save_queue(workers, sequential)
logger.info(
cstring(
"The last waiting queue has been emptied.",
color="FAIL",
))
log_memory_usage(color="FAIL", mode="info")
continue
if new_workers is not None:
for worker in new_workers:
workers.append(worker)
logger.debug("workers: " + repr(workers))
if self.thread_check_works_t.exitcode:
raise self.thread_check_works_t.exception
if self.thread_check_works_p.exitcode:
raise self.thread_check_works_p.exception
if len(workers) != self.nb_workers:
gc.collect()
if self._has_to_stop:
logger.info(
cstring(
"Will exist because of signal 12.",
"Waiting for all workers to finish...",
color="FAIL",
))
self._clear_save_queue(workers, sequential)
self.thread_check_works_t.has_to_stop = True
self.thread_check_works_p.has_to_stop = True
self.thread_check_works_t.join()
self.thread_check_works_p.join()
self.print_at_exit(time() - self.t_start)
log_memory_usage(time_as_str(2) + ": end of `compute`. mem usage")
if self._has_to_stop and has_to_exit:
logger.info(cstring("Exit with signal 99.", color="FAIL"))
exit(99)
self._reset_std_as_default()