def render_trial_results_par(self, opts=None):
"""
Render all trials in parallel
:return: None
"""
import multiprocessing
from cc3d.CompuCellSetup.CC3DCaller import CC3DCallerWorker
# Start workers
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
workers = [CC3DCallerWorker(tasks, results) for _ in range(self.num_workers)]
[w.start() for w in workers]
# Enqueue jobs
import time
for r in range(len(self.data_dirs)):
while tasks.full():
time.sleep(1)
tasks.put(_RenderDataJob(self.data_dirs[r], self.out_dirs[r], self.set_labs[r], self.run_labs[r], opts))
# Add a stop task for each of worker
for _ in workers:
while tasks.full():
time.sleep(1)
tasks.put(None)
tasks.join()
def export_all(self, num_workers: int = 1):
"""
Render and export spatial data in parallel
:param num_workers: {int} number of threads to do batch rendering
:return: None
"""
assert num_workers > 0
import multiprocessing
from cc3d.CompuCellSetup.CC3DCaller import CC3DCallerWorker
# Start workers
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
workers = [
CC3DCallerWorker(tasks, results) for _ in range(num_workers)
]
[w.start() for w in workers]
# Enqueue jobs
for r in range(len(self.lds_files)):
self._put_with_wait(
tasks,
_RenderDataJob(self.lds_files[r], self.screenshot_spec[r],
self.output_dirs[r], self.manipulators[r]))
# Add a stop task for each of worker
[self._put_with_wait(tasks, None) for _ in workers]
tasks.join()
def sim_run_single(cov2_vtm_sim_run: CoV2VTMSimRunAsync,
run_idx=0,
status: multiprocessing.Array = None):
# Start worker
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
worker = CC3DCallerWorker(tasks, results)
worker.start()
# Enqueue jobs
tasks.put(cov2_vtm_sim_run.generate_callable(run_idx))
print(f'CC3DCallerWorker {worker.name} launched')
# Add a stop task for each of worker
tasks.put(None)
# Monitor worker state
monitor_rate = 1
while worker.is_alive():
time.sleep(monitor_rate)
print(
f'CC3DCallerWorker {worker.name} finished with exit code {worker.exitcode}'
)
if worker.exitcode == 0:
# Fetch available results
try:
result = results.get(block=True, timeout=10)
sim_output = result['result']
except:
print(
f'CC3DCallerWorker {worker.name} returned no async result {run_idx}'
)
if status is not None:
status[run_idx] = -1
return False
print(
f'CC3DCallerWorker {worker.name} returned async result {run_idx}')
cov2_vtm_sim_run.sim_output[run_idx] = sim_output
cov2_vtm_sim_run.write_sim_inputs(run_idx)
if status is not None:
status[run_idx] = 2
return True
else:
if status is not None:
status[run_idx] = -1
return False
def run_cov2_vtm_sims(cov2_vtm_sim_run: CoV2VTMSimRun) -> CoV2VTMSimRun:
# Make complete list of jobs
run_list = [run_idx for run_idx in range(cov2_vtm_sim_run.num_runs)]
while True:
num_jobs_curr = len(run_list)
print('Doing CoV2VTMSimRun batch iteration with {} remaining jobs.'.format(num_jobs_curr))
# Start workers
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
workers = [CC3DCallerWorker(tasks, results) for i in range(cov2_vtm_sim_run.num_workers)]
[w.start() for w in workers]
# Enqueue jobs
[tasks.put(cov2_vtm_sim_run.generate_callable(run_idx)) for run_idx in run_list]
# Add a stop task for each of worker
[tasks.put(None) for w in workers]
# Monitor worker state
monitor_rate = 1
while [w for w in workers if w.is_alive()]:
time.sleep(monitor_rate)
# Fetch available results
while True:
result = results.get()
run_idx = result['tag']
sim_output = result['result']
print('Got CoV2VTMSimRun batch result {}'.format(run_idx))
cov2_vtm_sim_run.sim_output[run_idx] = sim_output
cov2_vtm_sim_run.write_sim_inputs(run_idx)
run_list.remove(run_idx)
if results.empty():
break
num_jobs_next = len(run_list)
print('CoV2VTMSimRun batch results processed with {} remaining jobs.'.format(len(run_list)))
[print('CC3DCallerWorker {} finished with exit code {}'.format(w.name, w.exitcode)) for w in workers]
if not run_list:
print('CoV2VTMSimRun batch complete!')
break
elif num_jobs_next == num_jobs_curr:
print('CoV2VTMSimRun batch run failed! Terminating early.')
break
return cov2_vtm_sim_run
def main():
num_workers = 4
number_of_runs = 10
# You may put a direct path to a simulation of your choice here and comment out simulation_fname line below
# simulation_fname = <direct path your simulation>
simulation_fname = join(dirname(dirname(__file__)), 'cellsort_2D',
'cellsort_2D.cc3d')
root_output_folder = join(expanduser('~'), 'CC3DCallerOutput')
# this creates a list of simulation file names where simulation_fname is repeated number_of_runs times
# you can create a list of different simulations if you want.
sim_fnames = [simulation_fname] * number_of_runs
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
# Start workers
num_consumers = multiprocessing.cpu_count() * 2
print('Creating %d consumers' % num_consumers)
workers = [CC3DCallerWorker(tasks, results) for i in range(num_workers)]
for w in workers:
w.start()
# Enqueue jobs
for i, sim_fname in enumerate(sim_fnames):
cc3d_caller = CC3DCaller(cc3d_sim_fname=sim_fname,
screenshot_output_frequency=10,
output_dir=join(root_output_folder,
f'cellsort_{i}'),
result_identifier_tag=i)
tasks.put(cc3d_caller)
# Add a stop task "poison pill" for each of the workers
for i in range(num_workers):
tasks.put(None)
# Wait for all of the tasks to finish
tasks.join()
# Start printing results
while number_of_runs:
result = results.get()
print('Result:', result)
number_of_runs -= 1
def run_trials(num_runs, iteration_num, lam_trial):
"""
Runs simulation and store simulation results
:param num_runs: number of simulation runs
:param iteration_num: integer label for storing results according to an iteration
:param lam_trial: chemotactic Lagrange multiplier to simulate
:return: mean horizontal center of mass over all runs
"""
root_output_folder = join(res_output_root, f'iteration_{iteration_num}')
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
# Start workers
workers = [CC3DCallerWorker(tasks, results) for i in range(num_workers)]
[w.start() for w in workers]
# Enqueue jobs
for i in range(num_runs):
cc3d_caller = CC3DCaller(cc3d_sim_fname=simulation_fname,
output_frequency=10,
screenshot_output_frequency=10,
output_dir=join(root_output_folder, f'trial_{i}'),
result_identifier_tag=i,
sim_input=lam_trial
)
tasks.put(cc3d_caller)
# Add a stop task for each of worker
for i in range(num_workers):
tasks.put(None)
# Wait for all of the tasks to finish
tasks.join()
# Return mean result
trial_results = []
while num_runs:
result = results.get()
trial_results.append(result['result'])
num_runs -= 1
return np.average(trial_results)
def render_trial_results_par(self, opts=None):
"""
Render all trials in parallel
:return: None
"""
import multiprocessing
from cc3d.CompuCellSetup.CC3DCaller import CC3DCallerWorker
# Start workers
tasks = multiprocessing.JoinableQueue()
results = multiprocessing.Queue()
workers = [CC3DCallerWorker(tasks, results) for _ in range(self.cov2_vtm_sim_run.num_workers)]
[w.start() for w in workers]
# Enqueue jobs
num_runs = len(self.cov2_vtm_sim_run.get_trial_dirs())
[tasks.put(_RenderJob(self.cov2_vtm_sim_run, run_idx, opts)) for run_idx in range(num_runs)]
# Add a stop task for each of worker
[tasks.put(None) for _ in workers]
tasks.join()