def one_explo(param, param_to_explore):
PARAMFILE = "params.xml"
MEASUREFILE = "measures.xml"
result_path = "/media/thor/data/ktsimu/small_explo_ld_slope_ld0"
result_path = "/home/hadim/local/data/ktsimu/explo_ld_slope_ld0"
number_simu = 500
name = "explo_%s_%s" % (param['name'], str(param['value']))
ncore = 4
paramtree = ParamTree(PARAMFILE)
paramtree.change_dic(param['name'], param['value'])
explorator = Explorator(result_path,
number_simu,
name=name,
ncore=ncore,
paramtree=paramtree,
measurefile=MEASUREFILE,
parameter_to_explore=parameter_to_explore,
pool_eval=True,
name_without_date=True
)
explorator.run()
p = ProcessExploration(results_path=explorator.results_path)
p.evaluate(run_all=True, debug=True)
del explorator
del p
gc.collect()
def __init__(self, results_path,
nsimu,
name="",
ncore=None,
paramtree=None,
measuretree=None,
paramfile=PARAMFILE,
measurefile=MEASUREFILE,
verbose=True,
name_without_date=False):
"""
:results_path: The path where simulation results are stored
:type results_path: string
:nsimu: Number of simulation to run
:type nsimu: int
:ncore: Number of process to launch at the same time
:type ncore: int
:param paramtree: The paramtree contains the parameters for the simulation
if paramtree is None, the parameters are read
from the file paramfile. Defaults to None.
:type paramtree: ParamTree instance or None
:param measuretree: The measuretree contains the observed characteristics
of the mitosis e.g. metaphase spindle elongation rate, etc.
if measuretree is None, the measures are read from the file
indicated by the measurefile argument. Defaults to None.
:type measuretree: ParamTree instance or None
:param paramfile: Path to a xml file to read the parameters from. Defaults to the
file params.xml in the module's default directory. Other parameter
files can be produced by editing and changing the default one.
If the paramtree argument is not None, paramfile is ignored
:type paramfile: string
:param measurefile: Path to a xml file to read the measures from. Defaults to the
file measures.xml in the module's default directory.
Other measure files can be produced by editing and changing
the default one. If the measuretree argument is not None, measurefile
is ignored
:type measurefile: string
"""
# Enable or disable log console
self.verbose = verbose
logger = logging.getLogger(__name__)
if not self.verbose:
logger.disabled = True
else:
logger.disabled = False
if paramtree is None:
self.paramtree = ParamTree(paramfile)
else:
self.paramtree = paramtree
if measuretree is None:
self.measuretree = ParamTree(measurefile, adimentionalized=False)
else:
self.measuretree = measuretree
# Reduce parameters
parameters.reduce_params(self.paramtree, self.measuretree)
self.results_path = results_path
self.nsimu = nsimu
if not ncore:
self.ncore = multiprocessing.cpu_count()
else:
self.ncore = ncore
self.last_progress = 0
self.name = name
# Make result directory
if not os.path.exists(self.results_path):
os.makedirs(self.results_path)
# Results directory according to date and time
if name_without_date and name:
dirname = name
else:
now = datetime.datetime.now()
if name:
dirname = now.strftime("%Y.%m.%d") + "_" + name
else:
dirname = now.strftime("%Y.%m.%d")
self.results_path = os.path.join(self.results_path, dirname)
# TODO: Ugly fix
if os.path.isdir(self.results_path):
self.results_path += "_2"
# Remove existing directory of it exist
if os.path.exists(self.results_path):
shutil.rmtree(self.results_path)
os.makedirs(self.results_path)
# Save params.xml and measures.xml in results dir
self.paramtree.save(os.path.join(self.results_path, "params.xml"))
self.measuretree.save(os.path.join(self.results_path, "measures.xml"))
self.raw_results_path = os.path.join(self.results_path, "raw")
os.makedirs(self.raw_results_path)
# Redirect log to run.log
logfile = os.path.join(self.results_path, "run.log")
handler = logging.FileHandler(logfile)
logging.getLogger(__name__).addHandler(handler)
class Launcher:
"""
"""
def __init__(self, results_path,
nsimu,
name="",
ncore=None,
paramtree=None,
measuretree=None,
paramfile=PARAMFILE,
measurefile=MEASUREFILE,
verbose=True,
name_without_date=False):
"""
:results_path: The path where simulation results are stored
:type results_path: string
:nsimu: Number of simulation to run
:type nsimu: int
:ncore: Number of process to launch at the same time
:type ncore: int
:param paramtree: The paramtree contains the parameters for the simulation
if paramtree is None, the parameters are read
from the file paramfile. Defaults to None.
:type paramtree: ParamTree instance or None
:param measuretree: The measuretree contains the observed characteristics
of the mitosis e.g. metaphase spindle elongation rate, etc.
if measuretree is None, the measures are read from the file
indicated by the measurefile argument. Defaults to None.
:type measuretree: ParamTree instance or None
:param paramfile: Path to a xml file to read the parameters from. Defaults to the
file params.xml in the module's default directory. Other parameter
files can be produced by editing and changing the default one.
If the paramtree argument is not None, paramfile is ignored
:type paramfile: string
:param measurefile: Path to a xml file to read the measures from. Defaults to the
file measures.xml in the module's default directory.
Other measure files can be produced by editing and changing
the default one. If the measuretree argument is not None, measurefile
is ignored
:type measurefile: string
"""
# Enable or disable log console
self.verbose = verbose
logger = logging.getLogger(__name__)
if not self.verbose:
logger.disabled = True
else:
logger.disabled = False
if paramtree is None:
self.paramtree = ParamTree(paramfile)
else:
self.paramtree = paramtree
if measuretree is None:
self.measuretree = ParamTree(measurefile, adimentionalized=False)
else:
self.measuretree = measuretree
# Reduce parameters
parameters.reduce_params(self.paramtree, self.measuretree)
self.results_path = results_path
self.nsimu = nsimu
if not ncore:
self.ncore = multiprocessing.cpu_count()
else:
self.ncore = ncore
self.last_progress = 0
self.name = name
# Make result directory
if not os.path.exists(self.results_path):
os.makedirs(self.results_path)
# Results directory according to date and time
if name_without_date and name:
dirname = name
else:
now = datetime.datetime.now()
if name:
dirname = now.strftime("%Y.%m.%d") + "_" + name
else:
dirname = now.strftime("%Y.%m.%d")
self.results_path = os.path.join(self.results_path, dirname)
# TODO: Ugly fix
if os.path.isdir(self.results_path):
self.results_path += "_2"
# Remove existing directory of it exist
if os.path.exists(self.results_path):
shutil.rmtree(self.results_path)
os.makedirs(self.results_path)
# Save params.xml and measures.xml in results dir
self.paramtree.save(os.path.join(self.results_path, "params.xml"))
self.measuretree.save(os.path.join(self.results_path, "measures.xml"))
self.raw_results_path = os.path.join(self.results_path, "raw")
os.makedirs(self.raw_results_path)
# Redirect log to run.log
logfile = os.path.join(self.results_path, "run.log")
handler = logging.FileHandler(logfile)
logging.getLogger(__name__).addHandler(handler)
def run(self):
"""
"""
logger.info("Starting %i simulations on %i core" % \
(self.nsimu, self.ncore))
if in_ipython():
parallel = False
else:
parallel = True
if parallel and os.name == 'posix':
os.system("taskset -p 0xff %d" % os.getpid())
params = [self.paramtree, self.measuretree, self.verbose]
all_params = []
for i in range(self.nsimu):
all_params.append([i, self.raw_results_path] + params)
queue = Queue()
self.start_time = time.time()
p = Pool(self.ncore, run_init, [queue], maxtasksperchild=4)
p.map_async(run_one, all_params, chunksize=1)
# Monitoring simulations
done = False
simu_ids = range(self.nsimu)
while not done:
mess = queue.get()
if mess["state"] == "stop":
simu_ids.remove(mess["id"])
self.log_progress(len(simu_ids))
if not simu_ids:
done = True
self.total_time = time.time() - self.start_time
p.close()
results_size = get_folder_size(self.results_path)
logger.info("Simulations are done")
logger.info("Results are stored in %s (%s MB)" % (self.results_path, results_size))
self.create_log()
del p
del queue
gc.collect()
def log_progress(self, simu_left, precision=2):
"""
Log the progression of the simulations
:param simu_left: Number of simulation remaining
:type simu_left: int
"""
progress = 100.0 - ((simu_left * 100.0) / self.nsimu)
progress = round(progress, precision)
# Don't print the same progression !
if progress > self.last_progress:
time_remaining, spent_time = self.estimate_remaining_time(simu_left)
output = "Progression : %0.2f%% | " % progress
output += "ETA : %s | " % time_remaining
output += "Spent time : %s" % spent_time
logger.info(output)
self.last_progress = progress
def estimate_remaining_time(self, simu_left):
"""
Estimate remaining time
"""
delta_time = time.time() - self.start_time
delta_simu = self.nsimu - simu_left
estimate_time = (simu_left * delta_time) / delta_simu
estimate_time = time.strftime('%H:%M:%S', time.gmtime(estimate_time))
spent_time = time.strftime('%H:%M:%S', time.gmtime(delta_time))
return estimate_time, spent_time
def create_log(self):
"""
Create logfile in results folder
"""
log = {}
log["name"] = self.name
log["duration"] = self.paramtree.absolute_dic['span']
log["dt"] = self.paramtree.absolute_dic['dt']
log["num_steps"] = int(log["duration"] / log["dt"])
log["number_of_simulations"] = self.nsimu
log["number_of_simulations"] = self.nsimu
log["spent_time"] = time.strftime('%H:%M:%S', time.gmtime(self.total_time))
log["results_folder_size_in_MB"] = get_folder_size(self.results_path)
f = open(os.path.join(self.results_path, "simu.log"), 'w')
f.write(json.dumps(log, sort_keys=True, indent=4))
f.close()
import argparse
import sys
sys.path.append('..')
from kt_simul.io.xml_handler import ParamTree
from kt_simul.core.simul_spindle import Metaphase
from kt_simul.io.simuio import SimuIO
from kt_simul.core import parameters
if __name__ == '__main__':
PARAMFILE = parameters.PARAMFILE
MEASUREFILE = parameters.MEASUREFILE
# Change some parameters
paramtree = ParamTree(PARAMFILE)
paramtree.change_dic('dt', 10)
paramtree.change_dic('span', 2000)
paramtree.change_dic('t_A', 1750)
measuretree = ParamTree(MEASUREFILE, adimentionalized=False)
# Arguments parser
parser = argparse.ArgumentParser(description='KtSimu Launcher')
parser.add_argument('--new', '-n', action='store_true', default=False,
help="Run a new simulation")
parser.add_argument('--results', '-r', type=str, default="simu.h5",
help="Specified results hdf5 file for new or already runned simulation")
args = parser.parse_args()
results_file = args.results