Exemplo n.º 1
0
def one_explo(param, param_to_explore):

    PARAMFILE = "params.xml"
    MEASUREFILE = "measures.xml"

    result_path = "/media/thor/data/ktsimu/small_explo_ld_slope_ld0"
    result_path = "/home/hadim/local/data/ktsimu/explo_ld_slope_ld0"
    number_simu = 500
    name = "explo_%s_%s" % (param['name'], str(param['value']))
    ncore = 4

    paramtree = ParamTree(PARAMFILE)
    paramtree.change_dic(param['name'], param['value'])

    explorator = Explorator(result_path,
                            number_simu,
                            name=name,
                            ncore=ncore,
                            paramtree=paramtree,
                            measurefile=MEASUREFILE,
                            parameter_to_explore=parameter_to_explore,
                            pool_eval=True,
                            name_without_date=True
                            )
    explorator.run()

    p = ProcessExploration(results_path=explorator.results_path)
    p.evaluate(run_all=True, debug=True)

    del explorator
    del p
    gc.collect()
Exemplo n.º 2
0
    def __init__(self, results_path,
                 nsimu,
                 name="",
                 ncore=None,
                 paramtree=None,
                 measuretree=None,
                 paramfile=PARAMFILE,
                 measurefile=MEASUREFILE,
                 verbose=True,
                 name_without_date=False):
        """

        :results_path: The path where simulation results are stored
        :type results_path: string

        :nsimu: Number of simulation to run
        :type nsimu: int

        :ncore: Number of process to launch at the same time
        :type ncore: int

        :param paramtree: The paramtree contains the parameters for the simulation
            if paramtree is None, the parameters are read
            from the file paramfile. Defaults to None.
        :type paramtree: ParamTree instance or None

        :param measuretree: The measuretree contains the observed characteristics
            of the mitosis e.g. metaphase spindle elongation rate, etc.
            if measuretree is None, the measures are read from the file
            indicated by the measurefile argument. Defaults to None.
        :type measuretree: ParamTree instance or None

        :param paramfile: Path to a xml file to read the parameters from. Defaults to the
            file params.xml in the module's default directory. Other parameter
            files can be produced by editing and changing the default one.
            If the paramtree argument is not None,  paramfile is ignored
        :type paramfile: string

        :param measurefile: Path to a xml file to read the measures from. Defaults to the
            file measures.xml in the module's default directory.
            Other measure files can be produced by editing and changing
            the default one. If the measuretree argument is not None, measurefile
            is ignored
        :type measurefile: string


        """

        # Enable or disable log console
        self.verbose = verbose
        logger = logging.getLogger(__name__)
        if not self.verbose:
            logger.disabled = True
        else:
            logger.disabled = False

        if paramtree is None:
            self.paramtree = ParamTree(paramfile)
        else:
            self.paramtree = paramtree
        if measuretree is None:
            self.measuretree = ParamTree(measurefile, adimentionalized=False)
        else:
            self.measuretree = measuretree

        # Reduce parameters
        parameters.reduce_params(self.paramtree, self.measuretree)

        self.results_path = results_path
        self.nsimu = nsimu
        if not ncore:
            self.ncore = multiprocessing.cpu_count()
        else:
            self.ncore = ncore

        self.last_progress = 0

        self.name = name

        # Make result directory
        if not os.path.exists(self.results_path):
            os.makedirs(self.results_path)

        # Results directory according to date and time
        if name_without_date and name:
            dirname = name
        else:
            now = datetime.datetime.now()
            if name:
                dirname = now.strftime("%Y.%m.%d") + "_" + name
            else:
                dirname = now.strftime("%Y.%m.%d")

        self.results_path = os.path.join(self.results_path, dirname)

        # TODO: Ugly fix
        if os.path.isdir(self.results_path):
            self.results_path += "_2"

        # Remove existing directory of it exist
        if os.path.exists(self.results_path):
            shutil.rmtree(self.results_path)
        os.makedirs(self.results_path)

        # Save params.xml and measures.xml in results dir
        self.paramtree.save(os.path.join(self.results_path, "params.xml"))
        self.measuretree.save(os.path.join(self.results_path, "measures.xml"))

        self.raw_results_path = os.path.join(self.results_path, "raw")
        os.makedirs(self.raw_results_path)

        # Redirect log to run.log
        logfile = os.path.join(self.results_path, "run.log")
        handler = logging.FileHandler(logfile)
        logging.getLogger(__name__).addHandler(handler)
Exemplo n.º 3
0
class Launcher:
    """
    """

    def __init__(self, results_path,
                 nsimu,
                 name="",
                 ncore=None,
                 paramtree=None,
                 measuretree=None,
                 paramfile=PARAMFILE,
                 measurefile=MEASUREFILE,
                 verbose=True,
                 name_without_date=False):
        """

        :results_path: The path where simulation results are stored
        :type results_path: string

        :nsimu: Number of simulation to run
        :type nsimu: int

        :ncore: Number of process to launch at the same time
        :type ncore: int

        :param paramtree: The paramtree contains the parameters for the simulation
            if paramtree is None, the parameters are read
            from the file paramfile. Defaults to None.
        :type paramtree: ParamTree instance or None

        :param measuretree: The measuretree contains the observed characteristics
            of the mitosis e.g. metaphase spindle elongation rate, etc.
            if measuretree is None, the measures are read from the file
            indicated by the measurefile argument. Defaults to None.
        :type measuretree: ParamTree instance or None

        :param paramfile: Path to a xml file to read the parameters from. Defaults to the
            file params.xml in the module's default directory. Other parameter
            files can be produced by editing and changing the default one.
            If the paramtree argument is not None,  paramfile is ignored
        :type paramfile: string

        :param measurefile: Path to a xml file to read the measures from. Defaults to the
            file measures.xml in the module's default directory.
            Other measure files can be produced by editing and changing
            the default one. If the measuretree argument is not None, measurefile
            is ignored
        :type measurefile: string


        """

        # Enable or disable log console
        self.verbose = verbose
        logger = logging.getLogger(__name__)
        if not self.verbose:
            logger.disabled = True
        else:
            logger.disabled = False

        if paramtree is None:
            self.paramtree = ParamTree(paramfile)
        else:
            self.paramtree = paramtree
        if measuretree is None:
            self.measuretree = ParamTree(measurefile, adimentionalized=False)
        else:
            self.measuretree = measuretree

        # Reduce parameters
        parameters.reduce_params(self.paramtree, self.measuretree)

        self.results_path = results_path
        self.nsimu = nsimu
        if not ncore:
            self.ncore = multiprocessing.cpu_count()
        else:
            self.ncore = ncore

        self.last_progress = 0

        self.name = name

        # Make result directory
        if not os.path.exists(self.results_path):
            os.makedirs(self.results_path)

        # Results directory according to date and time
        if name_without_date and name:
            dirname = name
        else:
            now = datetime.datetime.now()
            if name:
                dirname = now.strftime("%Y.%m.%d") + "_" + name
            else:
                dirname = now.strftime("%Y.%m.%d")

        self.results_path = os.path.join(self.results_path, dirname)

        # TODO: Ugly fix
        if os.path.isdir(self.results_path):
            self.results_path += "_2"

        # Remove existing directory of it exist
        if os.path.exists(self.results_path):
            shutil.rmtree(self.results_path)
        os.makedirs(self.results_path)

        # Save params.xml and measures.xml in results dir
        self.paramtree.save(os.path.join(self.results_path, "params.xml"))
        self.measuretree.save(os.path.join(self.results_path, "measures.xml"))

        self.raw_results_path = os.path.join(self.results_path, "raw")
        os.makedirs(self.raw_results_path)

        # Redirect log to run.log
        logfile = os.path.join(self.results_path, "run.log")
        handler = logging.FileHandler(logfile)
        logging.getLogger(__name__).addHandler(handler)

    def run(self):
        """
        """

        logger.info("Starting %i simulations on %i core" % \
            (self.nsimu, self.ncore))

        if in_ipython():
            parallel = False
        else:
            parallel = True

        if parallel and os.name == 'posix':
            os.system("taskset -p 0xff %d" % os.getpid())

        params = [self.paramtree, self.measuretree, self.verbose]
        all_params = []
        for i in range(self.nsimu):
            all_params.append([i, self.raw_results_path] + params)

        queue = Queue()
        self.start_time = time.time()
        p = Pool(self.ncore, run_init, [queue], maxtasksperchild=4)
        p.map_async(run_one, all_params, chunksize=1)

        # Monitoring simulations
        done = False
        simu_ids = range(self.nsimu)
        while not done:
            mess = queue.get()
            if mess["state"] == "stop":
                simu_ids.remove(mess["id"])
                self.log_progress(len(simu_ids))

            if not simu_ids:
                done = True
        self.total_time = time.time() - self.start_time

        p.close()

        results_size = get_folder_size(self.results_path)
        logger.info("Simulations are done")
        logger.info("Results are stored in %s (%s MB)" % (self.results_path, results_size))

        self.create_log()

        del p
        del queue
        gc.collect()

    def log_progress(self, simu_left, precision=2):
        """
        Log the progression of the simulations

        :param simu_left: Number of simulation remaining
        :type simu_left: int

        """
        progress = 100.0 - ((simu_left * 100.0) / self.nsimu)
        progress = round(progress, precision)
        # Don't print the same progression !
        if progress > self.last_progress:
            time_remaining, spent_time = self.estimate_remaining_time(simu_left)
            output = "Progression : %0.2f%% | " % progress
            output += "ETA : %s | " % time_remaining
            output += "Spent time : %s" % spent_time
            logger.info(output)
            self.last_progress = progress

    def estimate_remaining_time(self, simu_left):
        """
        Estimate remaining time
        """
        delta_time = time.time() - self.start_time
        delta_simu = self.nsimu - simu_left
        estimate_time = (simu_left * delta_time) / delta_simu
        estimate_time = time.strftime('%H:%M:%S', time.gmtime(estimate_time))
        spent_time = time.strftime('%H:%M:%S', time.gmtime(delta_time))
        return estimate_time, spent_time

    def create_log(self):
        """
        Create logfile in results folder
        """

        log = {}
        log["name"] = self.name
        log["duration"] = self.paramtree.absolute_dic['span']
        log["dt"] = self.paramtree.absolute_dic['dt']
        log["num_steps"] = int(log["duration"] / log["dt"])
        log["number_of_simulations"] = self.nsimu
        log["number_of_simulations"] = self.nsimu
        log["spent_time"] = time.strftime('%H:%M:%S', time.gmtime(self.total_time))
        log["results_folder_size_in_MB"] = get_folder_size(self.results_path)

        f = open(os.path.join(self.results_path, "simu.log"), 'w')
        f.write(json.dumps(log, sort_keys=True, indent=4))
        f.close()
Exemplo n.º 4
0
import argparse
import sys
sys.path.append('..')

from kt_simul.io.xml_handler import ParamTree
from kt_simul.core.simul_spindle import Metaphase
from kt_simul.io.simuio import SimuIO
from kt_simul.core import parameters

if __name__ == '__main__':

    PARAMFILE = parameters.PARAMFILE
    MEASUREFILE = parameters.MEASUREFILE

    # Change some parameters
    paramtree = ParamTree(PARAMFILE)
    paramtree.change_dic('dt', 10)
    paramtree.change_dic('span', 2000)
    paramtree.change_dic('t_A', 1750)

    measuretree = ParamTree(MEASUREFILE, adimentionalized=False)

    # Arguments parser
    parser = argparse.ArgumentParser(description='KtSimu Launcher')
    parser.add_argument('--new', '-n', action='store_true', default=False,
                         help="Run a new simulation")
    parser.add_argument('--results', '-r', type=str, default="simu.h5",
                         help="Specified results hdf5 file for new or already runned simulation")
    args = parser.parse_args()

    results_file = args.results