예제 #1
0
    def write_out_plots_current_step(self, same_steps_only=True):
        """
        This method will write out all plots available to the path
        configured in self.lab_run_directory.

        Parameters
        ---------
        same_steps_only : boolean, optional
            Write only if all experiment assistants in this lab assistant
            are currently in the same step.
        """
        step_string, same_step = self._compute_current_step_overall()

        if same_steps_only and not same_step:
            return

        plot_base = os.path.join(self.lab_run_directory, "plots")
        plot_step_base = os.path.join(plot_base, step_string)
        ensure_directory_exists(plot_step_base)

        plots_to_write = self.generate_all_plots()

        #finally write out all plots created above to their files
        for plot_name in plots_to_write.keys():
            plot_fig = plots_to_write[plot_name]

            write_plot_to_file(plot_fig, plot_name + "_" + step_string, plot_step_base)
    def _create_experiment_directory(self):
        global_start_date = time.time()

        date_name = datetime.datetime.utcfromtimestamp(global_start_date).strftime("%Y-%m-%d_%H:%M:%S")

        self.experiment_directory_base = os.path.join(self.write_directory_base, self.experiment.name + "_" + date_name)

        ensure_directory_exists(self.experiment_directory_base)
def get_logger(module, specific_log_name=None):
    """
    Abstraction from logging.getLogging, which also adds initialization.

    Logging is configured directly at root level (in the standard usecase, at
    least). You also have the opportunity to specify a certain directory to
    which details of only this logger (and all subloggers) are written.

    Currently, nothing is configurable from the outside. This is planned to be
    changed.

    Parameters
    ----------
    module : object or string
        The object for which we'd like to get the logger. The name of the
        logger is then, analogous to logging, set to
        module.__module__ + "." + module.__class__.__name__

        If the object is a string it will be taken as name directly.
    specific_log_name : string, optional
        If you want logging for this logger (and all sublogger) to a specific
        file, this allows you to set the corresponding filename.

    Returns
    -------
    logger: logging.logger
        A logging for module.
    """

    #if logger is already given as a string take directly. otherwise compute.
    if isinstance(module, basestring):
        new_logger_name = module
    else:
        new_logger_name = module.__module__ + "." + module.__class__.__name__

    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    LOG_ROOT = os.environ.get('APSIS_LOG_ROOT', '/tmp/APSIS_WRITING/logs')
    ensure_directory_exists(LOG_ROOT)
    global logging_intitialized
    if not logging_intitialized:
        logging_intitialized = True
        #initialize the root logger.
        project_dirname = os.path.dirname(apsis.__file__)
        log_config_file = os.path.join(project_dirname, 'config/logging.conf')
        logging.config.fileConfig(
            log_config_file,
            defaults={'logfilename': os.path.join(LOG_ROOT, "log")})

    logger_existed = False
    if new_logger_name in logging.Logger.manager.loggerDict:
        logger_existed = True
    logger = logging.getLogger(new_logger_name)
    if specific_log_name is not None and not logger_existed:
        fh = logging.FileHandler(os.path.join(LOG_ROOT, specific_log_name))
        fh.setFormatter(formatter)
        logger.addHandler(fh)
    return logger
예제 #4
0
    def __init__(self, name, optimizer, param_defs, experiment=None, optimizer_arguments=None,
                 minimization=True, write_directory_base="/tmp/APSIS_WRITING",
                 experiment_directory_base=None, csv_write_frequency=1):
        """
        Initializes the BasicExperimentAssistant.

        Parameters
        ----------
        name : string
            The name of the experiment. This does not have to be unique, but is
            for human orientation.
        optimizer : Optimizer instance or string
            This is an optimizer implementing the corresponding functions: It
            gets an experiment instance, and returns one or multiple candidates
            which should be evaluated next.
            Alternatively, it can be a string corresponding to the optimizer,
            as defined by apsis.utilities.optimizer_utils.
        param_defs : dict of ParamDef.
            This is the parameter space defining the experiment.
        experiment : Experiment
            Preinitialize this assistant with an existing experiment.
        optimizer_arguments=None : dict
            These are arguments for the optimizer. Refer to their documentation
            as to which are available.
        minimization=True : bool
            Whether the problem is one of minimization or maximization.
        write_directory_base : string, optional
            The global base directory for all writing. Will only be used
            for creation of experiment_directory_base if this is not given.
        experiment_directory_base : string or None, optional
            The directory to write all the results to. If not
            given a directory with timestamp will automatically be created
            in write_directory_base
        csv_write_frequency : int, optional
            States how often the csv file should be written to.
            If set to 0 no results will be written.
        """
        self.logger = get_logger(self)
        self.logger.info("Initializing experiment assistant.")
        self.optimizer = optimizer
        self.optimizer_arguments = optimizer_arguments

        if experiment is None:
            self.experiment = Experiment(name, param_defs, minimization)
        else:
            self.experiment = experiment

        self.csv_write_frequency = csv_write_frequency

        if self.csv_write_frequency != 0:
            self.write_directory_base = write_directory_base
            if experiment_directory_base is not None:
                self.experiment_directory_base = experiment_directory_base
                ensure_directory_exists(self.experiment_directory_base)
            else:
                self._create_experiment_directory()
        self.logger.info("Experiment assistant successfully initialized.")
예제 #5
0
    def _create_experiment_directory(self):
        global_start_date = time.time()

        date_name = datetime.datetime.utcfromtimestamp(
                global_start_date).strftime("%Y-%m-%d_%H:%M:%S")

        self.experiment_directory_base = os.path.join(self.write_directory_base,
                                    self.experiment.name + "_" + date_name)

        ensure_directory_exists(self.experiment_directory_base)
예제 #6
0
def get_logger(module, specific_log_name=None):
    """
    Abstraction from logging.getLogging, which also adds initialization.

    Logging is configured directly at root level (in the standard usecase, at
    least). You also have the opportunity to specify a certain directory to
    which details of only this logger (and all subloggers) are written.

    Currently, nothing is configurable from the outside. This is planned to be
    changed.

    Parameters
    ----------
    module : object or string
        The object for which we'd like to get the logger. The name of the
        logger is then, analogous to logging, set to
        module.__module__ + "." + module.__class__.__name__

        If the object is a string it will be taken as name directly.
    specific_log_name : string, optional
        If you want logging for this logger (and all sublogger) to a specific
        file, this allows you to set the corresponding filename.

    Returns
    -------
    logger: logging.logger
        A logging for module.
    """

    #if logger is already given as a string take directly. otherwise compute.
    if isinstance(module, basestring):
        new_logger_name = module
    else:
        new_logger_name = module.__module__ + "." + module.__class__.__name__

    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    LOG_ROOT = os.environ.get('APSIS_LOG_ROOT', '/tmp/APSIS_WRITING/logs')
    ensure_directory_exists(LOG_ROOT)
    global logging_intitialized
    if not logging_intitialized:
        logging_intitialized = True
        #initialize the root logger.
        project_dirname = os.path.dirname(apsis.__file__)
        log_config_file = os.path.join(project_dirname, 'config/logging.conf')
        logging.config.fileConfig(log_config_file, defaults={'logfilename': os.path.join(LOG_ROOT, "log")})

    logger_existed = False
    if new_logger_name in logging.Logger.manager.loggerDict:
        logger_existed = True
    logger = logging.getLogger(new_logger_name)
    if specific_log_name is not None and not logger_existed:
        fh = logging.FileHandler(os.path.join(LOG_ROOT, specific_log_name))
        fh.setFormatter(formatter)
        logger.addHandler(fh)
    return logger
예제 #7
0
 def _create_experiment_directory(self):
     """
     Generates an experiment directory from the base write directory.
     """
     global_start_date = time.time()
     date_name = datetime.datetime.utcfromtimestamp(
             global_start_date).strftime("%Y-%m-%d_%H:%M:%S")
     self._experiment_directory_base = os.path.join(
                                 self._write_directory_base,
                                 self._experiment.exp_id)
     ensure_directory_exists(self._experiment_directory_base)
예제 #8
0
    def _init_directory_structure(self):
        """
        Method to create the directory structure if not exists
        for results and plots writing
        """
        if self.lab_run_directory is None:
            date_name = datetime.datetime.utcfromtimestamp(
                self.global_start_date).strftime("%Y-%m-%d_%H:%M:%S")

            self.lab_run_directory = os.path.join(self.write_directory_base,
                                                  date_name)

            ensure_directory_exists(self.lab_run_directory)
예제 #9
0
    def write_plots(self):
        """
        Writes out the plots of this assistant.
        """
        fig = self.plot_result_per_step()
        filename = "result_per_step_%i" \
                   % len(self._experiment.candidates_finished)

        path = self._experiment_directory_base + "/plots"
        ensure_directory_exists(path)
        write_plot_to_file(fig, filename, path)
        write_plot_to_file(fig, "cur_state", self._experiment_directory_base)
        plt.close(fig)
예제 #10
0
    def __init__(self, optimizer_class, optimizer_arguments=None,
                 write_directory_base=None, experiment_directory=None,
                 csv_write_frequency=1):
        """
        Initializes this experiment assistant.

        Note that calling this function does not yet create an experiment, for
        that, use init_experiment. If there is an already existing experiment,
        you can just set self._experiment.

        Parameters
        ----------
        optimizer_class : subclass of Optimizer
            The class of the optimizer, used to initialize it.
        optimizer_arguments : dict, optional
            The dictionary of optimizer arguments. If None, default values will
            be used.
        experiment_directory_base : string, optional
            The folder to which the csv intermediary results and the plots will
            be written. Default is <write_directory_base>/exp_id.
        write_directory_base : string, optional
            The base directory. In the default case, this is dependant on the
            OS. On windows, it is set to ./APSIS_WRITING/. On Linux,
            it is set to /tmp/APSIS_WRITING/. If an
            experiment_directory has been given, this will be ignored.
        csv_write_frequency : int, optional
            This sets the frequency with which the csv file is written. If set
            to 1 (the default), it writes every step. If set to 2, every second
            and so on. Note that it still writes out every step eventually.
        """
        self._logger = get_logger(self)
        self._logger.info("Initializing experiment assistant.")
        self._csv_write_frequency = csv_write_frequency
        self._optimizer = optimizer_class
        self._optimizer_arguments = optimizer_arguments
        if self._csv_write_frequency != 0:
            if experiment_directory is not None:
                self._experiment_directory_base = experiment_directory
                ensure_directory_exists(self._experiment_directory_base)
            else:
                if write_directory_base is None:
                    if os.name == "nt":
                        self._write_directory_base = \
                            os.path.relpath("APSIS_WRITING")
                    else:
                        self._write_directory_base = "/tmp/APSIS_WRITING"
                else:
                    self._write_directory_base = write_directory_base
        self._logger.info("Experiment assistant for successfully "
                         "initialized.")
예제 #11
0
    def _load_exp_assistant_from_path(self, path):
        """
        This loads a complete exp_assistant from path.

        Specifically, it looks for exp_assistant.json in the path and restores
        optimizer_class, optimizer_arguments and write_dir from this. It then
        loads the experiment from the write_dir/experiment.json, then
        initializes both.

        Parameters
        ----------
        path : string
            The path from which to initialize. This must contain an
            exp_assistant.json as specified.
        """
        self._logger.debug("Loading Exp_assistant from path %s" % path)
        with open(path + "/exp_assistant.json", 'r') as infile:
            exp_assistant_json = json.load(infile)

        optimizer_class = exp_assistant_json["optimizer_class"]
        optimizer_arguments = exp_assistant_json["optimizer_arguments"]
        exp_ass_write_dir = exp_assistant_json["write_dir"]
        ensure_directory_exists(exp_ass_write_dir)
        self._logger.debug(
            "\tLoaded exp_parameters: "
            "optimizer_class: %s, optimizer_arguments: %s,"
            "write_dir: %s" %
            (optimizer_class, optimizer_arguments, exp_ass_write_dir))
        exp = self._load_experiment(path)
        self._logger.debug("\tLoaded Experiment. %s" % exp.to_dict())

        exp_ass = ExperimentAssistant(optimizer_class=optimizer_class,
                                      experiment=exp,
                                      optimizer_arguments=optimizer_arguments,
                                      write_dir=exp_ass_write_dir)

        if exp_ass.exp_id in self._exp_assistants:
            raise ValueError("Loaded exp_id is duplicated in experiment! id "
                             "is %s" % exp_ass.exp_id)
        self._exp_assistants[exp_ass.exp_id] = exp_ass
        self._logger.info("Successfully loaded experiment from %s." % path)
예제 #12
0
    def write_out_plots_current_step(self, exp_ass=None, same_steps_only=True):
        """
        This method will write out all plots available to the path
        configured in self.lab_run_directory.

        Parameters
        ---------
        exp_ass : list, optional
            List of experiment assistant names to include in the plots. Defaults to
            None, which is equivalent to all.
        same_steps_only : boolean, optional
            Write only if all experiment assistants in this lab assistant
            are currently in the same step.
        """
        min_step = self._get_min_step()
        if same_steps_only:
            plot_up_to = min_step
        else:
            plot_up_to = None

        plot_base = os.path.join(self._lab_run_directory, "plots")
        plot_step_base = os.path.join(plot_base, "step_" + str(min_step))
        ensure_directory_exists(plot_step_base)

        if exp_ass is None:
            exp_ass = self._exp_assistants.keys()

        plots_to_write = self.generate_all_plots(exp_ass, plot_up_to)


        #finally write out all plots created above to their files
        for plot_name in plots_to_write.keys():
            plot_fig = plots_to_write[plot_name]

            write_plot_to_file(plot_fig, plot_name + "_step" + str(min_step),
                               plot_step_base)
            plt.close(plot_fig)
예제 #13
0
    def init_experiment(self,
                        name,
                        optimizer,
                        param_defs,
                        exp_id=None,
                        notes=None,
                        optimizer_arguments=None,
                        minimization=True):
        """
        Initializes an experiment.

        Parameters
        ----------
        name : string
            name of the experiment.
        optimizer : string
            String representation of the optimizer.
        param_defs : dict of parameter definitions
            Dictionary of parameter definition classes.
        optimizer_arguments : dict, optional
            A dictionary defining the operation of the optimizer. See the
            respective documentation of the optimizers.
            Default is None, which are default values.
        exp_id : string or None, optional
            The id of the experiment, which will be used to reference it.
            Should be a proper uuid, and especially has to be unique. If it is
            not, an error may be returned.
        notes : jsonable object or None, optional
            Any note that you'd like to put in the experiment. Could be used
            to provide some details on the experiment, on the start time or the
            user starting it.
        minimization : bool, optional
            Whether the problem is one of minimization. Defaults to True.

        Returns
        -------
        exp_id : string
            String representing the id of the experiment or "failed" if failed.

        Raises
        ------
        ValueError :
            Iff there already is an experiment with the exp_id for this lab
            assistant. Does not occur if no exp_id is given.
        """
        self._logger.debug("Initializing new experiment. Parameters: "
                           "name: %s, optimizer: %s, param_defs: %s, "
                           "exp_id: %s, notes: %s, optimizer_arguments: %s, "
                           "minimization: %s" %
                           (name, optimizer, param_defs, exp_id, notes,
                            optimizer_arguments, minimization))
        if exp_id in self._exp_assistants.keys():
            raise ValueError("Already an experiment with id %s registered." %
                             exp_id)

        if exp_id is None:
            while True:
                exp_id = uuid.uuid4().hex
                if exp_id not in self._exp_assistants.keys():
                    break
            self._logger.debug("\tGenerated new exp_id: %s" % exp_id)

        if not self._write_dir:
            exp_assistant_write_directory = None
        else:
            exp_assistant_write_directory = os.path.join(self._write_dir +
                                                         "/" + exp_id)
            ensure_directory_exists(exp_assistant_write_directory)
        self._logger.debug("\tExp_ass directory: %s" %
                           exp_assistant_write_directory)

        exp = experiment.Experiment(name, param_defs, exp_id, notes,
                                    minimization)

        exp_ass = ExperimentAssistant(optimizer,
                                      experiment=exp,
                                      optimizer_arguments=optimizer_arguments,
                                      write_dir=exp_assistant_write_directory)
        self._exp_assistants[exp_id] = exp_ass
        self._logger.info("Experiment initialized successfully with id %s." %
                          exp_id)
        self._write_state_to_file()
        return exp_id
예제 #14
0
def get_logger(module, extra_info=None, save_path=None):
    """
    Abstraction from logging.getLogging, which also adds initialization.

    This loads the logging config from config/logging.conf.

    Parameters
    ----------
    module : object or string
        The object for which we'd like to get the logger. The name of the
        logger is then, analogous to logging, set to
        module.__module__ + "." + module.__class__.__name__
        If the object is a string it will be taken as name directly.

    extra_info : string, optional
        If None (the default), a usual logger is returned. If not, a
        logger_adapter is returned, which always prepends the corresponding
        string.

    save_path : string, optional
        The path on which to store the logging. If logging has been initialized
        previously, this is ignored (and a warning is logged). If a path has
        been specified in the config file, this is also ignored (and a warning
        is issued). Otherwise, this path replaces all instances of the token
        <SAVE_PATH> in the file_name of all handlers.
        If it does not end with "/" we'll automatically add it. That means both
        "/tmp/APSIS_WRITING" and "/tmp/APSIS_WRITING/" is treated identically,
        and logging is added in "/tmp/APSIS_WRITING/logs".

    Returns
    -------
    logger: logging.logger
        A logging for module.
    """
    #if logger is already given as a string take directly. otherwise compute.
    if isinstance(module, basestring):
        new_logger_name = module
    else:
        new_logger_name = module.__module__ + "." + module.__class__.__name__

    global testing

    global logging_intitialized
    if not logging_intitialized and not testing:
        logging_intitialized = True

        # Look for the logging config file.
        project_dirname = os.path.dirname(apsis.__file__)
        log_config_file = os.path.join(project_dirname, 'config/logging.conf')
        with open(log_config_file, "r") as conf_file:
            conf_dict = yaml.load(conf_file)
        handlers = conf_dict["handlers"]
        handler_keys = handlers.keys()
        for h in handler_keys:
            if "filename" in handlers[h]:
                if "<SAVE_PATH>" in handlers[h]["filename"]:
                    if not save_path.endswith("/"):
                        save_path += "/"
                    handlers[h]["filename"] = handlers[h]["filename"].replace(
                        "<SAVE_PATH>/", save_path).replace("<SAVE_PATH>",
                                                           save_path)
                ensure_directory_exists(os.path.dirname(handlers[h]["filename"]))

        logging.config.dictConfig(conf_dict)

    logger = logging.getLogger(new_logger_name)

    if extra_info:
        logger = AddInfoClass(logger, {"extra_info": extra_info})

    return logger
    def __init__(
        self,
        name,
        optimizer,
        param_defs,
        experiment=None,
        optimizer_arguments=None,
        minimization=True,
        write_directory_base="/tmp/APSIS_WRITING",
        experiment_directory_base=None,
        csv_write_frequency=1,
    ):
        """
        Initializes the BasicExperimentAssistant.

        Parameters
        ----------
        name : string
            The name of the experiment. This does not have to be unique, but is
            for human orientation.
        optimizer : Optimizer instance or string
            This is an optimizer implementing the corresponding functions: It
            gets an experiment instance, and returns one or multiple candidates
            which should be evaluated next.
            Alternatively, it can be a string corresponding to the optimizer,
            as defined by apsis.utilities.optimizer_utils.
        param_defs : dict of ParamDef.
            This is the parameter space defining the experiment.
        experiment : Experiment
            Preinitialize this assistant with an existing experiment.
        optimizer_arguments=None : dict
            These are arguments for the optimizer. Refer to their documentation
            as to which are available.
        minimization=True : bool
            Whether the problem is one of minimization or maximization.
        write_directory_base : string, optional
            The global base directory for all writing. Will only be used
            for creation of experiment_directory_base if this is not given.
        experiment_directory_base : string or None, optional
            The directory to write all the results to. If not
            given a directory with timestamp will automatically be created
            in write_directory_base
        csv_write_frequency : int, optional
            States how often the csv file should be written to.
            If set to 0 no results will be written.
        """
        self.logger = get_logger(self)
        self.logger.info("Initializing experiment assistant.")
        self.optimizer = optimizer
        self.optimizer_arguments = optimizer_arguments

        if experiment is None:
            self.experiment = Experiment(name, param_defs, minimization)
        else:
            self.experiment = experiment

        self.csv_write_frequency = csv_write_frequency

        if self.csv_write_frequency != 0:
            self.write_directory_base = write_directory_base
            if experiment_directory_base is not None:
                self.experiment_directory_base = experiment_directory_base
                ensure_directory_exists(self.experiment_directory_base)
            else:
                self._create_experiment_directory()
        self.logger.info("Experiment assistant successfully initialized.")