Example #1
0
def experiment(state, channel):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.

    Parameters
    ----------
    state: WRITEME
    channel: WRITEME
    """

    yaml_template = open(yaml_file).read()
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    file_params = expand(flatten(state.file_parameters), dict_type=ydict)
    # Hack to fill in file parameter strings first
    for param in file_params:
        yaml_template = yaml_template.replace("%%(%s)s" % param,
                                              file_params[param])

    yaml = yaml_template % hyper_parameters
    #    with open("/na/homes/dhjelm/pylearn2/pylearn2/jobman/nice_lr_search/%d.yaml"
    #              % state.id, "w") as f:
    #        f.write(yaml)
    train_object = yaml_parse.load(yaml)

    train_object.main_loop()
    state.results = extract_results(train_object.model)
    return channel.COMPLETE
def experiment(state, channel):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.

    Parameters
    ----------
    state: WRITEME
    channel: WRITEME
    """

    yaml_template = open(yaml_file).read()
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    file_params = expand(flatten(state.file_parameters), dict_type=ydict)
    # Hack to fill in file parameter strings first
    for param in file_params:
        yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param])

    yaml = yaml_template % hyper_parameters
    train_object = yaml_parse.load(yaml)

    state.pid = os.getpid()
    channel.save()
    train_object.main_loop()

    state.results = extract_results(train_object.model)
    return channel.COMPLETE
Example #3
0
def main(args):
    dataset_name = args.dataset_name

    logger.info("Getting dataset info for %s" % dataset_name)
    data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name)
    mask_file = path.join(data_path, "mask.npy")
    mask = np.load(mask_file)
    input_dim = (mask == 1).sum()

    user = path.expandvars("$USER")
    save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s"
                                  % (user, "rbm_simple_test"))

    # File parameters are path specific ones (not model specific).
    file_params = {"save_path": save_path,
                   }

    yaml_template = open(yaml_file).read()
    hyperparams = expand(flatten(experiment.default_hyperparams(input_dim=input_dim)),
                         dict_type=ydict)

    # Set additional hyperparams from command line args
    if args.learning_rate is not None:
        hyperparams["learning_rate"] = args.learning_rate
    if args.batch_size is not None:
        hyperparams["batch_size"] = args.batch_size

    for param in file_params:
        yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param])

    yaml = yaml_template % hyperparams

    logger.info("Training")
    train = yaml_parse.load(yaml)
    train.main_loop()
Example #4
0
def run_experiment_jobman(state, channel):
    """
    Main jobman experiment function, called by all jobs.
    """
    experiment_module = state["experiment&module"]
    experiment = imp.load_source("module.name", experiment_module)

    yaml_template = open(experiment.yaml_file).read()
    hyperparams = expand(flatten(translate(state.hyperparams, "pylearn2")),
                         dict_type=ydict)

    if not state["out&path"].endswith("job_%d" % state["jobman.id"]):
        state["out&path"] = path.join(state["out&path"],
                                      "job_%d" % state["jobman.id"])
    channel.save()
    out_path = path.join(state["out&path"])
    try:
        run_experiment(experiment,
                       hyperparams,
                       ask=False,
                       out_path=out_path,
                       dbdescr=state["dbdescr"],
                       job_id=state["jobman.id"])
    except ValueError as e:
        if str(e) == "KILLED":
            return channel.CANCELED
        else:
            return channel.ERR_RUN
    except:
        return channel.ERR_RUN

    print "Ending experiment"
    return channel.COMPLETE
Example #5
0
    def _eager_dicts(self, lazy_sql_rows, safe_session):
        eager_dicts = [None] * len(lazy_sql_rows)
        with safe_session.set_timer(60 * 5):
            for i, lazy_sql_row in enumerate(lazy_sql_rows):
                eager_dicts[i] = expand(dict(lazy_sql_row.iteritems()))

        return eager_dicts
Example #6
0
def run_experiment_jobman(state, channel):
    """
    Main jobman experiment function, called by all jobs.
    """
    experiment_module = state["experiment&module"]
    experiment = imp.load_source("module.name", experiment_module)

    yaml_template = open(experiment.yaml_file).read()
    hyperparams = expand(flatten(translate(state.hyperparams, "pylearn2")),
                         dict_type=ydict)

    if not state["out&path"].endswith("job_%d" % state["jobman.id"]):
        state["out&path"] = path.join(state["out&path"],
                                      "job_%d" % state["jobman.id"])
    channel.save()
    out_path = path.join(state["out&path"])
    try:
        run_experiment(experiment, hyperparams, ask=False, out_path=out_path,
                       dbdescr=state["dbdescr"], job_id=state["jobman.id"])
    except ValueError as e:
        if str(e) == "KILLED":
            return channel.CANCELED
        else:
            return channel.ERR_RUN
    except:
        return channel.ERR_RUN

    print "Ending experiment"
    return channel.COMPLETE
Example #7
0
    def _eager_dicts(self, lazy_sql_rows, safe_session):
        eager_dicts = [None] * len(lazy_sql_rows)
        with safe_session.set_timer(60 * 5):
            for i, lazy_sql_row in enumerate(lazy_sql_rows):
                eager_dicts[i] = expand(dict(lazy_sql_row.iteritems()))

        return eager_dicts
Example #8
0
def set_jobs(name, status, new_status):
    experiments = experiment_scheduler.load_experiments(
        cluster=None, filter_eq_dct=dict(name=name))

    if len(experiments) == 0:
        print "No experiments in database %s" % get_db_string("experiments")
        return

    experiment = experiments[0]

    if status == "pending":
        jobs = job_scheduler.load_pending_jobs(experiment["table"])
    elif status == "broken":
        jobs = job_scheduler.load_broken_jobs(experiment["table"])
    elif status == "completed":
        jobs = job_scheduler.load_completed_jobs(experiment["table"])
    elif status == "running":
        jobs = job_scheduler.load_running_jobs(experiment["table"])

    if new_status == "pending":
        sql_new_status = sql.START
        new_status = "pending"
    elif new_status == "running":
        sql_new_status = sql.RUNNING
    elif new_status == "completed":
        sql_new_status = sql.COMPLETE

    print "Setting %s jobs to %s status..." % (status, new_status)
    job_scheduler.update_jobs(experiment["table"], jobs,
                              expand({sql.STATUS: sql_new_status,
                                      'proc_status': new_status}))
Example #9
0
def train_experiment(state, channel):
    """
    Train a model specified in state, and extract required results.

    This function builds a YAML string from ``state.yaml_template``, taking
    the values of hyper-parameters from ``state.hyper_parameters``, creates
    the corresponding object and trains it (like train.py), then run the
    function in ``state.extract_results`` on it, and store the returned values
    into ``state.results``.

    To know how to use this function, you can check the example in tester.py
    (in the same directory).
    """
    yaml_template = state.yaml_template

    # Convert nested DD into nested ydict.
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters

    # Instantiate an object from YAML string
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)

    for ext in train_obj.extensions:
        if hasattr(ext, 'set_train_obj'):
            ext.set_train_obj(train_obj)
        if hasattr(ext, 'set_jobman_channel'):
            ext.set_jobman_channel(channel)
        if hasattr(ext, 'set_jobman_state'):
            ext.set_jobman_state(state)

    try:
        iter(train_obj)
        iterable = True
    except TypeError:
        iterable = False
    if iterable:
        raise NotImplementedError(
            ('Current implementation does not support running multiple '
             'models in one yaml string.  Please change the yaml template '
             'and parameters to contain only one single model.'))
    else:
        # print "Executing the model."
        train_obj.main_loop()
        # This line will call a function defined by the user and pass train_obj
        # to it.
        state.results = jobman.tools.resolve(state.extract_results)(train_obj)
        return channel.COMPLETE
Example #10
0
def train_experiment(state, channel):
    """
    Train a model specified in state, and extract required results.

    This function builds a YAML string from ``state.yaml_template``, taking
    the values of hyper-parameters from ``state.hyper_parameters``, creates
    the corresponding object and trains it (like train.py), then run the
    function in ``state.extract_results`` on it, and store the returned values
    into ``state.results``.

    To know how to use this function, you can check the example in tester.py
    (in the same directory).
    """
    yaml_template = state.yaml_template

    # Convert nested DD into nested ydict.
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters

    # Instantiate an object from YAML string
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)

    for ext in train_obj.extensions:
      if hasattr(ext, 'set_train_obj'):
        ext.set_train_obj( train_obj )
      if hasattr(ext, 'set_jobman_channel'):
        ext.set_jobman_channel( channel )
      if hasattr(ext, 'set_jobman_state'):
        ext.set_jobman_state( state )

    try:
        iter(train_obj)
        iterable = True
    except TypeError:
        iterable = False
    if iterable:
        raise NotImplementedError(
                ('Current implementation does not support running multiple '
                 'models in one yaml string.  Please change the yaml template '
                 'and parameters to contain only one single model.'))
    else:
        # print "Executing the model."
        train_obj.main_loop()
        # This line will call a function defined by the user and pass train_obj
        # to it.
        state.results = jobman.tools.resolve(state.extract_results)(train_obj)
        return channel.COMPLETE
Example #11
0
def run_experiment(experiment, **kwargs):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.
    TODO: add sigint handling.

    Parameters
    ----------
    experiment: module
        Experiment module.
    kwargs: dict
        Typically hyperparameters.
    """

    hyper_parameters = experiment.default_hyperparams()
    set_hyper_parameters(hyper_parameters, **kwargs)
    file_parameters = experiment.fileparams
    set_hyper_parameters(file_parameters, **kwargs)
    hyper_parameters.update(file_parameters)

    ih = MRIInputHandler()
    input_dim, variance_map_file = ih.get_input_params(hyper_parameters)
    hyper_parameters["nvis"] = input_dim
    hyper_parameters["variance_map_file"] = variance_map_file

    pid = os.getpid()
    out_path = serial.preprocess(
        hyper_parameters.get("out_path", "${PYLEARN2_OUTS}"))
    if not path.isdir(out_path):
        os.mkdir(out_path)
    if not path.isdir(path.join(out_path, "logs")):
        os.mkdir(path.join(out_path, "logs"))

    hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict)

    lh = LogHandler(experiment, hyper_parameters, out_path, pid)
    h = logging.StreamHandler(lh)
    monitor.log.addHandler(h)

    yaml_template = open(experiment.yaml_file).read()
    yaml = yaml_template % hyper_parameters
    train_object = yaml_parse.load(yaml)
    try:
        train_object.main_loop()
        lh.finish("COMPLETED")
    except KeyboardInterrupt:
        print("Quitting...")
        lh.finish("KILLED")
Example #12
0
def run_experiment(experiment, **kwargs):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.
    TODO: add sigint handling.

    Parameters
    ----------
    experiment: module
        Experiment module.
    kwargs: dict
        Typically hyperparameters.
    """

    hyper_parameters = experiment.default_hyperparams()
    set_hyper_parameters(hyper_parameters, **kwargs)
    file_parameters = experiment.fileparams
    set_hyper_parameters(file_parameters, **kwargs)
    hyper_parameters.update(file_parameters)

    ih = MRIInputHandler()
    input_dim, variance_map_file = ih.get_input_params(hyper_parameters)
    hyper_parameters["nvis"] = input_dim
    hyper_parameters["variance_map_file"] = variance_map_file

    pid = os.getpid()
    out_path = serial.preprocess(
        hyper_parameters.get("out_path", "${PYLEARN2_OUTS}"))
    if not path.isdir(out_path):
        os.mkdir(out_path)
    if not path.isdir(path.join(out_path, "logs")):
        os.mkdir(path.join(out_path, "logs"))

    hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict)

    lh = LogHandler(experiment, hyper_parameters, out_path, pid)
    h = logging.StreamHandler(lh)
    monitor.log.addHandler(h)

    yaml_template = open(experiment.yaml_file).read()
    yaml = yaml_template % hyper_parameters
    train_object = yaml_parse.load(yaml)
    try:
        train_object.main_loop()
        lh.finish("COMPLETED")
    except KeyboardInterrupt:
        print("Quitting...")
        lh.finish("KILLED")
Example #13
0
def train_experiment(state, channel):
    """
    Train a model specified in state, and extract required results.

    This function builds a YAML string from ``state.yaml_template``, taking
    the values of hyper-parameters from ``state.hyper_parameters``, creates
    the corresponding object and trains it (like train.py), then run the
    function in ``state.extract_results`` on it, and store the returned values
    into ``state.results``.

    To know how to use this function, you can check the example in tester.py
    (in the same directory).
    """
    yaml_template = state.yaml_template

    # Convert nested DD into nested ydict.
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters
    
    # write to .yaml file for ease of reproducibility
    fp = open('experiment.yaml', 'w')
    fp.write('%s' % final_yaml_str[2:]) 
    fp.close()

    # Instantiate an object from YAML string
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)

    try:
        iter(train_obj)
        iterable = True
    except TypeError:
        iterable = False
    if iterable:
        raise NotImplementedError(
                ('Current implementation does not support running multiple '
                 'models in one yaml string.  Please change the yaml template '
                 'and parameters to contain only one single model.'))
    else:
        # print "Executing the model."
        # (GD) HACK HACK
        train_obj.model.jobman_channel = channel
        train_obj.model.jobman_state = state
        train_obj.main_loop()
        return channel.COMPLETE
Example #14
0
def train_experiment(state, channel):
    """
    Train a model specified in state, and extract required results.

    This function builds a YAML string from ``state.yaml_template``, taking
    the values of hyper-parameters from ``state.hyper_parameters``, creates
    the corresponding object and trains it (like train.py), then run the
    function in ``state.extract_results`` on it, and store the returned values
    into ``state.results``.

    To know how to use this function, you can check the example in tester.py
    (in the same directory).
    """
    yaml_template = state.yaml_template

    # Convert nested DD into nested ydict.
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)

    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters

    # write to .yaml file for ease of reproducibility
    fp = open('experiment.yaml', 'w')
    fp.write('%s' % final_yaml_str[2:])
    fp.close()

    # Instantiate an object from YAML string
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)

    try:
        iter(train_obj)
        iterable = True
    except TypeError:
        iterable = False
    if iterable:
        raise NotImplementedError(
            ('Current implementation does not support running multiple '
             'models in one yaml string.  Please change the yaml template '
             'and parameters to contain only one single model.'))
    else:
        # print "Executing the model."
        # (GD) HACK HACK
        train_obj.model.jobman_channel = channel
        train_obj.model.jobman_state = state
        train_obj.main_loop()
        return channel.COMPLETE
Example #15
0
def main(dataset_name="smri"):
    logger.info("Getting dataset info for %s" % args.dataset_name)
    data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name)
    mask_file = path.join(data_path, "mask.npy")
    mask = np.load(mask_file)
    input_dim = (mask == 1).sum()
    if input_dim % 2 == 1:
        input_dim -= 1
    mri = MRI.MRI_Standard(which_set="full",
                           dataset_name=args.dataset_name,
                           unit_normalize=True,
                           even_input=True,
                           apply_mask=True)
    variance_map_file = path.join(data_path, "variance_map.npy")
    mri_nifti.save_variance_map(mri, variance_map_file)

    user = path.expandvars("$USER")
    save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s"
                                  % (user, "jobman_test"))

    file_params = {"save_path": save_path,
                   "variance_map_file": variance_map_file
                   }

    yaml_template = open(yaml_file).read()
    hyperparams = expand(flatten(mlp_experiment.default_hyperparams(input_dim=input_dim)),
                         dict_type=ydict)

    for param in hyperparams:
        if hasattr(args, param) and getattr(args, param):
            val = getattr(args, param)
            logger.info("Filling %s with %r" % (param, val))
            hyperparams[param] = type(hyperparams[param])(val)

    for param in file_params:
        yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param])

    yaml = yaml_template % hyperparams
    print yaml
    logger.info("Training")
    train = yaml_parse.load(yaml)
    train.main_loop()
Example #16
0
def main(args):
    dataset_name = args.dataset_name

    logger.info("Getting dataset info for %s" % dataset_name)
    data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name)
    mask_file = path.join(data_path, "mask.npy")
    mask = np.load(mask_file)
    input_dim = (mask == 1).sum()

    user = path.expandvars("$USER")
    save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" %
                                  (user, "rbm_simple_test"))

    # File parameters are path specific ones (not model specific).
    file_params = {
        "save_path": save_path,
    }

    yaml_template = open(yaml_file).read()
    hyperparams = expand(flatten(
        experiment.default_hyperparams(input_dim=input_dim)),
                         dict_type=ydict)

    # Set additional hyperparams from command line args
    if args.learning_rate is not None:
        hyperparams["learning_rate"] = args.learning_rate
    if args.batch_size is not None:
        hyperparams["batch_size"] = args.batch_size

    for param in file_params:
        yaml_template = yaml_template.replace("%%(%s)s" % param,
                                              file_params[param])

    yaml = yaml_template % hyperparams

    logger.info("Training")
    train = yaml_parse.load(yaml)
    train.main_loop()
for all layer in the same graph and reconstruction error. """
  
import cPickle
import matplotlib
matplotlib.use('Agg')
import pylab
import numpy

from jobman.tools import DD,expand
from jobman.parse import filemerge

rec = {}
err = {}
epochvect = {}

state = expand(DD(filemerge('orig.conf')))
val_run_per_tr_siz = state.validation_runs_for_each_trainingsize
val_run = [int(trainsize) for trainsize in val_run_per_tr_siz ]
val_run.sort()
epochstest = state.epochstest 
nepochs = state.nepochs

depth = state.depth

for i in range(depth):
    err.update({i:{}})
    rec.update({i:[]})
    epochvect.update({i:[]})
    for j in val_run:
        err[i].update({j:[]})
Example #18
0
def run_experiment(experiment,
                   hyper_parameters=None,
                   ask=True,
                   keep=False,
                   dbdescr=None,
                   job_id=None,
                   debug=False,
                   dataset_root="${PYLEARN2_NI_PATH}",
                   **kwargs):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.
    TODO: add sigint handling.

    Parameters
    ----------
    experiment: module
        Experiment module.
    kwargs: dict
        Typically hyperparameters.
    """

    # Fill the hyperparameter values.
    if hyper_parameters is None:
        hyper_parameters = experiment.default_hyperparams

    set_hyper_parameters(hyper_parameters, **kwargs)
    file_parameters = experiment.fileparams
    set_hyper_parameters(file_parameters, **kwargs)
    hyper_parameters.update(file_parameters)

    # Set the output path, default from environment variable $PYLEARN2_OUTS
    out_path = serial.preprocess(
        hyper_parameters.get("out_path", "${PYLEARN2_OUTS}"))
    if not path.isdir(out_path):
        os.makedirs(out_path)

    processing_flag = mp.Value("b", False)
    mem = mp.Value("f", 0.0)
    cpu = mp.Value("f", 0.0)
    last_processed = mp.Manager().dict()
    last_processed["value"] = "Never"

    lh = LogHandler(experiment, out_path, processing_flag, mem, cpu,
                    last_processed, dbdescr, job_id)
    h = logging.StreamHandler(lh)
    lh.logger.info("Hijacking pylearn2 logger (sweet)...")
    monitor.log.addHandler(h)

    try:
        # HACK TODO: fix this. For some reason knex formatted strings are
        # sometimes getting in.
        hyper_parameters = translate(hyper_parameters, "pylearn2")

        # Use the input hander to get input information.
        ih = input_handler.MRIInputHandler()
        input_dim, variance_map_file = ih.get_input_params(
            hyper_parameters, dataset_root=dataset_root)
        if hyper_parameters["nvis"] is None:
            hyper_parameters["nvis"] = input_dim

        # Hack for NICE. Need to rethink inner-dependencies of some model params.
        if ("encoder" in hyper_parameters.keys()
                and "nvis" in hyper_parameters["encoder"].keys()
                and hyper_parameters["encoder"]["nvis"] is None):
            hyper_parameters["encoder"]["nvis"] = input_dim

        # If there's min_lr, make it 1/10 learning_rate
        if "min_lr" in hyper_parameters.keys():
            hyper_parameters["min_lr"] = hyper_parameters["learning_rate"] / 10

        # Corruptor is a special case of hyper parameters that depends on input
        # file: variance_map. So we hack it in here.
        if "corruptor" in hyper_parameters.keys():
            if "variance_map" in hyper_parameters["corruptor"].keys():
                hyper_parameters["corruptor"]["variance_map"] =\
                "!pkl: %s" % variance_map_file
        else:
            hyper_parameters["variance_map_file"] = variance_map_file

        lh.write_json()

        # The Process id
        pid = os.getpid()
        lh.logger.info("Proces id is %d" % pid)

        # If any pdfs are in out_path, kill or quit
        json_file = path.join(out_path, "analysis.json")
        if (ask and (path.isfile(json_file)
                     or len(glob.glob(path.join(out_path, "*.pkl"))) > 0)):
            print("Results found in %s " "Proceeding will erase." % out_path)
            command = None
            while not command in ["yes", "no", "y", "n"]:
                command = raw_input("%s: " % "Proceed?")
                if command in ["yes", "y"]:
                    break
                elif command in ["no", "n"]:
                    exit()
                else:
                    print("Please enter yes(y) or no(n)")
            if path.isfile(json_file):
                with open(json_file) as f:
                    models = json.load(f)
                    for model in models.keys():
                        lh.logger.info("Removing results for model %s" % model)
                        try:
                            os.rmdir(path.join(out_path, "%s_images" % model))
                        except:
                            pass
                os.remove(json_file)

            for pkl in glob.glob(path.join(out_path, "*.pkl")):
                lh.logger.info("Removing %s" % pkl)
                os.remove(pkl)

        lh.logger.info("Making the train object")
        hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict)
        yaml_template = open(experiment.yaml_file).read()
        yaml = yaml_template % hyper_parameters
        train_object = yaml_parse.load(yaml)
        if debug:
            return train_object
        lh.write_json()

        lh.logger.info("Seting up subprocesses")
        lh.logger.info("Setting up listening socket")
        mp_ep, s_ep = mp.Pipe()
        p = mp.Process(target=server, args=(pid, s_ep))
        p.start()
        port = mp_ep.recv()
        lh.logger.info("Listening on port %d" % port)

        lh.logger.info("Starting model processor")
        model_processor = ModelProcessor(experiment, train_object.save_path,
                                         mp_ep, processing_flag,
                                         last_processed)
        model_processor.start()
        lh.logger.info("Model processor started")

        lh.logger.info("Starting stat processor")
        stat_processor = StatProcessor(pid, mem, cpu)
        stat_processor.start()
        lh.logger.info("Stat processor started")

        lh.update(hyperparams=hyper_parameters, yaml=yaml, pid=pid, port=port)
        lh.write_json()

    except Exception as e:
        lh.logger.exception(e)
        lh.finish("FAILED")
        raise e

    # Clean the model after running
    def clean():
        p.terminate()
        lh.logger.info("waiting for server...")
        p.join()
        model_processor.terminate()
        lh.logger.info("waiting for model processor...")
        model_processor.join()
        stat_processor.terminate()
        lh.logger.info("waiting for stat processor...")
        stat_processor.join()

        if keep:
            lh.logger.info("Keeping checkpoints")
        else:
            lh.logger.info("Cleaning checkpoints")
            try:
                os.remove(model_processor.best_checkpoint)
            except:
                pass
            try:
                os.remove(model_processor.checkpoint)
            except:
                pass

    # A signal handler so processes kill cleanly.
    def signal_handler(signum, frame):
        lh.logger.info("Forced quitting...")
        clean()
        lh.finish("KILLED")
        if dbdescr is None:
            exit()
        else:
            raise ValueError("KILLED")

    signal.signal(signal.SIGINT, signal_handler)

    # Main loop.
    try:
        lh.logger.info("Training...")
        train_object.main_loop()
        lh.logger.info("Training quit without exception")
    except Exception as e:
        lh.logger.exception(e)
        clean()
        lh.finish("FAILED")
        raise (e)

    # After complete, process model.
    lh.logger.info("Processing model results...")
    try:
        experiment.analyze_fn(model_processor.best_checkpoint,
                              model_processor.out_path)
    except IOError:
        experiment.analyze_fn(model_processor.checkpoint,
                              model_processor.out_path)
    except Exception as e:
        lh.logger.error(e)

    # Clean checkpoints.
    clean()
    lh.logger.info("Finished experiment.")
    lh.finish("COMPLETED")
    return
Example #19
0
def run_experiment(experiment, hyper_parameters=None, ask=True, keep=False,
                   dbdescr=None, job_id=None, debug=False,
                   dataset_root="${PYLEARN2_NI_PATH}", **kwargs):
    """
    Experiment function.
    Used by jobman to run jobs. Must be loaded externally.
    TODO: add sigint handling.

    Parameters
    ----------
    experiment: module
        Experiment module.
    kwargs: dict
        Typically hyperparameters.
    """

    # Fill the hyperparameter values.
    if hyper_parameters is None:
        hyper_parameters = experiment.default_hyperparams

    set_hyper_parameters(hyper_parameters, **kwargs)
    file_parameters = experiment.fileparams
    set_hyper_parameters(file_parameters, **kwargs)
    hyper_parameters.update(file_parameters)

    # Set the output path, default from environment variable $PYLEARN2_OUTS
    out_path = serial.preprocess(
        hyper_parameters.get("out_path", "${PYLEARN2_OUTS}"))
    if not path.isdir(out_path):
        os.makedirs(out_path)

    processing_flag = mp.Value("b", False)
    mem = mp.Value("f", 0.0)
    cpu = mp.Value("f", 0.0)
    last_processed = mp.Manager().dict()
    last_processed["value"] = "Never"

    lh = LogHandler(experiment, out_path, processing_flag, mem, cpu,
                    last_processed, dbdescr, job_id)
    h = logging.StreamHandler(lh)
    lh.logger.info("Hijacking pylearn2 logger (sweet)...")
    monitor.log.addHandler(h)

    try:
        # HACK TODO: fix this. For some reason knex formatted strings are
        # sometimes getting in.
        hyper_parameters = translate(hyper_parameters, "pylearn2")

        # Use the input hander to get input information.
        ih = input_handler.MRIInputHandler()
        input_dim, variance_map_file = ih.get_input_params(
            hyper_parameters, dataset_root=dataset_root)
        if hyper_parameters["nvis"] is None:
            hyper_parameters["nvis"] = input_dim

        # Hack for NICE. Need to rethink inner-dependencies of some model params.
        if ("encoder" in hyper_parameters.keys()
            and "nvis" in hyper_parameters["encoder"].keys()
            and hyper_parameters["encoder"]["nvis"] is None):
            hyper_parameters["encoder"]["nvis"] = input_dim

        # If there's min_lr, make it 1/10 learning_rate
        if "min_lr" in hyper_parameters.keys():
            hyper_parameters["min_lr"] = hyper_parameters["learning_rate"] / 10

        # Corruptor is a special case of hyper parameters that depends on input
        # file: variance_map. So we hack it in here.
        if "corruptor" in hyper_parameters.keys():
            if "variance_map" in hyper_parameters["corruptor"].keys():
                hyper_parameters["corruptor"]["variance_map"] =\
                "!pkl: %s" % variance_map_file
        else:
            hyper_parameters["variance_map_file"] = variance_map_file

        lh.write_json()

        # The Process id
        pid = os.getpid()
        lh.logger.info("Proces id is %d" % pid)

        # If any pdfs are in out_path, kill or quit
        json_file = path.join(out_path, "analysis.json")
        if (ask and (path.isfile(json_file) or
                     len(glob.glob(path.join(out_path, "*.pkl"))) > 0)):
            print ("Results found in %s "
                   "Proceeding will erase." % out_path)
            command = None
            while not command in ["yes", "no", "y", "n"]:
                command = raw_input("%s: " % "Proceed?")
                if command in ["yes", "y"]:
                    break
                elif command in ["no", "n"]:
                    exit()
                else:
                    print ("Please enter yes(y) or no(n)")
            if path.isfile(json_file):
                with open(json_file) as f:
                    models = json.load(f)
                    for model in models.keys():
                        lh.logger.info("Removing results for model %s" % model)
                        try:
                            os.rmdir(path.join(out_path, "%s_images" % model))
                        except:
                            pass
                os.remove(json_file)

            for pkl in glob.glob(path.join(out_path, "*.pkl")):
                lh.logger.info("Removing %s" % pkl)
                os.remove(pkl)

        lh.logger.info("Making the train object")
        hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict)
        yaml_template = open(experiment.yaml_file).read()
        yaml = yaml_template % hyper_parameters
        train_object = yaml_parse.load(yaml)
        if debug:
            return train_object
        lh.write_json()

        lh.logger.info("Seting up subprocesses")
        lh.logger.info("Setting up listening socket")
        mp_ep, s_ep = mp.Pipe()
        p = mp.Process(target=server, args=(pid, s_ep))
        p.start()
        port = mp_ep.recv()
        lh.logger.info("Listening on port %d" % port)

        lh.logger.info("Starting model processor")
        model_processor = ModelProcessor(experiment, train_object.save_path,
                                         mp_ep, processing_flag, last_processed)
        model_processor.start()
        lh.logger.info("Model processor started")

        lh.logger.info("Starting stat processor")
        stat_processor = StatProcessor(pid, mem, cpu)
        stat_processor.start()
        lh.logger.info("Stat processor started")

        lh.update(hyperparams=hyper_parameters,
                  yaml=yaml,
                  pid=pid,
                  port=port)
        lh.write_json()

    except Exception as e:
        lh.logger.exception(e)
        lh.finish("FAILED")
        raise e

    # Clean the model after running
    def clean():
        p.terminate()
        lh.logger.info("waiting for server...")
        p.join()
        model_processor.terminate()
        lh.logger.info("waiting for model processor...")
        model_processor.join()
        stat_processor.terminate()
        lh.logger.info("waiting for stat processor...")
        stat_processor.join()

        if keep:
            lh.logger.info("Keeping checkpoints")
        else:
            lh.logger.info("Cleaning checkpoints")
            try:
                os.remove(model_processor.best_checkpoint)
            except:
                pass
            try:
                os.remove(model_processor.checkpoint)
            except:
                pass

    # A signal handler so processes kill cleanly.
    def signal_handler(signum, frame):
        lh.logger.info("Forced quitting...")
        clean()
        lh.finish("KILLED")
        if dbdescr is None:
            exit()
        else:
            raise ValueError("KILLED")

    signal.signal(signal.SIGINT, signal_handler)

    # Main loop.
    try:
        lh.logger.info("Training...")
        train_object.main_loop()
        lh.logger.info("Training quit without exception")
    except Exception as e:
        lh.logger.exception(e)
        clean()
        lh.finish("FAILED")
        raise(e)

    # After complete, process model.
    lh.logger.info("Processing model results...")
    try:
        experiment.analyze_fn(model_processor.best_checkpoint,
                                   model_processor.out_path)
    except IOError:
        experiment.analyze_fn(model_processor.checkpoint,
                              model_processor.out_path)
    except Exception as e:
        lh.logger.error(e)

    # Clean checkpoints.
    clean()
    lh.logger.info("Finished experiment.")
    lh.finish("COMPLETED")
    return
    state.hyper_parameters = {
        'trainfile': 'train_gray_uvd_rot_1562_31.h5',
        'N': 10 * 64,
        'batch_size': 64 * 2,
        'c1': c1,
        'kernel_c1': kernel_c1,
        'pool_c1': pool_c1,
        'c2': c2,
        'kernel_c2': kernel_c2,
        'pool_c2': pool_c2,
        'irange_c1': irange_c1,
        'irange_c2': irange_c2,
        'irange_hd1': irange_hd1,
        'irange_hd2': irange_hd2,
        'irange_out': irange_out,
        'hd1': hd1,
        'hd2': hd2,
        'output_dim': outdim,
        'lamda': lamda,
        'decay': decay,
        'max_epochs': 100,
        'save_best_path': save_best_path
    }

    yaml_template = state.yaml_template
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)
    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)
    train_obj.main_loop()
Example #21
0
for all layer in the same graph and reconstruction error. """

import cPickle
import matplotlib
matplotlib.use('Agg')
import pylab
import numpy

from jobman.tools import DD, expand
from jobman.parse import filemerge

rec = {}
err = {}
epochvect = {}

state = expand(DD(filemerge('orig.conf')))
val_run_per_tr_siz = state.validation_runs_for_each_trainingsize
val_run = [int(trainsize) for trainsize in val_run_per_tr_siz]
val_run.sort()
epochstest = state.epochstest
nepochs = state.nepochs

depth = state.depth

for i in range(depth):
    err.update({i: {}})
    rec.update({i: []})
    epochvect.update({i: []})
    for j in val_run:
        err[i].update({j: []})
Example #22
0
                        'n': 2*64,
                            'batch_size': 64,
                            'c1': c1,
                            'kernel_c1':kernel_c1,
                            'pool_c1':pool_c1,
                            'c2': c2,
                            'kernel_c2':kernel_c2,
                            'pool_c2':pool_c2,
                            'irange_c1':irange_c1,
                            'irange_c2':irange_c2,
                            'irange_hd1':irange_hd1,
                            'irange_hd2':irange_hd2,
                            'irange_out':irange_out,
                            'hd1': 2592,
                            'hd2': 36,
                            'output_dim':constants.NUM_JNTS * 3,
                            'lamda':lamda,
                            'decay':decay,
                            'max_epochs': 50,
                            'save_best_path': save_best_path
            }

    yaml_template = state.yaml_template
    hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict)
    # This will be the complete yaml string that should be executed
    final_yaml_str = yaml_template % hyper_parameters
    train_obj = pylearn2.config.yaml_parse.load(final_yaml_str)
    train_obj.main_loop()


# hadag
Example #23
0
    numpy.random.seed(state.seed)
    datatrain = (PATH_DATA+NAME_DATA+'_1.pkl.gz',PATH_DATA+NAME_LABEL+'_1.pkl.gz')
    datatrainsave = PATH_SAVE+'/train.libsvm'
    datatest = (PATH_DATA+NAME_DATATEST+'_1.pkl.gz',PATH_DATA+NAME_LABELTEST+'_1.pkl.gz')
    datatestsave = PATH_SAVE+'/test.libsvm'

    depthbegin = 0

    #monitor best performance for reconstruction and classification
    state.bestrec = []
    state.bestrecepoch = []
    state.besterr = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE])
    state.besterrepoch = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE])

    if MODEL_RELOAD != None:
        oldstate = expand(DD(filemerge(MODEL_RELOAD+'../current.conf')))
        DEPTH = oldstate.depth + DEPTH
        depthbegin = oldstate.depth
        ACT = oldstate.act + ACT
        N_HID = oldstate.n_hid + N_HID
        NOISE = oldstate.noise + NOISE
        ACTIVATION_REGULARIZATION_COEFF = oldstate.activation_regularization_coeff + ACTIVATION_REGULARIZATION_COEFF
        WEIGHT_REGULARIZATION_COEFF = oldstate.weight_regularization_coeff[:-1] + WEIGHT_REGULARIZATION_COEFF
        NEPOCHS = oldstate.nepochs + NEPOCHS
        LR = oldstate.lr + LR
        NOISE_LVL = oldstate.noise_lvl + NOISE_LVL
        EPOCHSTEST = oldstate.epochstest + EPOCHSTEST
        state.bestrec = oldstate.bestrec
        state.bestrecepoch = oldstate.bestrec
        del oldstate
Example #24
0
    numpy.random.seed(state.seed)
    datatrain = (PATH_DATA+NAME_DATA+'_1.pkl',PATH_DATA+NAME_LABEL+'_1.pkl')
    datatrainsave = PATH_SAVE+'/train.libsvm'
    datatest = (PATH_DATA+NAME_DATATEST+'_1.pkl',PATH_DATA+NAME_LABELTEST+'_1.pkl')
    datatestsave = PATH_SAVE+'/test.libsvm'

    depthbegin = 0

    #monitor best performance for reconstruction and classification
    state.bestrec = []
    state.bestrecepoch = []
    state.besterr = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE])
    state.besterrepoch = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE])

    if MODEL_RELOAD != None:
        oldstate = expand(DD(filemerge(MODEL_RELOAD+'../current.conf')))
        DEPTH = oldstate.depth + DEPTH
        depthbegin = oldstate.depth
        ACT = oldstate.act + ACT
        N_HID = oldstate.n_hid + N_HID
        NOISE = oldstate.noise + NOISE
        ACTIVATION_REGULARIZATION_COEFF = oldstate.activation_regularization_coeff + ACTIVATION_REGULARIZATION_COEFF
        WEIGHT_REGULARIZATION_COEFF = oldstate.weight_regularization_coeff[:-1] + WEIGHT_REGULARIZATION_COEFF
        NEPOCHS = oldstate.nepochs + NEPOCHS
        LR = oldstate.lr + LR
        NOISE_LVL = oldstate.noise_lvl + NOISE_LVL
        EPOCHSTEST = oldstate.epochstest + EPOCHSTEST
        state.bestrec = oldstate.bestrec
        state.bestrecepoch = oldstate.bestrec
        del oldstate