def experiment(state, channel): """ Experiment function. Used by jobman to run jobs. Must be loaded externally. Parameters ---------- state: WRITEME channel: WRITEME """ yaml_template = open(yaml_file).read() hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) file_params = expand(flatten(state.file_parameters), dict_type=ydict) # Hack to fill in file parameter strings first for param in file_params: yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param]) yaml = yaml_template % hyper_parameters # with open("/na/homes/dhjelm/pylearn2/pylearn2/jobman/nice_lr_search/%d.yaml" # % state.id, "w") as f: # f.write(yaml) train_object = yaml_parse.load(yaml) train_object.main_loop() state.results = extract_results(train_object.model) return channel.COMPLETE
def experiment(state, channel): """ Experiment function. Used by jobman to run jobs. Must be loaded externally. Parameters ---------- state: WRITEME channel: WRITEME """ yaml_template = open(yaml_file).read() hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) file_params = expand(flatten(state.file_parameters), dict_type=ydict) # Hack to fill in file parameter strings first for param in file_params: yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param]) yaml = yaml_template % hyper_parameters train_object = yaml_parse.load(yaml) state.pid = os.getpid() channel.save() train_object.main_loop() state.results = extract_results(train_object.model) return channel.COMPLETE
def main(args): dataset_name = args.dataset_name logger.info("Getting dataset info for %s" % dataset_name) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name) mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() user = path.expandvars("$USER") save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" % (user, "rbm_simple_test")) # File parameters are path specific ones (not model specific). file_params = {"save_path": save_path, } yaml_template = open(yaml_file).read() hyperparams = expand(flatten(experiment.default_hyperparams(input_dim=input_dim)), dict_type=ydict) # Set additional hyperparams from command line args if args.learning_rate is not None: hyperparams["learning_rate"] = args.learning_rate if args.batch_size is not None: hyperparams["batch_size"] = args.batch_size for param in file_params: yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param]) yaml = yaml_template % hyperparams logger.info("Training") train = yaml_parse.load(yaml) train.main_loop()
def run_experiment_jobman(state, channel): """ Main jobman experiment function, called by all jobs. """ experiment_module = state["experiment&module"] experiment = imp.load_source("module.name", experiment_module) yaml_template = open(experiment.yaml_file).read() hyperparams = expand(flatten(translate(state.hyperparams, "pylearn2")), dict_type=ydict) if not state["out&path"].endswith("job_%d" % state["jobman.id"]): state["out&path"] = path.join(state["out&path"], "job_%d" % state["jobman.id"]) channel.save() out_path = path.join(state["out&path"]) try: run_experiment(experiment, hyperparams, ask=False, out_path=out_path, dbdescr=state["dbdescr"], job_id=state["jobman.id"]) except ValueError as e: if str(e) == "KILLED": return channel.CANCELED else: return channel.ERR_RUN except: return channel.ERR_RUN print "Ending experiment" return channel.COMPLETE
def _eager_dicts(self, lazy_sql_rows, safe_session): eager_dicts = [None] * len(lazy_sql_rows) with safe_session.set_timer(60 * 5): for i, lazy_sql_row in enumerate(lazy_sql_rows): eager_dicts[i] = expand(dict(lazy_sql_row.iteritems())) return eager_dicts
def set_jobs(name, status, new_status): experiments = experiment_scheduler.load_experiments( cluster=None, filter_eq_dct=dict(name=name)) if len(experiments) == 0: print "No experiments in database %s" % get_db_string("experiments") return experiment = experiments[0] if status == "pending": jobs = job_scheduler.load_pending_jobs(experiment["table"]) elif status == "broken": jobs = job_scheduler.load_broken_jobs(experiment["table"]) elif status == "completed": jobs = job_scheduler.load_completed_jobs(experiment["table"]) elif status == "running": jobs = job_scheduler.load_running_jobs(experiment["table"]) if new_status == "pending": sql_new_status = sql.START new_status = "pending" elif new_status == "running": sql_new_status = sql.RUNNING elif new_status == "completed": sql_new_status = sql.COMPLETE print "Setting %s jobs to %s status..." % (status, new_status) job_scheduler.update_jobs(experiment["table"], jobs, expand({sql.STATUS: sql_new_status, 'proc_status': new_status}))
def train_experiment(state, channel): """ Train a model specified in state, and extract required results. This function builds a YAML string from ``state.yaml_template``, taking the values of hyper-parameters from ``state.hyper_parameters``, creates the corresponding object and trains it (like train.py), then run the function in ``state.extract_results`` on it, and store the returned values into ``state.results``. To know how to use this function, you can check the example in tester.py (in the same directory). """ yaml_template = state.yaml_template # Convert nested DD into nested ydict. hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) # This will be the complete yaml string that should be executed final_yaml_str = yaml_template % hyper_parameters # Instantiate an object from YAML string train_obj = pylearn2.config.yaml_parse.load(final_yaml_str) for ext in train_obj.extensions: if hasattr(ext, 'set_train_obj'): ext.set_train_obj(train_obj) if hasattr(ext, 'set_jobman_channel'): ext.set_jobman_channel(channel) if hasattr(ext, 'set_jobman_state'): ext.set_jobman_state(state) try: iter(train_obj) iterable = True except TypeError: iterable = False if iterable: raise NotImplementedError( ('Current implementation does not support running multiple ' 'models in one yaml string. Please change the yaml template ' 'and parameters to contain only one single model.')) else: # print "Executing the model." train_obj.main_loop() # This line will call a function defined by the user and pass train_obj # to it. state.results = jobman.tools.resolve(state.extract_results)(train_obj) return channel.COMPLETE
def train_experiment(state, channel): """ Train a model specified in state, and extract required results. This function builds a YAML string from ``state.yaml_template``, taking the values of hyper-parameters from ``state.hyper_parameters``, creates the corresponding object and trains it (like train.py), then run the function in ``state.extract_results`` on it, and store the returned values into ``state.results``. To know how to use this function, you can check the example in tester.py (in the same directory). """ yaml_template = state.yaml_template # Convert nested DD into nested ydict. hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) # This will be the complete yaml string that should be executed final_yaml_str = yaml_template % hyper_parameters # Instantiate an object from YAML string train_obj = pylearn2.config.yaml_parse.load(final_yaml_str) for ext in train_obj.extensions: if hasattr(ext, 'set_train_obj'): ext.set_train_obj( train_obj ) if hasattr(ext, 'set_jobman_channel'): ext.set_jobman_channel( channel ) if hasattr(ext, 'set_jobman_state'): ext.set_jobman_state( state ) try: iter(train_obj) iterable = True except TypeError: iterable = False if iterable: raise NotImplementedError( ('Current implementation does not support running multiple ' 'models in one yaml string. Please change the yaml template ' 'and parameters to contain only one single model.')) else: # print "Executing the model." train_obj.main_loop() # This line will call a function defined by the user and pass train_obj # to it. state.results = jobman.tools.resolve(state.extract_results)(train_obj) return channel.COMPLETE
def run_experiment(experiment, **kwargs): """ Experiment function. Used by jobman to run jobs. Must be loaded externally. TODO: add sigint handling. Parameters ---------- experiment: module Experiment module. kwargs: dict Typically hyperparameters. """ hyper_parameters = experiment.default_hyperparams() set_hyper_parameters(hyper_parameters, **kwargs) file_parameters = experiment.fileparams set_hyper_parameters(file_parameters, **kwargs) hyper_parameters.update(file_parameters) ih = MRIInputHandler() input_dim, variance_map_file = ih.get_input_params(hyper_parameters) hyper_parameters["nvis"] = input_dim hyper_parameters["variance_map_file"] = variance_map_file pid = os.getpid() out_path = serial.preprocess( hyper_parameters.get("out_path", "${PYLEARN2_OUTS}")) if not path.isdir(out_path): os.mkdir(out_path) if not path.isdir(path.join(out_path, "logs")): os.mkdir(path.join(out_path, "logs")) hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict) lh = LogHandler(experiment, hyper_parameters, out_path, pid) h = logging.StreamHandler(lh) monitor.log.addHandler(h) yaml_template = open(experiment.yaml_file).read() yaml = yaml_template % hyper_parameters train_object = yaml_parse.load(yaml) try: train_object.main_loop() lh.finish("COMPLETED") except KeyboardInterrupt: print("Quitting...") lh.finish("KILLED")
def train_experiment(state, channel): """ Train a model specified in state, and extract required results. This function builds a YAML string from ``state.yaml_template``, taking the values of hyper-parameters from ``state.hyper_parameters``, creates the corresponding object and trains it (like train.py), then run the function in ``state.extract_results`` on it, and store the returned values into ``state.results``. To know how to use this function, you can check the example in tester.py (in the same directory). """ yaml_template = state.yaml_template # Convert nested DD into nested ydict. hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) # This will be the complete yaml string that should be executed final_yaml_str = yaml_template % hyper_parameters # write to .yaml file for ease of reproducibility fp = open('experiment.yaml', 'w') fp.write('%s' % final_yaml_str[2:]) fp.close() # Instantiate an object from YAML string train_obj = pylearn2.config.yaml_parse.load(final_yaml_str) try: iter(train_obj) iterable = True except TypeError: iterable = False if iterable: raise NotImplementedError( ('Current implementation does not support running multiple ' 'models in one yaml string. Please change the yaml template ' 'and parameters to contain only one single model.')) else: # print "Executing the model." # (GD) HACK HACK train_obj.model.jobman_channel = channel train_obj.model.jobman_state = state train_obj.main_loop() return channel.COMPLETE
def main(dataset_name="smri"): logger.info("Getting dataset info for %s" % args.dataset_name) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name) mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() if input_dim % 2 == 1: input_dim -= 1 mri = MRI.MRI_Standard(which_set="full", dataset_name=args.dataset_name, unit_normalize=True, even_input=True, apply_mask=True) variance_map_file = path.join(data_path, "variance_map.npy") mri_nifti.save_variance_map(mri, variance_map_file) user = path.expandvars("$USER") save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" % (user, "jobman_test")) file_params = {"save_path": save_path, "variance_map_file": variance_map_file } yaml_template = open(yaml_file).read() hyperparams = expand(flatten(mlp_experiment.default_hyperparams(input_dim=input_dim)), dict_type=ydict) for param in hyperparams: if hasattr(args, param) and getattr(args, param): val = getattr(args, param) logger.info("Filling %s with %r" % (param, val)) hyperparams[param] = type(hyperparams[param])(val) for param in file_params: yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param]) yaml = yaml_template % hyperparams print yaml logger.info("Training") train = yaml_parse.load(yaml) train.main_loop()
def main(args): dataset_name = args.dataset_name logger.info("Getting dataset info for %s" % dataset_name) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name) mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() user = path.expandvars("$USER") save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" % (user, "rbm_simple_test")) # File parameters are path specific ones (not model specific). file_params = { "save_path": save_path, } yaml_template = open(yaml_file).read() hyperparams = expand(flatten( experiment.default_hyperparams(input_dim=input_dim)), dict_type=ydict) # Set additional hyperparams from command line args if args.learning_rate is not None: hyperparams["learning_rate"] = args.learning_rate if args.batch_size is not None: hyperparams["batch_size"] = args.batch_size for param in file_params: yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param]) yaml = yaml_template % hyperparams logger.info("Training") train = yaml_parse.load(yaml) train.main_loop()
for all layer in the same graph and reconstruction error. """ import cPickle import matplotlib matplotlib.use('Agg') import pylab import numpy from jobman.tools import DD,expand from jobman.parse import filemerge rec = {} err = {} epochvect = {} state = expand(DD(filemerge('orig.conf'))) val_run_per_tr_siz = state.validation_runs_for_each_trainingsize val_run = [int(trainsize) for trainsize in val_run_per_tr_siz ] val_run.sort() epochstest = state.epochstest nepochs = state.nepochs depth = state.depth for i in range(depth): err.update({i:{}}) rec.update({i:[]}) epochvect.update({i:[]}) for j in val_run: err[i].update({j:[]})
def run_experiment(experiment, hyper_parameters=None, ask=True, keep=False, dbdescr=None, job_id=None, debug=False, dataset_root="${PYLEARN2_NI_PATH}", **kwargs): """ Experiment function. Used by jobman to run jobs. Must be loaded externally. TODO: add sigint handling. Parameters ---------- experiment: module Experiment module. kwargs: dict Typically hyperparameters. """ # Fill the hyperparameter values. if hyper_parameters is None: hyper_parameters = experiment.default_hyperparams set_hyper_parameters(hyper_parameters, **kwargs) file_parameters = experiment.fileparams set_hyper_parameters(file_parameters, **kwargs) hyper_parameters.update(file_parameters) # Set the output path, default from environment variable $PYLEARN2_OUTS out_path = serial.preprocess( hyper_parameters.get("out_path", "${PYLEARN2_OUTS}")) if not path.isdir(out_path): os.makedirs(out_path) processing_flag = mp.Value("b", False) mem = mp.Value("f", 0.0) cpu = mp.Value("f", 0.0) last_processed = mp.Manager().dict() last_processed["value"] = "Never" lh = LogHandler(experiment, out_path, processing_flag, mem, cpu, last_processed, dbdescr, job_id) h = logging.StreamHandler(lh) lh.logger.info("Hijacking pylearn2 logger (sweet)...") monitor.log.addHandler(h) try: # HACK TODO: fix this. For some reason knex formatted strings are # sometimes getting in. hyper_parameters = translate(hyper_parameters, "pylearn2") # Use the input hander to get input information. ih = input_handler.MRIInputHandler() input_dim, variance_map_file = ih.get_input_params( hyper_parameters, dataset_root=dataset_root) if hyper_parameters["nvis"] is None: hyper_parameters["nvis"] = input_dim # Hack for NICE. Need to rethink inner-dependencies of some model params. if ("encoder" in hyper_parameters.keys() and "nvis" in hyper_parameters["encoder"].keys() and hyper_parameters["encoder"]["nvis"] is None): hyper_parameters["encoder"]["nvis"] = input_dim # If there's min_lr, make it 1/10 learning_rate if "min_lr" in hyper_parameters.keys(): hyper_parameters["min_lr"] = hyper_parameters["learning_rate"] / 10 # Corruptor is a special case of hyper parameters that depends on input # file: variance_map. So we hack it in here. if "corruptor" in hyper_parameters.keys(): if "variance_map" in hyper_parameters["corruptor"].keys(): hyper_parameters["corruptor"]["variance_map"] =\ "!pkl: %s" % variance_map_file else: hyper_parameters["variance_map_file"] = variance_map_file lh.write_json() # The Process id pid = os.getpid() lh.logger.info("Proces id is %d" % pid) # If any pdfs are in out_path, kill or quit json_file = path.join(out_path, "analysis.json") if (ask and (path.isfile(json_file) or len(glob.glob(path.join(out_path, "*.pkl"))) > 0)): print("Results found in %s " "Proceeding will erase." % out_path) command = None while not command in ["yes", "no", "y", "n"]: command = raw_input("%s: " % "Proceed?") if command in ["yes", "y"]: break elif command in ["no", "n"]: exit() else: print("Please enter yes(y) or no(n)") if path.isfile(json_file): with open(json_file) as f: models = json.load(f) for model in models.keys(): lh.logger.info("Removing results for model %s" % model) try: os.rmdir(path.join(out_path, "%s_images" % model)) except: pass os.remove(json_file) for pkl in glob.glob(path.join(out_path, "*.pkl")): lh.logger.info("Removing %s" % pkl) os.remove(pkl) lh.logger.info("Making the train object") hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict) yaml_template = open(experiment.yaml_file).read() yaml = yaml_template % hyper_parameters train_object = yaml_parse.load(yaml) if debug: return train_object lh.write_json() lh.logger.info("Seting up subprocesses") lh.logger.info("Setting up listening socket") mp_ep, s_ep = mp.Pipe() p = mp.Process(target=server, args=(pid, s_ep)) p.start() port = mp_ep.recv() lh.logger.info("Listening on port %d" % port) lh.logger.info("Starting model processor") model_processor = ModelProcessor(experiment, train_object.save_path, mp_ep, processing_flag, last_processed) model_processor.start() lh.logger.info("Model processor started") lh.logger.info("Starting stat processor") stat_processor = StatProcessor(pid, mem, cpu) stat_processor.start() lh.logger.info("Stat processor started") lh.update(hyperparams=hyper_parameters, yaml=yaml, pid=pid, port=port) lh.write_json() except Exception as e: lh.logger.exception(e) lh.finish("FAILED") raise e # Clean the model after running def clean(): p.terminate() lh.logger.info("waiting for server...") p.join() model_processor.terminate() lh.logger.info("waiting for model processor...") model_processor.join() stat_processor.terminate() lh.logger.info("waiting for stat processor...") stat_processor.join() if keep: lh.logger.info("Keeping checkpoints") else: lh.logger.info("Cleaning checkpoints") try: os.remove(model_processor.best_checkpoint) except: pass try: os.remove(model_processor.checkpoint) except: pass # A signal handler so processes kill cleanly. def signal_handler(signum, frame): lh.logger.info("Forced quitting...") clean() lh.finish("KILLED") if dbdescr is None: exit() else: raise ValueError("KILLED") signal.signal(signal.SIGINT, signal_handler) # Main loop. try: lh.logger.info("Training...") train_object.main_loop() lh.logger.info("Training quit without exception") except Exception as e: lh.logger.exception(e) clean() lh.finish("FAILED") raise (e) # After complete, process model. lh.logger.info("Processing model results...") try: experiment.analyze_fn(model_processor.best_checkpoint, model_processor.out_path) except IOError: experiment.analyze_fn(model_processor.checkpoint, model_processor.out_path) except Exception as e: lh.logger.error(e) # Clean checkpoints. clean() lh.logger.info("Finished experiment.") lh.finish("COMPLETED") return
def run_experiment(experiment, hyper_parameters=None, ask=True, keep=False, dbdescr=None, job_id=None, debug=False, dataset_root="${PYLEARN2_NI_PATH}", **kwargs): """ Experiment function. Used by jobman to run jobs. Must be loaded externally. TODO: add sigint handling. Parameters ---------- experiment: module Experiment module. kwargs: dict Typically hyperparameters. """ # Fill the hyperparameter values. if hyper_parameters is None: hyper_parameters = experiment.default_hyperparams set_hyper_parameters(hyper_parameters, **kwargs) file_parameters = experiment.fileparams set_hyper_parameters(file_parameters, **kwargs) hyper_parameters.update(file_parameters) # Set the output path, default from environment variable $PYLEARN2_OUTS out_path = serial.preprocess( hyper_parameters.get("out_path", "${PYLEARN2_OUTS}")) if not path.isdir(out_path): os.makedirs(out_path) processing_flag = mp.Value("b", False) mem = mp.Value("f", 0.0) cpu = mp.Value("f", 0.0) last_processed = mp.Manager().dict() last_processed["value"] = "Never" lh = LogHandler(experiment, out_path, processing_flag, mem, cpu, last_processed, dbdescr, job_id) h = logging.StreamHandler(lh) lh.logger.info("Hijacking pylearn2 logger (sweet)...") monitor.log.addHandler(h) try: # HACK TODO: fix this. For some reason knex formatted strings are # sometimes getting in. hyper_parameters = translate(hyper_parameters, "pylearn2") # Use the input hander to get input information. ih = input_handler.MRIInputHandler() input_dim, variance_map_file = ih.get_input_params( hyper_parameters, dataset_root=dataset_root) if hyper_parameters["nvis"] is None: hyper_parameters["nvis"] = input_dim # Hack for NICE. Need to rethink inner-dependencies of some model params. if ("encoder" in hyper_parameters.keys() and "nvis" in hyper_parameters["encoder"].keys() and hyper_parameters["encoder"]["nvis"] is None): hyper_parameters["encoder"]["nvis"] = input_dim # If there's min_lr, make it 1/10 learning_rate if "min_lr" in hyper_parameters.keys(): hyper_parameters["min_lr"] = hyper_parameters["learning_rate"] / 10 # Corruptor is a special case of hyper parameters that depends on input # file: variance_map. So we hack it in here. if "corruptor" in hyper_parameters.keys(): if "variance_map" in hyper_parameters["corruptor"].keys(): hyper_parameters["corruptor"]["variance_map"] =\ "!pkl: %s" % variance_map_file else: hyper_parameters["variance_map_file"] = variance_map_file lh.write_json() # The Process id pid = os.getpid() lh.logger.info("Proces id is %d" % pid) # If any pdfs are in out_path, kill or quit json_file = path.join(out_path, "analysis.json") if (ask and (path.isfile(json_file) or len(glob.glob(path.join(out_path, "*.pkl"))) > 0)): print ("Results found in %s " "Proceeding will erase." % out_path) command = None while not command in ["yes", "no", "y", "n"]: command = raw_input("%s: " % "Proceed?") if command in ["yes", "y"]: break elif command in ["no", "n"]: exit() else: print ("Please enter yes(y) or no(n)") if path.isfile(json_file): with open(json_file) as f: models = json.load(f) for model in models.keys(): lh.logger.info("Removing results for model %s" % model) try: os.rmdir(path.join(out_path, "%s_images" % model)) except: pass os.remove(json_file) for pkl in glob.glob(path.join(out_path, "*.pkl")): lh.logger.info("Removing %s" % pkl) os.remove(pkl) lh.logger.info("Making the train object") hyper_parameters = expand(flatten(hyper_parameters), dict_type=ydict) yaml_template = open(experiment.yaml_file).read() yaml = yaml_template % hyper_parameters train_object = yaml_parse.load(yaml) if debug: return train_object lh.write_json() lh.logger.info("Seting up subprocesses") lh.logger.info("Setting up listening socket") mp_ep, s_ep = mp.Pipe() p = mp.Process(target=server, args=(pid, s_ep)) p.start() port = mp_ep.recv() lh.logger.info("Listening on port %d" % port) lh.logger.info("Starting model processor") model_processor = ModelProcessor(experiment, train_object.save_path, mp_ep, processing_flag, last_processed) model_processor.start() lh.logger.info("Model processor started") lh.logger.info("Starting stat processor") stat_processor = StatProcessor(pid, mem, cpu) stat_processor.start() lh.logger.info("Stat processor started") lh.update(hyperparams=hyper_parameters, yaml=yaml, pid=pid, port=port) lh.write_json() except Exception as e: lh.logger.exception(e) lh.finish("FAILED") raise e # Clean the model after running def clean(): p.terminate() lh.logger.info("waiting for server...") p.join() model_processor.terminate() lh.logger.info("waiting for model processor...") model_processor.join() stat_processor.terminate() lh.logger.info("waiting for stat processor...") stat_processor.join() if keep: lh.logger.info("Keeping checkpoints") else: lh.logger.info("Cleaning checkpoints") try: os.remove(model_processor.best_checkpoint) except: pass try: os.remove(model_processor.checkpoint) except: pass # A signal handler so processes kill cleanly. def signal_handler(signum, frame): lh.logger.info("Forced quitting...") clean() lh.finish("KILLED") if dbdescr is None: exit() else: raise ValueError("KILLED") signal.signal(signal.SIGINT, signal_handler) # Main loop. try: lh.logger.info("Training...") train_object.main_loop() lh.logger.info("Training quit without exception") except Exception as e: lh.logger.exception(e) clean() lh.finish("FAILED") raise(e) # After complete, process model. lh.logger.info("Processing model results...") try: experiment.analyze_fn(model_processor.best_checkpoint, model_processor.out_path) except IOError: experiment.analyze_fn(model_processor.checkpoint, model_processor.out_path) except Exception as e: lh.logger.error(e) # Clean checkpoints. clean() lh.logger.info("Finished experiment.") lh.finish("COMPLETED") return
state.hyper_parameters = { 'trainfile': 'train_gray_uvd_rot_1562_31.h5', 'N': 10 * 64, 'batch_size': 64 * 2, 'c1': c1, 'kernel_c1': kernel_c1, 'pool_c1': pool_c1, 'c2': c2, 'kernel_c2': kernel_c2, 'pool_c2': pool_c2, 'irange_c1': irange_c1, 'irange_c2': irange_c2, 'irange_hd1': irange_hd1, 'irange_hd2': irange_hd2, 'irange_out': irange_out, 'hd1': hd1, 'hd2': hd2, 'output_dim': outdim, 'lamda': lamda, 'decay': decay, 'max_epochs': 100, 'save_best_path': save_best_path } yaml_template = state.yaml_template hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) # This will be the complete yaml string that should be executed final_yaml_str = yaml_template % hyper_parameters train_obj = pylearn2.config.yaml_parse.load(final_yaml_str) train_obj.main_loop()
for all layer in the same graph and reconstruction error. """ import cPickle import matplotlib matplotlib.use('Agg') import pylab import numpy from jobman.tools import DD, expand from jobman.parse import filemerge rec = {} err = {} epochvect = {} state = expand(DD(filemerge('orig.conf'))) val_run_per_tr_siz = state.validation_runs_for_each_trainingsize val_run = [int(trainsize) for trainsize in val_run_per_tr_siz] val_run.sort() epochstest = state.epochstest nepochs = state.nepochs depth = state.depth for i in range(depth): err.update({i: {}}) rec.update({i: []}) epochvect.update({i: []}) for j in val_run: err[i].update({j: []})
'n': 2*64, 'batch_size': 64, 'c1': c1, 'kernel_c1':kernel_c1, 'pool_c1':pool_c1, 'c2': c2, 'kernel_c2':kernel_c2, 'pool_c2':pool_c2, 'irange_c1':irange_c1, 'irange_c2':irange_c2, 'irange_hd1':irange_hd1, 'irange_hd2':irange_hd2, 'irange_out':irange_out, 'hd1': 2592, 'hd2': 36, 'output_dim':constants.NUM_JNTS * 3, 'lamda':lamda, 'decay':decay, 'max_epochs': 50, 'save_best_path': save_best_path } yaml_template = state.yaml_template hyper_parameters = expand(flatten(state.hyper_parameters), dict_type=ydict) # This will be the complete yaml string that should be executed final_yaml_str = yaml_template % hyper_parameters train_obj = pylearn2.config.yaml_parse.load(final_yaml_str) train_obj.main_loop() # hadag
numpy.random.seed(state.seed) datatrain = (PATH_DATA+NAME_DATA+'_1.pkl.gz',PATH_DATA+NAME_LABEL+'_1.pkl.gz') datatrainsave = PATH_SAVE+'/train.libsvm' datatest = (PATH_DATA+NAME_DATATEST+'_1.pkl.gz',PATH_DATA+NAME_LABELTEST+'_1.pkl.gz') datatestsave = PATH_SAVE+'/test.libsvm' depthbegin = 0 #monitor best performance for reconstruction and classification state.bestrec = [] state.bestrecepoch = [] state.besterr = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE]) state.besterrepoch = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE]) if MODEL_RELOAD != None: oldstate = expand(DD(filemerge(MODEL_RELOAD+'../current.conf'))) DEPTH = oldstate.depth + DEPTH depthbegin = oldstate.depth ACT = oldstate.act + ACT N_HID = oldstate.n_hid + N_HID NOISE = oldstate.noise + NOISE ACTIVATION_REGULARIZATION_COEFF = oldstate.activation_regularization_coeff + ACTIVATION_REGULARIZATION_COEFF WEIGHT_REGULARIZATION_COEFF = oldstate.weight_regularization_coeff[:-1] + WEIGHT_REGULARIZATION_COEFF NEPOCHS = oldstate.nepochs + NEPOCHS LR = oldstate.lr + LR NOISE_LVL = oldstate.noise_lvl + NOISE_LVL EPOCHSTEST = oldstate.epochstest + EPOCHSTEST state.bestrec = oldstate.bestrec state.bestrecepoch = oldstate.bestrec del oldstate
numpy.random.seed(state.seed) datatrain = (PATH_DATA+NAME_DATA+'_1.pkl',PATH_DATA+NAME_LABEL+'_1.pkl') datatrainsave = PATH_SAVE+'/train.libsvm' datatest = (PATH_DATA+NAME_DATATEST+'_1.pkl',PATH_DATA+NAME_LABELTEST+'_1.pkl') datatestsave = PATH_SAVE+'/test.libsvm' depthbegin = 0 #monitor best performance for reconstruction and classification state.bestrec = [] state.bestrecepoch = [] state.besterr = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE]) state.besterrepoch = dict([(`trainsize`, []) for trainsize in VALIDATION_TRAININGSIZE]) if MODEL_RELOAD != None: oldstate = expand(DD(filemerge(MODEL_RELOAD+'../current.conf'))) DEPTH = oldstate.depth + DEPTH depthbegin = oldstate.depth ACT = oldstate.act + ACT N_HID = oldstate.n_hid + N_HID NOISE = oldstate.noise + NOISE ACTIVATION_REGULARIZATION_COEFF = oldstate.activation_regularization_coeff + ACTIVATION_REGULARIZATION_COEFF WEIGHT_REGULARIZATION_COEFF = oldstate.weight_regularization_coeff[:-1] + WEIGHT_REGULARIZATION_COEFF NEPOCHS = oldstate.nepochs + NEPOCHS LR = oldstate.lr + LR NOISE_LVL = oldstate.noise_lvl + NOISE_LVL EPOCHSTEST = oldstate.epochstest + EPOCHSTEST state.bestrec = oldstate.bestrec state.bestrecepoch = oldstate.bestrec del oldstate