def load_experiments(args): dataset_name = args.dataset_name db = sql.db("postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % {"user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) logger.info("Getting dataset info for %s%s" % (dataset_name, ", transposed" if args.transposed else "")) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name) if args.transposed: logger.info("Data in transpose...") mri = MRI.MRI_Transposed(dataset_name=args.dataset_name, unit_normalize=True, even_input=True, apply_mask=True) input_dim = mri.X.shape[1] variance_map_file = path.join(data_path, "transposed_variance_map.npy") else: mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() if input_dim % 2 == 1: input_dim -= 1 mri = MRI.MRI_Standard(which_set="full", dataset_name=args.dataset_name, unit_normalize=True, even_input=True, apply_mask=True) variance_map_file = path.join(data_path, "variance_map.npy") mri_nifti.save_variance_map(mri, variance_map_file) for items in nested_generator(layer_depth_generator("encoder.layer_depths", xrange(4, 6), 5), hidden_generator("encoder.nhid", 4), float_generator("weight_decay.coeffs.z", 3, 0.1, 0.001, log_scale=True)): # logger.info("Adding NICE experiment with hyperparameters %s" % (items, )) state = DD() experiment_hyperparams = nice_experiment.default_hyperparams(input_dim) if args.transposed: experiment_hyperparams["data_class"] = "MRI_Transposed" if args.logistic: experiment_hyperparams["prior"]["__builder__"] =\ "nice.pylearn2.models.nice.StandardLogistic" for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] entry[split_keys[-1]] = value experiment_hyperparams["dataset_name"] = dataset_name h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ [tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()]))) user = path.expandvars("$USER") save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%d" % (user, h)) file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams sql.insert_job( nice_experiment.experiment, flatten(state), db ) db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name db = sql.db( "postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % { "user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) logger.info("Getting dataset info for %s" % dataset_name) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name) mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() if input_dim % 2 == 1: input_dim -= 1 mri = MRI.MRI_Standard(which_set="full", dataset_name=dataset_name, unit_normalize=True, even_input=True, apply_mask=True) variance_map_file = path.join(data_path, "variance_map.npy") mri_nifti.save_variance_map(mri, variance_map_file) for items in jg.nested_generator( jg.hidden_generator("nhid1", 1), jg.hidden_generator("nhid2", 1), ): state = DD() experiment_hyperparams = mlp_experiment.default_hyperparams(input_dim) for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] entry[split_keys[-1]] = value experiment_hyperparams["dataset_name"] = dataset_name h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ flatten(experiment_hyperparams).values()))) user = path.expandvars("$USER") save_path = serial.preprocess( "/export/mialab/users/%s/pylearn2_outs/%d" % (user, h)) file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams state.pid = 0 sql.insert_job(mlp_experiment.experiment, flatten(state), db) db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name # Load the database and table. db = sql.db( "postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % { "user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) # Don't worry about this yet. input_handler = InputHandler() # For generating models, we use a special set of jobman generators, made # for convenience. for items in jg.nested_generator( jg.float_generator("learning_rate", 3, 0.01, 0.0001, log_scale=True), jg.list_generator("nhid", [50, 100, 200, 300]), ): logger.info("Adding RBM experiment across hyperparameters %s" % (items, )) state = DD() # Load experiment hyperparams from experiment experiment_hyperparams = experiment.default_hyperparams() # Set them with values in our loop. for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] assert split_keys[-1] in entry, ( "Key not found in hyperparams: %s" % split_keys[-1]) entry[split_keys[-1]] = value # Set the dataset name experiment_hyperparams["dataset_name"] = dataset_name # Get the input dim and variance map. Don't worry about variance maps right now, # they aren't used here. input_dim, variance_map_file = input_handler.get_input_params( args, experiment_hyperparams) logger.info("%s\n%s\n" % (input_dim, variance_map_file)) # Set the input dimensionality by the data experiment_hyperparams["nvis"] = input_dim # Set the minimum learning rate relative to the initial learning rate. experiment_hyperparams[ "min_lr"] = experiment_hyperparams["learning_rate"] / 10 # Make a unique hash for experiments. Remember that lists, dicts, and other data # types may not be hashable, so you may need to do some special processing. In # this case we convert the lists to tuples. h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ [tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()]))) # Save path for the experiments. In this case we are sharing a directory in my # export directory so IT can blame me. save_path = serial.preprocess( "/export/mialab/users/dhjelm/pylearn2_outs/rbm_demo/%d" % h) # We save file params separately as they aren't model specific. file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams user = path.expandvars("$USER") state.created_by = user # Finally we add the experiment to the table. sql.insert_job(experiment.experiment, flatten(state), db) # A view can be used when querying the database using psql. May not be needed in future. db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name db = sql.db( "postgres://%(user)s@%(host)s:" "%(port)d/%(database)s?table=%(table)s" % { "user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) input_handler = InputHandler() for items in jg.nested_generator( jg.list_generator( "encoder.layer_depths", [[3, 5, 5, 5, 3], [5, 5, 5, 5, 5], [2, 4, 4, 2]]), jg.list_generator("variance_normalize", [False, 2]), jg.float_generator("weight_decay.coeff", 4, 0.1, 0.0001, log_scale=True), jg.list_generator("prior.__builder__", [ "nice.pylearn2.models.nice.StandardNormal", "nice.pylearn2.models.nice.StandardLogistic" ])): logger.info("Adding NICE experiment across hyperparameters %s" % (items, )) state = DD() experiment_hyperparams = nice_experiment.default_hyperparams() for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] assert split_keys[-1] in entry,\ ("Key not found in hyperparams: %s, " "found: %s" % (split_keys[-1], entry.keys())) entry[split_keys[-1]] = value experiment_hyperparams["dataset_name"] = dataset_name input_dim, variance_map_file = input_handler.get_input_params( args, experiment_hyperparams) logger.info("%s\n%s\n" % (input_dim, variance_map_file)) experiment_hyperparams["nvis"] = input_dim experiment_hyperparams["encoder"]["nvis"] = input_dim h = abs(hash(frozenset( flatten(experiment_hyperparams).keys() +\ [tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()]))) user = path.expandvars("$USER") save_path = serial.preprocess( "/export/mialab/users/%s/pylearn2_outs/%d" % (user, h)) file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams sql.insert_job(nice_experiment.experiment, flatten(state), db) db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name db = sql.db( "postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % { "user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) logger.info("Getting dataset info for %s%s" % (dataset_name, ", transposed" if args.transposed else "")) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name) if args.transposed: logger.info("Data in transpose...") mri = MRI.MRI_Transposed(dataset_name=args.dataset_name, unit_normalize=True, even_input=True, apply_mask=True) input_dim = mri.X.shape[1] variance_map_file = path.join(data_path, "transposed_variance_map.npy") else: mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() if input_dim % 2 == 1: input_dim -= 1 mri = MRI.MRI_Standard(which_set="full", dataset_name=args.dataset_name, unit_normalize=True, even_input=True, apply_mask=True) variance_map_file = path.join(data_path, "variance_map.npy") mri_nifti.save_variance_map(mri, variance_map_file) for items in nested_generator( layer_depth_generator("encoder.layer_depths", xrange(4, 6), 5), hidden_generator("encoder.nhid", 4), float_generator("weight_decay.coeffs.z", 3, 0.1, 0.001, log_scale=True)): # logger.info("Adding NICE experiment with hyperparameters %s" % (items, )) state = DD() experiment_hyperparams = nice_experiment.default_hyperparams(input_dim) if args.transposed: experiment_hyperparams["data_class"] = "MRI_Transposed" if args.logistic: experiment_hyperparams["prior"]["__builder__"] =\ "nice.pylearn2.models.nice.StandardLogistic" for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] entry[split_keys[-1]] = value experiment_hyperparams["dataset_name"] = dataset_name h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ [tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()]))) user = path.expandvars("$USER") save_path = serial.preprocess( "/export/mialab/users/%s/pylearn2_outs/%d" % (user, h)) file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams sql.insert_job(nice_experiment.experiment, flatten(state), db) db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name db = sql.db("postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % {"user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) logger.info("Getting dataset info for %s" % dataset_name) data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name) mask_file = path.join(data_path, "mask.npy") mask = np.load(mask_file) input_dim = (mask == 1).sum() if input_dim % 2 == 1: input_dim -= 1 mri = MRI.MRI_Standard(which_set="full", dataset_name=dataset_name, unit_normalize=True, even_input=True, apply_mask=True) variance_map_file = path.join(data_path, "variance_map.npy") mri_nifti.save_variance_map(mri, variance_map_file) for items in jg.nested_generator(jg.hidden_generator("nhid1", 1), jg.hidden_generator("nhid2", 1), ): state = DD() experiment_hyperparams = mlp_experiment.default_hyperparams(input_dim) for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] entry[split_keys[-1]] = value experiment_hyperparams["dataset_name"] = dataset_name h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ flatten(experiment_hyperparams).values()))) user = path.expandvars("$USER") save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%d" % (user, h)) file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams state.pid = 0 sql.insert_job( mlp_experiment.experiment, flatten(state), db ) db.createView("%s_view" % args.table)
def load_experiments(args): dataset_name = args.dataset_name # Load the database and table. db = sql.db("postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" % {"user": args.user, "host": args.host, "port": args.port, "database": args.database, "table": args.table, }) # Don't worry about this yet. input_handler = InputHandler() # For generating models, we use a special set of jobman generators, made # for convenience. for items in jg.nested_generator( jg.float_generator("learning_rate", 3, 0.01, 0.0001, log_scale=True), jg.list_generator("nhid", [50, 100, 200, 300]), ): logger.info("Adding RBM experiment across hyperparameters %s" % (items, )) state = DD() # Load experiment hyperparams from experiment experiment_hyperparams = experiment.default_hyperparams() # Set them with values in our loop. for key, value in items: split_keys = key.split(".") entry = experiment_hyperparams for k in split_keys[:-1]: entry = entry[k] assert split_keys[-1] in entry, ("Key not found in hyperparams: %s" % split_keys[-1]) entry[split_keys[-1]] = value # Set the dataset name experiment_hyperparams["dataset_name"] = dataset_name # Get the input dim and variance map. Don't worry about variance maps right now, # they aren't used here. input_dim, variance_map_file = input_handler.get_input_params(args, experiment_hyperparams) logger.info("%s\n%s\n" % (input_dim, variance_map_file)) # Set the input dimensionality by the data experiment_hyperparams["nvis"] = input_dim # Set the minimum learning rate relative to the initial learning rate. experiment_hyperparams["min_lr"] = experiment_hyperparams["learning_rate"] / 10 # Make a unique hash for experiments. Remember that lists, dicts, and other data # types may not be hashable, so you may need to do some special processing. In # this case we convert the lists to tuples. h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\ [tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()]))) # Save path for the experiments. In this case we are sharing a directory in my # export directory so IT can blame me. save_path = serial.preprocess("/export/mialab/users/dhjelm/pylearn2_outs/rbm_demo/%d" % h) # We save file params separately as they aren't model specific. file_params = { "save_path": save_path, "variance_map_file": variance_map_file, } state.file_parameters = file_params state.hyper_parameters = experiment_hyperparams user = path.expandvars("$USER") state.created_by = user # Finally we add the experiment to the table. sql.insert_job( experiment.experiment, flatten(state), db ) # A view can be used when querying the database using psql. May not be needed in future. db.createView("%s_view" % args.table)