def train_nice(args):
vn = True
center = True
logger.info("Getting dataset info for %s" % args.dataset_name)
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name)
if args.transposed:
logger.info("Data in transpose...")
mri = MRI.MRI_Transposed(dataset_name=args.dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
input_dim = mri.X.shape[1]
variance_map_file = path.join(data_path, "transposed_variance_map.npy")
else:
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=args.dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
variance_map_file = path.join(data_path, "variance_map.npy")
save_variance_map(mri, variance_map_file)
logger.info("Input shape: %d" % input_dim)
p = path.abspath(path.dirname(__file__))
yaml_file = path.join(p, "nice_mri.yaml")
user = path.expandvars("$USER")
if args.out_name is not None:
out_name = args.out_name
else:
out_name = args.dataset_name
save_file = "nice_%s%s%s" % (out_name, "_transposed" if args.transposed
else "", "_logistic" if args.logistic else "")
save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" %
(user, save_file))
if path.isfile(save_path + ".pkl") or path.isfile(save_path + "_best.pkl"):
answer = None
while answer not in ["Y", "N", "y", "n"]:
answer = raw_input("%s already exists, continuing will overwrite."
"\nOverwrite? (Y/N)[N]: " % save_path) or "N"
if answer not in ["Y", "N", "y", "n"]:
print "Please answer Y or N"
if answer in ["N", "n"]:
print "If you want to run without overwrite, consider using the -o option."
sys.exit()
logger.info("Saving to prefix %s" % save_path)
if not path.isfile(variance_map_file):
raise ValueError("Variance map file %s not found." % variance_map_file)
train(yaml_file, save_path, input_dim, args.transposed, args.logistic,
variance_map_file, args.dataset_name)
def get_input_params(self, args, hyperparams):
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" +
args.dataset_name)
data_class = hyperparams["data_class"]
variance_normalize = hyperparams.get("variance_normalize", False)
unit_normalize = hyperparams.get("unit_normalize", False)
demean = hyperparams.get("demean", False)
assert not (variance_normalize and unit_normalize)
logger.info((data_class, variance_normalize, unit_normalize, demean))
h = hash((data_class, variance_normalize, unit_normalize, demean))
if self.d.get(h, False):
return self.d[h]
else:
if data_class == "MRI_Transposed":
assert not variance_normalize
mri = MRI.MRI_Transposed(dataset_name=args.dataset_name,
unit_normalize=unit_normalize,
demean=demean,
even_input=True,
apply_mask=True)
input_dim = mri.X.shape[1]
variance_file_name = (
"variance_map_transposed%s%s.npy" %
("_un" if unit_normalize else "", "_dm" if demean else ""))
elif data_class == "MRI_Standard":
assert not demean
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=args.dataset_name,
unit_normalize=unit_normalize,
variance_normalize=variance_normalize,
even_input=True,
apply_mask=True)
variance_file_name = ("variance_map%s%s.npy" %
("_un" if unit_normalize else "",
"_vn" if variance_normalize else ""))
logger.info(variance_file_name)
logger.info(
(data_class, variance_normalize, unit_normalize, demean))
variance_map_file = path.join(data_path, variance_file_name)
if not path.isfile(variance_map_file):
logger.info("Saving variance file %s" % variance_map_file)
mri_nifti.save_variance_map(mri, variance_map_file)
self.d[h] = (input_dim, variance_map_file)
return self.d[h]
def train_nice(args):
vn = True
center = True
if args.transposed:
fmri = MRI.MRI_Transposed(dataset_name=args.dataset_name,
even_input=True)
input_dim = fmri.X.shape[1]
del fmri
else:
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" +
args.dataset_name)
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
logging.info("Input shape: %d" % input_dim)
p = path.abspath(path.dirname(__file__))
yaml_file = path.join(p, "nice_%s.yaml" % args.dataset_name)
user = path.expandvars("$USER")
save_file = "nice_%s%s%s" % (args.dataset_name,
"_transposed" if args.transposed else "",
"_logistic" if args.logistic else "")
save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s" %
(user, save_file))
variance_map_file = path.join(data_path, "variance_map.npy")
if not path.isfile(variance_map_file):
raise ValueError("Variance map file %s not found." % variance_map_file)
train(yaml_file, save_path, input_dim, args.transposed, args.logistic,
variance_map_file)
def main(dataset_name="smri"):
logger.info("Getting dataset info for %s" % args.dataset_name)
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name)
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=args.dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
variance_map_file = path.join(data_path, "variance_map.npy")
mri_nifti.save_variance_map(mri, variance_map_file)
user = path.expandvars("$USER")
save_path = serial.preprocess("/export/mialab/users/%s/pylearn2_outs/%s"
% (user, "jobman_test"))
file_params = {"save_path": save_path,
"variance_map_file": variance_map_file
}
yaml_template = open(yaml_file).read()
hyperparams = expand(flatten(mlp_experiment.default_hyperparams(input_dim=input_dim)),
dict_type=ydict)
for param in hyperparams:
if hasattr(args, param) and getattr(args, param):
val = getattr(args, param)
logger.info("Filling %s with %r" % (param, val))
hyperparams[param] = type(hyperparams[param])(val)
for param in file_params:
yaml_template = yaml_template.replace("%%(%s)s" % param, file_params[param])
yaml = yaml_template % hyperparams
print yaml
logger.info("Training")
train = yaml_parse.load(yaml)
train.main_loop()
def load_experiments(args):
dataset_name = args.dataset_name
db = sql.db(
"postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" %
{
"user": args.user,
"host": args.host,
"port": args.port,
"database": args.database,
"table": args.table,
})
logger.info("Getting dataset info for %s" % dataset_name)
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name)
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
variance_map_file = path.join(data_path, "variance_map.npy")
mri_nifti.save_variance_map(mri, variance_map_file)
for items in jg.nested_generator(
jg.hidden_generator("nhid1", 1),
jg.hidden_generator("nhid2", 1),
):
state = DD()
experiment_hyperparams = mlp_experiment.default_hyperparams(input_dim)
for key, value in items:
split_keys = key.split(".")
entry = experiment_hyperparams
for k in split_keys[:-1]:
entry = entry[k]
entry[split_keys[-1]] = value
experiment_hyperparams["dataset_name"] = dataset_name
h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\
flatten(experiment_hyperparams).values())))
user = path.expandvars("$USER")
save_path = serial.preprocess(
"/export/mialab/users/%s/pylearn2_outs/%d" % (user, h))
file_params = {
"save_path": save_path,
"variance_map_file": variance_map_file,
}
state.file_parameters = file_params
state.hyper_parameters = experiment_hyperparams
state.pid = 0
sql.insert_job(mlp_experiment.experiment, flatten(state), db)
db.createView("%s_view" % args.table)
def get_input_params(self, hyperparams):
"""
Get the input parameters given data hyperparameters.
Parameters
----------
hyperparams: dict
Hyperparameters.
Returns
-------
input_dim, variance_map_file: int, str
Input dimensionality and the location of the variance map file.
"""
dataset_name = hyperparams["dataset_name"]
data_class = hyperparams["data_class"]
variance_normalize = hyperparams.get("variance_normalize", False)
unit_normalize = hyperparams.get("unit_normalize", False)
demean = hyperparams.get("demean", False)
assert not (variance_normalize and unit_normalize)
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + dataset_name)
h = hash((data_class, variance_normalize, unit_normalize, demean))
if self.d.get(h, False):
return self.d[h]
else:
if data_class == "MRI_Transposed":
assert not variance_normalize
mri = MRI.MRI_Transposed(dataset_name=dataset_name,
unit_normalize=unit_normalize,
demean=demean,
even_input=True,
apply_mask=True)
input_dim = mri.X.shape[1]
variance_file_name = (
"variance_map_transposed%s%s.npy" %
("_un" if unit_normalize else "", "_dm" if demean else ""))
elif data_class == "MRI_Standard":
assert not demean
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=dataset_name,
unit_normalize=unit_normalize,
variance_normalize=variance_normalize,
even_input=True,
apply_mask=True)
variance_file_name = ("variance_map%s%s.npy" %
("_un" if unit_normalize else "",
"_vn" if variance_normalize else ""))
variance_map_file = path.join(data_path, variance_file_name)
if not path.isfile(variance_map_file):
mri_nifti.save_variance_map(mri, variance_map_file)
self.d[h] = (input_dim, variance_map_file)
return self.d[h]
def get_input_params(self,
hyperparams,
dataset_root="${PYLEARN2_NI_PATH}"):
"""
Get the input parameters given data hyperparameters.
Parameters
----------
hyperparams: dict
Hyperparameters.
Returns
-------
input_dim, variance_map_file: int, str
Input dimensionality and the location of the variance map file.
"""
assert "dataset_name" in hyperparams.keys(), hyperparams
dataset_name = hyperparams["dataset_name"]
data_class = hyperparams["data_class"]
variance_normalize = hyperparams.get("variance_normalize", False)
unit_normalize = hyperparams.get("unit_normalize", False)
demean = hyperparams.get("demean", False)
even_input = False # Fix this
assert not ((variance_normalize == 1) and unit_normalize), (
"Cannot variance normalize and unit normalize in the same direction"
)
data_path = serial.preprocess(dataset_root + dataset_name)
h = hash((data_class, variance_normalize, unit_normalize, demean))
if self.d.get(h, False):
return self.d[h]
else:
if data_class == "MRI_Transposed":
mri = MRI.MRI_Transposed(dataset_name=dataset_name,
unit_normalize=unit_normalize,
variance_normalize=variance_normalize,
demean=demean,
even_input=even_input,
apply_mask=True,
dataset_root=dataset_root)
input_dim = mri.X.shape[1]
variance_file_name = (
"variance_map_transposed%s%s.npy" %
("_un" if unit_normalize else "", "_dm" if demean else ""))
elif data_class == "MRI_Standard":
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
#if input_dim % 2 == 1:
# input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=dataset_name,
unit_normalize=unit_normalize,
demean=demean,
variance_normalize=variance_normalize,
even_input=even_input,
apply_mask=True,
dataset_root=dataset_root)
variance_file_name = ("variance_map%s%s%s.npy" %
("_un" if unit_normalize else "",
"_vn" if variance_normalize else "",
"_dm" if demean else ""))
else:
raise ValueError("Unknown data class: %s" % data_class)
variance_map_file = path.join(data_path, variance_file_name)
if not path.isfile(variance_map_file):
mri_utils.save_variance_map(mri, variance_map_file)
self.d[h] = (input_dim, variance_map_file)
return self.d[h]
def load_experiments(args):
dataset_name = args.dataset_name
db = sql.db(
"postgres://%(user)s@%(host)s:%(port)d/%(database)s?table=%(table)s" %
{
"user": args.user,
"host": args.host,
"port": args.port,
"database": args.database,
"table": args.table,
})
logger.info("Getting dataset info for %s%s" %
(dataset_name, ", transposed" if args.transposed else ""))
data_path = serial.preprocess("${PYLEARN2_NI_PATH}/" + args.dataset_name)
if args.transposed:
logger.info("Data in transpose...")
mri = MRI.MRI_Transposed(dataset_name=args.dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
input_dim = mri.X.shape[1]
variance_map_file = path.join(data_path, "transposed_variance_map.npy")
else:
mask_file = path.join(data_path, "mask.npy")
mask = np.load(mask_file)
input_dim = (mask == 1).sum()
if input_dim % 2 == 1:
input_dim -= 1
mri = MRI.MRI_Standard(which_set="full",
dataset_name=args.dataset_name,
unit_normalize=True,
even_input=True,
apply_mask=True)
variance_map_file = path.join(data_path, "variance_map.npy")
mri_nifti.save_variance_map(mri, variance_map_file)
for items in nested_generator(
layer_depth_generator("encoder.layer_depths", xrange(4, 6), 5),
hidden_generator("encoder.nhid", 4),
float_generator("weight_decay.coeffs.z",
3,
0.1,
0.001,
log_scale=True)):
# logger.info("Adding NICE experiment with hyperparameters %s" % (items, ))
state = DD()
experiment_hyperparams = nice_experiment.default_hyperparams(input_dim)
if args.transposed:
experiment_hyperparams["data_class"] = "MRI_Transposed"
if args.logistic:
experiment_hyperparams["prior"]["__builder__"] =\
"nice.pylearn2.models.nice.StandardLogistic"
for key, value in items:
split_keys = key.split(".")
entry = experiment_hyperparams
for k in split_keys[:-1]:
entry = entry[k]
entry[split_keys[-1]] = value
experiment_hyperparams["dataset_name"] = dataset_name
h = abs(hash(frozenset(flatten(experiment_hyperparams).keys() +\
[tuple(v) if isinstance(v, list) else v for v in flatten(experiment_hyperparams).values()])))
user = path.expandvars("$USER")
save_path = serial.preprocess(
"/export/mialab/users/%s/pylearn2_outs/%d" % (user, h))
file_params = {
"save_path": save_path,
"variance_map_file": variance_map_file,
}
state.file_parameters = file_params
state.hyper_parameters = experiment_hyperparams
sql.insert_job(nice_experiment.experiment, flatten(state), db)
db.createView("%s_view" % args.table)