def extract_output(config, best_epoch):
# load best model
model_file = os.path.join(config.experiment_root, 'epochs',
'epoch{}.pkl'.format(best_epoch))
print 'loading ' + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = 'test'
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), '..', 'run',
'dataset_template.yaml'),
params=config,
)
with log_timing(log, 'processing dataset'):
y_real, y_pred, output = process_dataset(model, dataset)
return y_real, y_pred, output
def train_mlp(params):
train, yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'cross_trial_template.yaml'),
params=params,
);
save_yaml_file(yaml_str, os.path.join(params.experiment_root, 'settings.yaml'));
with log_timing(log, 'training network'):
train.main_loop();
def train_convnet(config):
train, yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'train_convnet_template.yaml'),
params=config,
);
save_yaml_file(yaml_str, os.path.join(config.experiment_root, 'settings.yaml'));
with log_timing(log, 'training network'):
train.main_loop();
def train_mlp(params):
train, yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'cross_trial_template.yaml'),
params=params,
)
save_yaml_file(yaml_str,
os.path.join(params.experiment_root, 'settings.yaml'))
with log_timing(log, 'training network'):
train.main_loop()
def train_convnet(config):
train, yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'train_convnet_template.yaml'),
params=config,
)
save_yaml_file(yaml_str,
os.path.join(config.experiment_root, 'settings.yaml'))
with log_timing(log, 'training network'):
train.main_loop()
def load_datasets_for_subjects(dataset_params, subjects, suffix=''):
datasets = {}
for key, params in dataset_params.items():
if not key in dataset_names:
continue;
params['subjects'] = subjects;
params['name'] = params['name']+suffix;
dataset_config = merge_params(config, params);
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'run', 'dataset_template.yaml'),
params=dataset_config,
);
# log.info('dataset loaded. X={} y={}'.format(dataset.X.shape, dataset.y.shape));
datasets[key+suffix] = dataset;
del dataset, dataset_yaml;
return datasets;
def load_datasets_for_subjects(dataset_params, subjects, suffix=''):
datasets = {}
for key, params in dataset_params.items():
if not key in dataset_names:
continue
params['subjects'] = subjects
params['name'] = params['name'] + suffix
dataset_config = merge_params(config, params)
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'run',
'dataset_template.yaml'),
params=dataset_config,
)
# log.info('dataset loaded. X={} y={}'.format(dataset.X.shape, dataset.y.shape));
datasets[key + suffix] = dataset
del dataset, dataset_yaml
return datasets
def train_mlp(params):
# sda_file = os.path.join(params.experiment_root, 'sda', 'sda_all.pkl');
# check whether pre-trained SDA is there
pretrained = True;
for i in xrange(len(params.hidden_layers_sizes)):
sda_layer_file = params.get(('layer{}_content').format(i));
if not os.path.isfile(sda_layer_file):
log.info('did not find pre-trained SDA layer model at {}. re-computing SDA'.format(sda_layer_file));
pretrained = False;
break;
else:
log.info('found pre-trained SDA layer model at {}'.format(sda_layer_file));
if not pretrained:
train_sda(params);
n_layers = len(params.hidden_layers_sizes);
if params.learning_rule == 'AdaDelta':
yaml_template = 'train_sda_mlp_template.AdaDelta.yaml'
else:
if n_layers == 3:
yaml_template = 'train_sda_mlp_template.Momentum.yaml'
elif n_layers == 2:
yaml_template = 'train_sda_mlp_template.Momentum.2layers.yaml'
else:
raise '{} layers not supported'.format(n_layers);
train, train_yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), yaml_template),
params=params,
);
save_yaml_file(train_yaml_str, os.path.join(params.experiment_root, 'mlp_train.yaml'));
with log_timing(log, 'training MLP'):
train.main_loop();
log.info('done');
def extract_output(config, best_epoch):
# load best model
model_file = os.path.join(config.experiment_root, "epochs", "epoch{}.pkl".format(best_epoch))
print "loading " + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = "test"
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config
)
with log_timing(log, "processing dataset"):
y_real, y_pred, output = process_dataset(model, dataset)
return y_real, y_pred, output
def train_sda(params):
input_trainset, trainset_yaml_str = load_yaml_file(
os.path.join(os.path.dirname(__file__), 'train_sda_dataset_template.yaml'),
params=params,
);
log.info('... building the model');
# build layers
layer_dims = [params.input_length];
layer_dims.extend(params.hidden_layers_sizes);
layers = [];
for i in xrange(1, len(layer_dims)):
layer_params = {
'name' : 'da'+str(i),
'n_inputs' : layer_dims[i-1],
'n_outputs' : layer_dims[i],
'corruption_level' : params.pretrain.corruption_levels[i-1],
'input_range' : numpy.sqrt(6. / (layer_dims[i-1] + layer_dims[i])),
'random_seed' : params.random_seed,
}
layers.append(load_yaml_file(
os.path.join(os.path.dirname(__file__), 'train_sda_layer_template.yaml'),
params=layer_params,
)[0]);
# unsupervised pre-training
log.info('... pre-training the model');
start_time = time.clock();
for i in xrange(len(layers)):
# reset corruption to make sure input is not corrupted
for layer in layers:
layer.set_corruption_level(0);
if i == 0:
trainset = input_trainset;
elif i == 1:
trainset = TransformerDataset( raw = input_trainset, transformer = layers[0] );
else:
trainset = TransformerDataset( raw = input_trainset, transformer = StackedBlocks( layers[0:i] ));
# set corruption for layer to train
layers[i].set_corruption_level(params.pretrain.corruption_levels[i]);
# FIXME: this is not so nice but we have to do it this way as YAML is not flexible enough
trainer = get_layer_trainer_sgd_autoencoder(
layers[i],
trainset,
learning_rate = params.pretrain.learning_rate,
max_epochs = params.pretrain.epochs,
batch_size = params.pretrain.batch_size,
name='pre-train'+str(i));
log.info('unsupervised training layer %d, %s '%(i, layers[i].__class__));
trainer.main_loop();
# theano.printing.pydotprint_variables(
# layer_trainer.algorithm.sgd_update.maker.fgraph.outputs[0],
# outfile='pylearn2-sgd_update.png',
# var_with_name_simple=True);
end_time = time.clock();
log.info('pre-training code ran for {0:.2f}m'.format((end_time - start_time) / 60.));
if params.untie_weights:
# now untie the decoder weights
log.info('untying decoder weights');
for layer in layers:
layer.untie_weights();
# construct multi-layer training functions
# unsupervised training
log.info('... training the model');
sdae = None;
for depth in xrange(1, len(layers)+1):
first_layer_i = len(layers)-depth;
log.debug('training layers {}..{}'.format(first_layer_i,len(layers)-1));
group = layers[first_layer_i:len(layers)];
# log.debug(group);
# reset corruption
for layer in layers:
layer.set_corruption_level(0);
if first_layer_i == 0:
trainset = input_trainset;
elif first_layer_i == 1:
trainset = TransformerDataset( raw = input_trainset, transformer = layers[0] );
else:
trainset = TransformerDataset( raw = input_trainset, transformer = StackedBlocks( layers[0:first_layer_i] ));
# set corruption for input layer of stack to train
# layers[first_layer_i].set_corruption_level(stage2_corruption_levels[first_layer_i]);
corruptor = LoggingCorruptor(
BinomialCorruptor(
corruption_level=params.pretrain_finetune.corruption_levels[first_layer_i]),
name='depth {}'.format(depth));
sdae = StackedDenoisingAutoencoder(group, corruptor);
trainer = get_layer_trainer_sgd_autoencoder(
sdae,
trainset,
learning_rate = params.pretrain_finetune.learning_rate,
max_epochs = params.pretrain_finetune.epochs,
batch_size = params.pretrain_finetune.batch_size,
name='multi-train'+str(depth)
);
log.info('unsupervised multi-layer training %d'%(i));
trainer.main_loop()
end_time = time.clock()
log.info('full training code ran for {0:.2f}m'.format((end_time - start_time) / 60.));
# save the model
model_file = os.path.join(params.experiment_root, 'sda', 'sda_all.pkl');
with log_timing(log, 'saving SDA model to {}'.format(model_file)):
serial.save(model_file, sdae);
if params.untie_weights:
# save individual layers for later (with untied weights)
for i, layer in enumerate(sdae.autoencoders):
layer_file = os.path.join(params.experiment_root, 'sda', 'sda_layer{}_untied.pkl'.format(i));
with log_timing(log, 'saving SDA layer {} model to {}'.format(i, layer_file)):
serial.save(layer_file, layer);
# save individual layers for later (with tied weights)
for i, layer in enumerate(sdae.autoencoders):
if params.untie_weights:
layer.tie_weights();
layer_file = os.path.join(params.experiment_root, 'sda', 'sda_layer{}_tied.pkl'.format(i));
with log_timing(log, 'saving SDA layer {} model to {}'.format(i, layer_file)):
serial.save(layer_file, layer);
log.info('done');
return sdae;
def extract_cube(experiment_path, best_epoch, config):
# load best model
model_file = os.path.join(experiment_path, 'epochs',
'epoch{}.pkl'.format(best_epoch))
print 'loading ' + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = 'test'
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), '..', 'run',
'dataset_template.yaml'),
params=config,
)
with log_timing(log, 'processing dataset'):
y_real, y_pred, output = process_dataset(model, dataset)
print classification_report(y_real, y_pred)
print confusion_matrix(y_real, y_pred)
misclass = np.not_equal(y_real, y_pred).astype(int)
print misclass.mean()
print '----- sequence aggregration -----'
s_real, s_pred, s_predf, s_predp = aggregate_classification(
dataset.sequence_partitions, y_real, y_pred, output)
print classification_report(s_real, s_predf)
print confusion_matrix(s_real, s_predf)
print(s_real != s_predf).mean()
print '----- channel aggregration -----'
t_real, t_pred, t_predf, t_predp = aggregate_classification(
dataset.trial_partitions, y_real, y_pred, output)
print classification_report(t_real, t_predf)
print confusion_matrix(t_real, t_predf)
print(t_real != t_predf).mean()
cube = DataCube()
cube.add(dataset.metadata, misclass)
for cat, entry in cube.store.items():
print cat
for key, values in cube.store[cat].items():
print '{:>30} : {:.3f}'.format(key, np.mean(values))
print np.mean(cube.get_entries())
header = ' | '
for c in xrange(18):
header += ' {:2} '.format(c)
header += ' avg '
print header
for r in xrange(48):
line = '{:>3} | '.format(r)
for c in xrange(18):
line += ' ' + cube.get_entries_mean_str(channels=[c], stimuli=[r])
line += ' ' + cube.get_entries_mean_str(stimuli=[r])
# average over all channels
print line
print
line = '{:>3} | '.format('avg')
for c in xrange(18):
line += ' ' + cube.get_entries_mean_str(channels=[c])
# average over all stimuli
line += ' ' + cube.get_entries_mean_str()
# average over all stimuli and channels
print line
return cube
def extract_cube(experiment_path, best_epoch, config):
# load best model
model_file = os.path.join(experiment_path, "epochs", "epoch{}.pkl".format(best_epoch))
print "loading " + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = "test"
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config
)
with log_timing(log, "processing dataset"):
y_real, y_pred, output = process_dataset(model, dataset)
print classification_report(y_real, y_pred)
print confusion_matrix(y_real, y_pred)
misclass = np.not_equal(y_real, y_pred).astype(int)
print misclass.mean()
print "----- sequence aggregration -----"
s_real, s_pred, s_predf, s_predp = aggregate_classification(dataset.sequence_partitions, y_real, y_pred, output)
print classification_report(s_real, s_predf)
print confusion_matrix(s_real, s_predf)
print (s_real != s_predf).mean()
print "----- channel aggregration -----"
t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output)
print classification_report(t_real, t_predf)
print confusion_matrix(t_real, t_predf)
print (t_real != t_predf).mean()
cube = DataCube()
cube.add(dataset.metadata, misclass)
for cat, entry in cube.store.items():
print cat
for key, values in cube.store[cat].items():
print "{:>30} : {:.3f}".format(key, np.mean(values))
print np.mean(cube.get_entries())
header = " | "
for c in xrange(18):
header += " {:2} ".format(c)
header += " avg "
print header
for r in xrange(48):
line = "{:>3} | ".format(r)
for c in xrange(18):
line += " " + cube.get_entries_mean_str(channels=[c], stimuli=[r])
line += " " + cube.get_entries_mean_str(stimuli=[r])
# average over all channels
print line
print
line = "{:>3} | ".format("avg")
for c in xrange(18):
line += " " + cube.get_entries_mean_str(channels=[c])
# average over all stimuli
line += " " + cube.get_entries_mean_str()
# average over all stimuli and channels
print line
return cube
