def get_layers(self):
# make into list if nonlin only one
if not hasattr(self.nonlin_before_merge, '__len__'):
nonlins_before_merge = ((self.nonlin_before_merge,) *
len(self.networks))
else:
nonlins_before_merge = self.nonlin_before_merge
layers_per_net = [net.get_layers() for net in self.networks]
# Check that all have same number of sample preds
n_sample_preds = get_n_sample_preds(layers_per_net[0][-1])
for layers in layers_per_net:
assert get_n_sample_preds(layers[-1]) == n_sample_preds
# remove dense softmax replace by dense linear
reduced_layers = [replace_dense_softmax_by_dense_linear(all_l, n_f,
nonlin_before_merge=nonlin,
batch_norm_before_merge=self.batch_norm_before_merge)
for all_l, n_f, nonlin in zip(layers_per_net, self.n_features_per_net,
nonlins_before_merge)]
# hopefully still works with new method below:)
use_same_input_layer(reduced_layers)
final_layers = [layers[-1] for layers in reduced_layers]
l_merged = ConcatLayer(final_layers)
l_merged = DenseLayer(l_merged,num_units=self.n_classes,
nonlinearity=softmax)
return lasagne.layers.get_all_layers(l_merged)
def get_layers(self):
# make into list if nonlin only one
if not hasattr(self.nonlin_before_merge, '__len__'):
nonlins_before_merge = ((self.nonlin_before_merge,) *
len(self.networks))
else:
nonlins_before_merge = self.nonlin_before_merge
layers_per_net = [net.get_layers() for net in self.networks]
# Check that all have same number of sample preds
n_sample_preds = get_n_sample_preds(layers_per_net[0][-1])
for layers in layers_per_net:
assert get_n_sample_preds(layers[-1]) == n_sample_preds
# remove dense softmax replace by dense linear
reduced_layers = [replace_dense_softmax_by_dense_linear(all_l, n_f,
nonlin_before_merge=nonlin,
batch_norm_before_merge=self.batch_norm_before_merge)
for all_l, n_f, nonlin in zip(layers_per_net, self.n_features_per_net,
nonlins_before_merge)]
# hopefully still works with new method below:)
use_same_input_layer(reduced_layers)
final_layers = [layers[-1] for layers in reduced_layers]
l_merged = ConcatLayer(final_layers)
l_merged = DenseLayer(l_merged,num_units=self.n_classes,
nonlinearity=softmax)
return lasagne.layers.get_all_layers(l_merged)
def tied_neighbours_cnt_model_logdomain(preds, targets, final_layer):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
# just for checkup
# TODOREMOVE!!
#return categorical_crossentropy_logdomain(preds, targets)
return tied_neighbours_logdomain(preds, n_sample_preds, n_classes)
def tied_neighbours_cnt_model_logdomain(preds, targets, final_layer):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
# just for checkup
# TODOREMOVE!!
#return categorical_crossentropy_logdomain(preds, targets)
return tied_neighbours_logdomain(preds, n_sample_preds, n_classes)
def initialize(self):
""" Initialize data containers and theano functions for training."""
self.rng = RandomState(30948348)
self.data_batches = []
self.y_batches = []
self.input_time_length = get_input_time_length(self.cnt_model)
self.n_sample_preds = get_n_sample_preds(self.cnt_model)
self.n_classes = self.cnt_model.output_shape[1]
# create train function
log.info("Compile train function...")
self._create_train_function()
log.info("Done compiling train function.")
def create_trials_and_do_fft(exp):
train_set = exp.dataset_provider.get_train_merged_valid_test(
exp.dataset)['train']
n_sample_preds = get_n_sample_preds(exp.final_layer)
trials, targets = get_trials_targets(train_set, n_sample_preds,
exp.iterator)
# apparently trials x batch size x chan x time x 1?
ffted = np.fft.rfft(trials, axis=3)
amplitudes = np.abs(ffted)
phases = np.angle(ffted)
# targets are trials x samples x classes
return trials, amplitudes, phases, targets
def initialize(self):
""" Initialize data containers and theano functions for training."""
self.rng = RandomState(30948348)
self.data_batches = []
self.y_batches = []
self.input_time_length = get_input_time_length(self.cnt_model)
self.n_sample_preds = get_n_sample_preds(self.cnt_model)
self.n_classes = self.cnt_model.output_shape[1]
# create train function
log.info("Compile train function...")
self._create_train_function()
log.info("Done compiling train function.")
def compute_trial_acts(model, i_layer, iterator, train_set):
"""Compute activations per trial per sample for given layer of the model.
Parameters
----------
model: Lasagne layer
Final layer of the model.
i_layer: int
Index of layer to compute activations for.
iterator: DatasetIterator
Iterator to get batches from.
train_set: Dataset (Cnt)
Dataset to use.
Returns
-------
trial_acts: 3darray of float
Activations per trial per sample. #trialx#channelx#sample
"""
# compute number of inputs per trial
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y,
check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
# +1 since trial ends is inclusive
n_trial_len = i_trial_ends[0] - i_trial_starts[0] + 1
n_inputs_per_trial = int(
np.ceil(n_trial_len / float(iterator.n_sample_preds)))
log.info("Create theano function...")
all_layers = lasagne.layers.get_all_layers(model)
all_out_fn = create_pred_fn(all_layers[i_layer])
assert (iterator.input_time_length == get_input_time_length(model))
assert (iterator.n_sample_preds == get_n_sample_preds(model))
log.info("Compute activations...")
all_outs_per_batch = [
all_out_fn(batch[0])
for batch in iterator.get_batches(train_set, False)
]
batch_sizes = [
len(batch[0]) for batch in iterator.get_batches(train_set, False)
]
all_outs_per_batch = np.array(all_outs_per_batch)
n_trials = len(i_trial_starts)
log.info("Transform to trial activations...")
trial_acts = get_trial_acts(all_outs_per_batch,
batch_sizes,
n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=iterator.n_sample_preds)
log.info("Done.")
return trial_acts
def get_y_for_subject(pred_fn, test_set_0, test_set_1, iterator, final_layer):
"""Assumes there was no resampling!!"""
batch_gen_0 = iterator.get_batches(test_set_0, shuffle=False)
all_preds_0 = [pred_fn(batch[0]) for batch in batch_gen_0]
batch_gen_1 = iterator.get_batches(test_set_1, shuffle=False)
all_preds_1 = [pred_fn(batch[0]) for batch in batch_gen_1]
n_sample_preds = get_n_sample_preds(final_layer)
input_time_length = lasagne.layers.get_all_layers(final_layer)[0].shape[2]
n_samples_0 = test_set_0.get_topological_view().shape[0]
preds_arr_0 = get_reshaped_cnt_preds(all_preds_0, n_samples_0,
input_time_length, n_sample_preds)
n_samples_1 = test_set_1.get_topological_view().shape[0]
preds_arr_1 = get_reshaped_cnt_preds(all_preds_1, n_samples_1,
input_time_length, n_sample_preds)
series_preds = [preds_arr_0, preds_arr_1]
return series_preds
def get_y_for_subject(pred_fn, test_set_0, test_set_1, iterator, final_layer):
"""Assumes there was no resampling!!"""
batch_gen_0 = iterator.get_batches(test_set_0, shuffle=False)
all_preds_0 = [pred_fn(batch[0]) for batch in batch_gen_0]
batch_gen_1 = iterator.get_batches(test_set_1, shuffle=False)
all_preds_1 = [pred_fn(batch[0]) for batch in batch_gen_1]
n_sample_preds = get_n_sample_preds(final_layer)
input_time_length = lasagne.layers.get_all_layers(final_layer)[0].shape[2]
n_samples_0 = test_set_0.get_topological_view().shape[0]
preds_arr_0 = get_reshaped_cnt_preds(all_preds_0, n_samples_0,
input_time_length, n_sample_preds)
n_samples_1 = test_set_1.get_topological_view().shape[0]
preds_arr_1 = get_reshaped_cnt_preds(all_preds_1, n_samples_1,
input_time_length, n_sample_preds)
series_preds = [preds_arr_0, preds_arr_1]
return series_preds
def compute_trial_acts(model, i_layer, iterator, train_set):
"""Compute activations per trial per sample for given layer of the model.
Parameters
----------
model: Lasagne layer
Final layer of the model.
i_layer: int
Index of layer to compute activations for.
iterator: DatasetIterator
Iterator to get batches from.
train_set: Dataset (Cnt)
Dataset to use.
Returns
-------
trial_acts: 3darray of float
Activations per trial per sample. #trialx#channelx#sample
"""
# compute number of inputs per trial
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y, check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
# +1 since trial ends is inclusive
n_trial_len = i_trial_ends[0] - i_trial_starts[0] + 1
n_inputs_per_trial = int(np.ceil(n_trial_len / float(iterator.n_sample_preds)))
log.info("Create theano function...")
all_layers = lasagne.layers.get_all_layers(model)
all_out_fn = create_pred_fn(all_layers[i_layer])
assert(iterator.input_time_length == get_input_time_length(model))
assert(iterator.n_sample_preds == get_n_sample_preds(model))
log.info("Compute activations...")
all_outs_per_batch = [all_out_fn(batch[0])
for batch in iterator.get_batches(train_set, False)]
batch_sizes = [len(batch[0]) for batch in iterator.get_batches(train_set, False)]
all_outs_per_batch = np.array(all_outs_per_batch)
n_trials = len(i_trial_starts)
log.info("Transform to trial activations...")
trial_acts = get_trial_acts(all_outs_per_batch,
batch_sizes, n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=iterator.n_sample_preds)
log.info("Done.")
return trial_acts
def tied_neighbours_cnt_model_masked(preds, targets, final_layer):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
eps = 1e-8
# preds = T.clip(preds, eps, 1-eps)
preds_per_trial_row = preds.reshape((-1, n_sample_preds, n_classes))
earlier_neighbours = preds_per_trial_row[:, :-1]
later_neighbours = preds_per_trial_row[:, 1:]
earlier_neighbours = (T.gt(earlier_neighbours, eps) * earlier_neighbours +
T.le(earlier_neighbours, eps) * earlier_neighbours +
eps)
loss = categorical_crossentropy(earlier_neighbours, later_neighbours)
# now mask out loss where there is no active target
targets_per_trial_row = targets.reshape((-1, n_sample_preds, n_classes))
# :-1] assumies empty targets are at the start not at end
loss = loss * T.gt(T.sum(targets_per_trial_row, axis=2), 0)[:, :-1]
return loss
def transform_to_meaned_trial_env(env, model, i_layer, train_set,
n_inputs_per_trial):
all_layers = lasagne.layers.get_all_layers(model)
layer = all_layers[i_layer]
field_size = get_receptive_field_size(layer)
trial_starts, trial_ends = compute_trial_start_end_samples(train_set.y)
assert len(np.unique(trial_ends - trial_starts)) == 1
n_trials = len(trial_starts)
# +1 as trial end is inclusive
n_trial_len = np.unique(trial_ends - trial_starts)[0] + 1
# Afterwards env is list empty lists(!!)
n_sample_preds = get_n_sample_preds(model)
trial_env = get_meaned_trial_env(env, field_size=field_size,
n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=n_sample_preds)
return trial_env
def unit_output_class_corrs(model, iterator, train_set, i_layer):
# only need targets, ignore trials
# always get targets as from final layer
this_final_layer = lasagne.layers.get_all_layers(model)[-1]
_, targets = get_trials_targets(train_set,
get_n_sample_preds(this_final_layer), iterator)
trial_acts = compute_trial_acts(this_final_layer, i_layer, iterator,
train_set)
# only take those targets where we have predictions for
# a bit hacky: we know targets are same for each trial, so we just
# take last ones, eventhough they come form overlapping batches
# targets are #trials x #samples x #classes
unmeaned_targets = targets - np.mean(targets, axis=(1), keepdims=True)
assert np.all(unmeaned_targets == 0), ("Each trial should only have one "
"unique label")
relevant_targets = targets[:,:trial_acts.shape[2]]
unit_class_corrs = wrap_reshape_topo(corr, trial_acts, relevant_targets,
axis_a=(0,2), axis_b=(0,1))
return unit_class_corrs
def create_experiment(yaml_filename, seed=9859295):
"""Utility function to create experiment from yaml file"""
# for reproducibility for layer weights
# should be same seed as in experiment_runner.py
lasagne.random.set_rng(RandomState(seed))
train_dict = yaml_parse.load(open(yaml_filename, 'r'))
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
dataset = train_dict['dataset']
splitter = train_dict['dataset_splitter']
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
# for backwards compatibility input time length also
input_time_length = get_input_time_length(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
if hasattr(monitor, 'input_time_length'):
monitor.input_time_length = input_time_length
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# add early stop chan, encessary for backwards compatibility
exp_args = train_dict['exp_args']
exp_args['remember_best_chan'] = train_dict['exp_args'].pop(
'remember_best_chan', 'valid_misclass')
exp_args['run_after_early_stop'] = train_dict['exp_args'].pop(
'run_after_early_stop', True)
exp = Experiment(final_layer, dataset, splitter, **exp_args)
assert len(np.setdiff1d(
layers, lasagne.layers.get_all_layers(final_layer))) == 0, (
"All layers "
"should be used, unused {:s}".format(
str(
np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer)))))
return exp
def create_experiment(yaml_filename, seed=9859295):
"""Utility function to create experiment from yaml file"""
# for reproducibility for layer weights
# should be same seed as in experiment_runner.py
lasagne.random.set_rng(RandomState(seed))
train_dict = yaml_parse.load(open(yaml_filename, 'r'))
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
dataset = train_dict['dataset']
splitter = train_dict['dataset_splitter']
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
# for backwards compatibility input time length also
input_time_length = get_input_time_length(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
if hasattr(monitor, 'input_time_length'):
monitor.input_time_length = input_time_length
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# add early stop chan, encessary for backwards compatibility
exp_args = train_dict['exp_args']
exp_args['remember_best_chan'] = train_dict['exp_args'].pop('remember_best_chan',
'valid_misclass')
exp_args['run_after_early_stop'] = train_dict['exp_args'].pop('run_after_early_stop',
True)
exp = Experiment(final_layer, dataset, splitter,
**exp_args)
assert len(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer))) == 0, ("All layers "
"should be used, unused {:s}".format(str(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer)))))
return exp
def unit_output_class_corrs(model, iterator, train_set, i_layer):
# only need targets, ignore trials
# always get targets as from final layer
this_final_layer = lasagne.layers.get_all_layers(model)[-1]
_, targets = get_trials_targets(train_set,
get_n_sample_preds(this_final_layer),
iterator)
trial_acts = compute_trial_acts(this_final_layer, i_layer, iterator,
train_set)
# only take those targets where we have predictions for
# a bit hacky: we know targets are same for each trial, so we just
# take last ones, eventhough they come form overlapping batches
# targets are #trials x #samples x #classes
unmeaned_targets = targets - np.mean(targets, axis=(1), keepdims=True)
assert np.all(unmeaned_targets == 0), ("Each trial should only have one "
"unique label")
relevant_targets = targets[:, :trial_acts.shape[2]]
unit_class_corrs = wrap_reshape_topo(corr,
trial_acts,
relevant_targets,
axis_a=(0, 2),
axis_b=(0, 1))
return unit_class_corrs
def create_submission_csv_for_one_subject(folder_name, kaggle_set, iterator, preprocessor,
final_layer, submission_id):
### Load and preprocess data
kaggle_set.load()
# remember test series lengths before and after resampling to more accurately pad predictions
# later (padding due to the lost samples)
kaggle_set.load_test_data()
test_series_lengths = [len(series) for series in kaggle_set.test_X_series]
kaggle_set.resample_test_data()
test_series_lengths_resampled = [len(series) for series in kaggle_set.test_X_series]
X_train = deepcopy(np.concatenate(kaggle_set.train_X_series)[:,:,np.newaxis,np.newaxis])
X_test_0 = deepcopy(kaggle_set.test_X_series[0][:,:,np.newaxis,np.newaxis])
X_test_1 = deepcopy(kaggle_set.test_X_series[1][:,:,np.newaxis,np.newaxis])
# create dense design matrix sets
train_set = DenseDesignMatrixWrapper(
topo_view=X_train,
y=None, axes=('b','c',0,1))
fake_test_y = np.ones((len(X_test_0), 6))
test_set_0 = DenseDesignMatrixWrapper(
topo_view=X_test_0,
y=fake_test_y)
fake_test_y = np.ones((len(X_test_1), 6))
test_set_1 = DenseDesignMatrixWrapper(
topo_view=X_test_1,
y=fake_test_y)
log.info("Preprocessing data...")
preprocessor.apply(train_set, can_fit=True)
preprocessor.apply(test_set_0, can_fit=False)
preprocessor.apply(test_set_1, can_fit=False)
### Create prediction function and create predictions
log.info("Create prediction functions...")
input_var = lasagne.layers.get_all_layers(final_layer)[0].input_var
predictions = lasagne.layers.get_output(final_layer, deterministic=True)
pred_fn = theano.function([input_var], predictions)
log.info("Make predictions...")
batch_gen_0 = iterator.get_batches(test_set_0, shuffle=False)
all_preds_0 = [pred_fn(batch[0]) for batch in batch_gen_0]
batch_gen_1 = iterator.get_batches(test_set_1, shuffle=False)
all_preds_1 = [pred_fn(batch[0]) for batch in batch_gen_1]
### Pad and reshape predictions
n_sample_preds = get_n_sample_preds(final_layer)
input_time_length = lasagne.layers.get_all_layers(final_layer)[0].shape[2]
n_samples_0 = test_set_0.get_topological_view().shape[0]
preds_arr_0 = get_reshaped_cnt_preds(all_preds_0, n_samples_0,
input_time_length, n_sample_preds)
n_samples_1 = test_set_1.get_topological_view().shape[0]
preds_arr_1 = get_reshaped_cnt_preds(all_preds_1, n_samples_1,
input_time_length, n_sample_preds)
series_preds = [preds_arr_0, preds_arr_1]
assert len(series_preds[0]) == test_series_lengths_resampled[0]
assert len(series_preds[1]) == test_series_lengths_resampled[1]
assert False, ("TODO: here only duplicate if resample half is true for the dataset.. "
"also take care how to create submission cv if trained on all subjects")
series_preds_duplicated = [np.repeat(preds, 2,axis=0) for preds in series_preds]
n_classes = preds_arr_0.shape[1]
# pad missing ones with zeros
missing_0 = test_series_lengths[0] - len(series_preds_duplicated[0])
full_preds_0 = np.append(np.zeros((missing_0, n_classes), dtype=np.float32),
series_preds_duplicated[0], axis=0)
missing_1 = test_series_lengths[1] - len(series_preds_duplicated[1])
full_preds_1 = np.append(np.zeros((missing_1, n_classes), dtype=np.float32),
series_preds_duplicated[1], axis=0)
assert len(full_preds_0) == test_series_lengths[0]
assert len(full_preds_1) == test_series_lengths[1]
full_series_preds = [full_preds_0, full_preds_1]
assert sum([len(a) for a in full_series_preds]) == np.sum(test_series_lengths)
### Create csv
log.info("Create csv...")
csv_filename = "{:02d}".format(submission_id) + '.csv'
csv_filename = os.path.join(folder_name, csv_filename)
cols = ['HandStart','FirstDigitTouch',
'BothStartLoadPhase','LiftOff',
'Replace','BothReleased']
# collect ids
all_ids = []
all_preds = []
for i_series in (9,10):
id_prefix = "subj{:d}_series{:d}_".format(kaggle_set.i_subject, i_series)
this_preds = full_series_preds[i_series-9] # respect offsets
all_preds.extend(this_preds)
this_ids = [id_prefix + str(i_sample) for i_sample in range(this_preds.shape[0])]
all_ids.extend(this_ids)
all_ids = np.array(all_ids)
all_preds = np.array(all_preds)
submission = pd.DataFrame(index=all_ids,
columns=cols,
data=all_preds)
submission.to_csv(csv_filename, index_label='id',float_format='%.3f')
log.info("Done")
def create_submission_csv_for_all_subject_model(folder_name,
all_sub_kaggle_set,
dataset_provider, iterator,
final_layer, submission_id):
all_sub_kaggle_set.load()
assert all_sub_kaggle_set.resample_half == False, ("Not implemented for "
"resample half")
all_sub_kaggle_set.load_test()
# following line will just do the preprocessing already on the train set...
dataset_provider.get_train_merged_valid_test(all_sub_kaggle_set)
test_sets_per_subj = []
for i_subject in range(12):
kaggle_set = all_sub_kaggle_set.kaggle_sets[i_subject]
this_sets = []
for i_test_series in range(2):
# Get input
X_test = kaggle_set.test_X_series[i_test_series][:, :, np.newaxis,
np.newaxis]
fake_test_y = np.ones((len(X_test), 6))
test_set = DenseDesignMatrixWrapper(topo_view=X_test,
y=fake_test_y)
if dataset_provider.preprocessor is not None:
dataset_provider.preprocessor.apply(test_set, can_fit=False)
this_sets.append(test_set)
assert len(this_sets) == 2
test_sets_per_subj.append(this_sets)
### Create prediction function and create predictions
log.info("Create prediction functions...")
input_var = lasagne.layers.get_all_layers(final_layer)[0].input_var
predictions = lasagne.layers.get_output(final_layer, deterministic=True)
pred_fn = theano.function([input_var], predictions)
log.info("Setup iterator...")
n_sample_preds = get_n_sample_preds(final_layer)
iterator.n_sample_preds = n_sample_preds
log.info("Make predictions...")
preds_per_subject = []
for i_subject in range(12):
log.info("Predictions for Subject {:d}...".format(i_subject + 1))
test_sets_subj = test_sets_per_subj[i_subject]
preds = get_y_for_subject(pred_fn, test_sets_subj[0],
test_sets_subj[1], iterator, final_layer)
preds_per_subject.append(preds)
log.info("Done")
log.info("Create csv...")
cols = [
'HandStart', 'FirstDigitTouch', 'BothStartLoadPhase', 'LiftOff',
'Replace', 'BothReleased'
]
# collect ids
all_ids = []
all_preds = []
for i_subject in range(12):
pred_subj_per_series = preds_per_subject[i_subject]
for i_series in (9, 10):
id_prefix = "subj{:d}_series{:d}_".format(i_subject + 1, i_series)
this_preds = pred_subj_per_series[i_series - 9] # respect offsets
all_preds.extend(this_preds)
this_ids = [
id_prefix + str(i_sample)
for i_sample in range(this_preds.shape[0])
]
all_ids.extend(this_ids)
all_ids = np.array(all_ids)
all_preds = np.array(all_preds)
assert all_ids.shape == (3144171, )
assert all_preds.shape == (3144171, 6)
submission = pd.DataFrame(index=all_ids, columns=cols, data=all_preds)
csv_output = StringIO.StringIO()
submission.to_csv(csv_output, index_label='id', float_format='%.3f')
csv_str = csv_output.getvalue()
log.info("Create zip...")
zip_file_name = os.path.join(folder_name, "{:d}.zip".format(submission_id))
submission_zip_file = ZipFile(zip_file_name, 'w', ZIP_DEFLATED)
submission_zip_file.writestr("submission.csv", csv_str)
submission_zip_file.close()
log.info("Done")
def create_submission_csv_for_one_subject(folder_name, kaggle_set, iterator,
preprocessor, final_layer,
submission_id):
### Load and preprocess data
kaggle_set.load()
# remember test series lengths before and after resampling to more accurately pad predictions
# later (padding due to the lost samples)
kaggle_set.load_test_data()
test_series_lengths = [len(series) for series in kaggle_set.test_X_series]
kaggle_set.resample_test_data()
test_series_lengths_resampled = [
len(series) for series in kaggle_set.test_X_series
]
X_train = deepcopy(
np.concatenate(kaggle_set.train_X_series)[:, :, np.newaxis,
np.newaxis])
X_test_0 = deepcopy(kaggle_set.test_X_series[0][:, :, np.newaxis,
np.newaxis])
X_test_1 = deepcopy(kaggle_set.test_X_series[1][:, :, np.newaxis,
np.newaxis])
# create dense design matrix sets
train_set = DenseDesignMatrixWrapper(topo_view=X_train,
y=None,
axes=('b', 'c', 0, 1))
fake_test_y = np.ones((len(X_test_0), 6))
test_set_0 = DenseDesignMatrixWrapper(topo_view=X_test_0, y=fake_test_y)
fake_test_y = np.ones((len(X_test_1), 6))
test_set_1 = DenseDesignMatrixWrapper(topo_view=X_test_1, y=fake_test_y)
log.info("Preprocessing data...")
preprocessor.apply(train_set, can_fit=True)
preprocessor.apply(test_set_0, can_fit=False)
preprocessor.apply(test_set_1, can_fit=False)
### Create prediction function and create predictions
log.info("Create prediction functions...")
input_var = lasagne.layers.get_all_layers(final_layer)[0].input_var
predictions = lasagne.layers.get_output(final_layer, deterministic=True)
pred_fn = theano.function([input_var], predictions)
log.info("Make predictions...")
batch_gen_0 = iterator.get_batches(test_set_0, shuffle=False)
all_preds_0 = [pred_fn(batch[0]) for batch in batch_gen_0]
batch_gen_1 = iterator.get_batches(test_set_1, shuffle=False)
all_preds_1 = [pred_fn(batch[0]) for batch in batch_gen_1]
### Pad and reshape predictions
n_sample_preds = get_n_sample_preds(final_layer)
input_time_length = lasagne.layers.get_all_layers(final_layer)[0].shape[2]
n_samples_0 = test_set_0.get_topological_view().shape[0]
preds_arr_0 = get_reshaped_cnt_preds(all_preds_0, n_samples_0,
input_time_length, n_sample_preds)
n_samples_1 = test_set_1.get_topological_view().shape[0]
preds_arr_1 = get_reshaped_cnt_preds(all_preds_1, n_samples_1,
input_time_length, n_sample_preds)
series_preds = [preds_arr_0, preds_arr_1]
assert len(series_preds[0]) == test_series_lengths_resampled[0]
assert len(series_preds[1]) == test_series_lengths_resampled[1]
assert False, (
"TODO: here only duplicate if resample half is true for the dataset.. "
"also take care how to create submission cv if trained on all subjects"
)
series_preds_duplicated = [
np.repeat(preds, 2, axis=0) for preds in series_preds
]
n_classes = preds_arr_0.shape[1]
# pad missing ones with zeros
missing_0 = test_series_lengths[0] - len(series_preds_duplicated[0])
full_preds_0 = np.append(np.zeros((missing_0, n_classes),
dtype=np.float32),
series_preds_duplicated[0],
axis=0)
missing_1 = test_series_lengths[1] - len(series_preds_duplicated[1])
full_preds_1 = np.append(np.zeros((missing_1, n_classes),
dtype=np.float32),
series_preds_duplicated[1],
axis=0)
assert len(full_preds_0) == test_series_lengths[0]
assert len(full_preds_1) == test_series_lengths[1]
full_series_preds = [full_preds_0, full_preds_1]
assert sum([len(a)
for a in full_series_preds]) == np.sum(test_series_lengths)
### Create csv
log.info("Create csv...")
csv_filename = "{:02d}".format(submission_id) + '.csv'
csv_filename = os.path.join(folder_name, csv_filename)
cols = [
'HandStart', 'FirstDigitTouch', 'BothStartLoadPhase', 'LiftOff',
'Replace', 'BothReleased'
]
# collect ids
all_ids = []
all_preds = []
for i_series in (9, 10):
id_prefix = "subj{:d}_series{:d}_".format(kaggle_set.i_subject,
i_series)
this_preds = full_series_preds[i_series - 9] # respect offsets
all_preds.extend(this_preds)
this_ids = [
id_prefix + str(i_sample)
for i_sample in range(this_preds.shape[0])
]
all_ids.extend(this_ids)
all_ids = np.array(all_ids)
all_preds = np.array(all_preds)
submission = pd.DataFrame(index=all_ids, columns=cols, data=all_preds)
submission.to_csv(csv_filename, index_label='id', float_format='%.3f')
log.info("Done")
def run(
ex,
data_folder,
subject_id,
n_chans,
only_return_exp,
):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
load_sensor_names = None
train_filename = 'A{:02d}T.mat'.format(subject_id)
test_filename = 'A{:02d}E.mat'.format(subject_id)
train_filepath = os.path.join(data_folder, train_filename)
test_filepath = os.path.join(data_folder, test_filename)
# trial ivan in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
segment_ival = [1500, 4000]
train_loader = BCICompetition4Set2A(train_filepath,
load_sensor_names=load_sensor_names)
test_loader = BCICompetition4Set2A(test_filepath,
load_sensor_names=load_sensor_names)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}), (bandpass_cnt, {
'low_cut_hz': 0,
'high_cut_hz': 38,
}), (exponential_standardize_cnt, {})]
marker_def = {
'1- Right Hand': [1],
'2 - Left Hand': [2],
'3 - Rest': [3],
'4 - Feet': [4]
}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
from braindecode.mywyrm.clean import MaxAbsCleaner
train_cleaner = MaxAbsCleaner(segment_ival=[0, 4000],
threshold=800,
marker_def=marker_def)
test_cleaner = MaxAbsCleaner(segment_ival=[0, 4000],
threshold=800,
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
in_chans = train_set.get_topological_view().shape[1]
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
num_filters_time = 25
filter_time_length = 10
num_filters_spat = 25
pool_time_length = 3
pool_time_stride = 3
num_filters_2 = 50
filter_length_2 = 10
num_filters_3 = 100
filter_length_3 = 10
num_filters_4 = 200
filter_length_4 = 10
final_dense_length = 2
n_classes = 4
final_nonlin = softmax
first_nonlin = elu
first_pool_mode = 'max'
first_pool_nonlin = identity
later_nonlin = elu
later_pool_mode = 'max'
later_pool_nonlin = identity
drop_in_prob = 0.0
drop_prob = 0.5
batch_norm_alpha = 0.1
double_time_convs = False
split_first_layer = True
batch_norm = True
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
d5net = Deep5Net(in_chans=in_chans,
input_time_length=input_time_length,
num_filters_time=num_filters_time,
filter_time_length=filter_time_length,
num_filters_spat=num_filters_spat,
pool_time_length=pool_time_length,
pool_time_stride=pool_time_stride,
num_filters_2=num_filters_2,
filter_length_2=filter_length_2,
num_filters_3=num_filters_3,
filter_length_3=filter_length_3,
num_filters_4=num_filters_4,
filter_length_4=filter_length_4,
final_dense_length=final_dense_length,
n_classes=n_classes,
final_nonlin=final_nonlin,
first_nonlin=first_nonlin,
first_pool_mode=first_pool_mode,
first_pool_nonlin=first_pool_nonlin,
later_nonlin=later_nonlin,
later_pool_mode=later_pool_mode,
later_pool_nonlin=later_pool_nonlin,
drop_in_prob=drop_in_prob,
drop_prob=drop_prob,
batch_norm_alpha=batch_norm_alpha,
double_time_convs=double_time_convs,
split_first_layer=split_first_layer,
batch_norm=batch_norm)
final_layer = d5net.get_layers()[-1]
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
RuntimeMonitor()
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
stop_criterion = Or([
NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
loss_expression = categorical_crossentropy
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop = True
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
ex.info['runtime'] = run_time
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
def tied_neighbours_cnt_model(preds, targets, final_layer):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
return tied_neighbours(preds, n_sample_preds, n_classes)
def tied_losses_cnt_model(preds, targets, final_layer, n_pairs):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
return tied_losses(preds, n_sample_preds, n_classes, n_pairs)
def _run_experiments_with_string(self, experiment_index, train_str):
assert experiment_index >= self._get_start_id()
assert experiment_index < self._get_stop_id()
lasagne.random.set_rng(RandomState(9859295))
# Save train string now, will be overwritten later after
# input dimensions determined, save now for debug in
# case of crash
if not self._dry_run:
self._save_train_string(train_str, experiment_index)
starttime = time.time()
train_dict = self._load_without_layers(train_str)
log.info("With params...")
if not self._quiet:
pprint(train_dict['original_params'])
if self._dry_run:
# Do not do the loading or training...
# Only go until here to show the train params
return
if self._batch_test:
# TODO: put into function
# load layers, load data with dimensions of the layer
# create experiment with max epochs 2, run
from braindecode.datasets.random import RandomSet
train_str = train_str.replace('in_cols', '1')
train_str = train_str.replace('in_sensors', '32')
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
n_chans = layers[0].shape[1]
n_classes = final_layer.output_shape[1]
n_samples = 500000
# set n sample perds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# make at least batches
n_samples = int(n_sample_preds * 1.5 * 200)
dataset = RandomSet(topo_shape=[n_samples, n_chans, 1, 1],
y_shape=[n_samples, n_classes])
dataset.load()
splitter = FixedTrialSplitter(n_train_trials=int(n_samples*0.8),
valid_set_fraction=0.1)
train_dict['exp_args']['preprocessor'] = None
train_dict['exp_args']['stop_criterion'] = MaxEpochs(1)
train_dict['exp_args']['iterator'].batch_size = 1
# TODO: set stop criterion to max epochs =1
# change batch_size in iterator
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run_until_early_stop()
datasets = exp.dataset_provider.get_train_valid_test(exp.dataset)
for batch_size in range(32,200,5):
train_dict['exp_args']['stop_criterion'].num_epochs += 2
log.info("Running with batch size {:d}".format(batch_size))
train_dict['exp_args']['iterator'].batch_size = batch_size
exp.run_until_stop(datasets, remember_best=False)
return
dataset = train_dict['dataset']
dataset.load()
iterator = train_dict['exp_args']['iterator']
splitter = train_dict['dataset_splitter']
if dataset.__class__.__name__ == 'EpilepsySet':
log.info("Reducing to float16 for epilepsy set...")
dataset.seizure_topo = np.float16(dataset.seizure_topo)
dataset.non_seizure_topo = np.float16(dataset.non_seizure_topo)
else:
# todo: remove this?
log.info("Determining dataset dimensions to set possible model params...")
train_set = splitter.split_into_train_valid_test(dataset)['train']
batch_gen = iterator.get_batches(train_set, shuffle=True)
dummy_batch_topo = batch_gen.next()[0]
del train_set
# not for ultrasound: assert 'in_sensors' in train_str
# not for cnt net assert 'in_rows' in train_str
# not for resnet: assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dummy_batch_topo.shape[1]))
train_str = train_str.replace('in_rows',
str(dummy_batch_topo.shape[2]))
train_str = train_str.replace('in_cols',
str(dummy_batch_topo.shape[3]))
self._save_train_string(train_str, experiment_index)
# reset rng for actual loading of layers, so you can reproduce it
# when you load the file later
lasagne.random.set_rng(RandomState(9859295))
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
assert len(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer))) == 0, ("All layers "
"should be used, unused {:s}".format(str(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer)))))
# Set n sample preds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
# Can this be moved up and duplication in if clause( batch test,
# more above) be removed?
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
if not self._cross_validation:
# for now lets not do that, current models seem fine again.
# if (dataset.__class__.__name__ == 'EpilepsySet') and self._pred_loss_hack:
# from braindecode.epilepsy.experiment import EpilepsyExperiment
# log.info("Creating epilepsy experiment with the pred loss hack")
# exp = EpilepsyExperiment(final_layer, dataset, splitter,
# **train_dict['exp_args'])
# else:
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run()
endtime = time.time()
model = exp.final_layer
# dummy predictions targets
predictions = [0,3,1,2,3,4]
targets = [3,4,1,2,3,4]
result_or_results = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp.monitor_chans,
predictions=predictions,
targets=targets)
else: # cross validation
assert False, ("cross validation not used in long time, not up to date"
" for example targets predictions not added")
# default 5 folds for now
n_folds = train_dict['num_cv_folds']
exp_cv = ExperimentCrossValidation(final_layer,
dataset, exp_args=train_dict['exp_args'], n_folds=n_folds,
shuffle=self._shuffle)
exp_cv.run()
endtime = time.time()
result_or_results = []
for i_fold in xrange(n_folds):
res = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp_cv.all_monitor_chans[i_fold],
predictions=[0,3,1,2,3,4],
targets=[3,4,1,2,3,4])
result_or_results.append(res)
model = exp_cv.all_layers
if not os.path.exists(self._folder_paths[experiment_index]):
os.makedirs(self._folder_paths[experiment_index])
result_file_name = self._get_result_save_path(experiment_index)
log.info("Saving result...")
with open(result_file_name, 'w') as resultfile:
pickle.dump(result_or_results, resultfile)
model_file_name = self._get_model_save_path(experiment_index)
param_file_name = model_file_name.replace('.pkl', '.npy')
np.save(param_file_name, lasagne.layers.get_all_param_values(model))
# Possibly make kaggle submission file
if isinstance(dataset, KaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_one_subject(self._folder_paths[experiment_index],
exp.dataset, iterator,
train_dict['exp_args']['preprocessor'],
final_layer, experiment_save_id)
elif isinstance(dataset, AllSubjectsKaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_all_subject_model(
self._folder_paths[experiment_index],
exp.dataset, exp.dataset_provider, iterator,
final_layer, experiment_save_id)
elif isinstance(splitter, SeveralSetsSplitter):
pass # nothing to do in this case
# very hacky create predictions targets :)
# Not done earlier as there were weird theano crashes
if exp.monitors[2].__class__.__name__ == 'CntTrialMisclassMonitor':
del dataset
del exp
add_labels_to_cnt_exp_result(self._base_save_paths[experiment_index])
def run(ex, data_folder, subject_id, n_chans, clean_train,
low_cut_hz, train_start_ms,kappa_mode, loss_expression,
network,
filt_order,
only_return_exp,):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
valid_subject_id = subject_
other_subject_ids = range(1,subject_id) + range(subject_id+1, 10)
other_sets = [create_dataset(
data_folder, other_sid, train_start_ms, low_cut_hz,
filt_order, clean_train) for other_sid in other_subject_ids]
test_set = create_dataset(
data_folder, subject_id, train_start_ms, low_cut_hz,
filt_order, clean_train)
combined_set = other_sets + [test_set]
def merge_train_test(single_combined_set):
return concatenate_sets(single_combined_set.train_set,
single_combined_set.test_set)
if not only_return_exp:
for i_set, this_set in enumerate(combined_set):
log.info("Loading {:d} of {:d}".format(i_set + 1,
len(combined_set)))
this_set.load()
merged_sets = [merge_train_test(s) for s in combined_set]
combined_set = CombinedSet(merged_sets)
in_chans = merged_sets[0].get_topological_view().shape[1]
else:
in_chans = n_chans
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
if network == 'deep':
final_layer = create_deep_net(in_chans, input_time_length)
else:
assert network == 'shallow'
final_layer = create_shallow_net(in_chans, input_time_length)
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.1, use_test_as_valid=False)
iterator = CntWindowTrialIterator(batch_size=45,input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length, mode=kappa_mode),
RuntimeMonitor(),]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800#100
# real values for paper were 80 and 800
stop_criterion = Or([NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop=True
exp = Experiment(final_layer, dataset,splitter,None,iterator, loss_expression,updates_expression, updates_modifier, monitors,
stop_criterion, remember_best_chan, run_after_early_stop, batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
save_npy_artifact(ex, lasagne.layers.get_all_param_values(exp.final_layer),
'model_params.npy')
def create_submission_csv_for_all_subject_model(folder_name,
all_sub_kaggle_set, dataset_provider, iterator, final_layer,
submission_id):
all_sub_kaggle_set.load()
assert all_sub_kaggle_set.resample_half == False, ("Not implemented for "
"resample half")
all_sub_kaggle_set.load_test()
# following line will just do the preprocessing already on the train set...
dataset_provider.get_train_merged_valid_test(all_sub_kaggle_set)
test_sets_per_subj = []
for i_subject in range(12):
kaggle_set = all_sub_kaggle_set.kaggle_sets[i_subject]
this_sets = []
for i_test_series in range(2):
# Get input
X_test = kaggle_set.test_X_series[i_test_series][:,:,np.newaxis,np.newaxis]
fake_test_y = np.ones((len(X_test), 6))
test_set = DenseDesignMatrixWrapper(
topo_view=X_test,
y=fake_test_y)
if dataset_provider.preprocessor is not None:
dataset_provider.preprocessor.apply(test_set, can_fit=False)
this_sets.append(test_set)
assert len(this_sets) == 2
test_sets_per_subj.append(this_sets)
### Create prediction function and create predictions
log.info("Create prediction functions...")
input_var = lasagne.layers.get_all_layers(final_layer)[0].input_var
predictions = lasagne.layers.get_output(final_layer, deterministic=True)
pred_fn = theano.function([input_var], predictions)
log.info("Setup iterator...")
n_sample_preds = get_n_sample_preds(final_layer)
iterator.n_sample_preds = n_sample_preds
log.info("Make predictions...")
preds_per_subject = []
for i_subject in range(12):
log.info("Predictions for Subject {:d}...".format(i_subject + 1))
test_sets_subj = test_sets_per_subj[i_subject]
preds = get_y_for_subject(pred_fn, test_sets_subj[0], test_sets_subj[1],
iterator, final_layer)
preds_per_subject.append(preds)
log.info("Done")
log.info("Create csv...")
cols = ['HandStart','FirstDigitTouch',
'BothStartLoadPhase','LiftOff',
'Replace','BothReleased']
# collect ids
all_ids = []
all_preds = []
for i_subject in range(12):
pred_subj_per_series = preds_per_subject[i_subject]
for i_series in (9,10):
id_prefix = "subj{:d}_series{:d}_".format(i_subject+1, i_series)
this_preds = pred_subj_per_series[i_series-9] # respect offsets
all_preds.extend(this_preds)
this_ids = [id_prefix + str(i_sample) for i_sample in range(this_preds.shape[0])]
all_ids.extend(this_ids)
all_ids = np.array(all_ids)
all_preds = np.array(all_preds)
assert all_ids.shape == (3144171,)
assert all_preds.shape == (3144171,6)
submission = pd.DataFrame(index=all_ids,
columns=cols,
data=all_preds)
csv_output = StringIO.StringIO()
submission.to_csv(csv_output, index_label='id',float_format='%.3f')
csv_str = csv_output.getvalue()
log.info("Create zip...")
zip_file_name = os.path.join(folder_name, "{:d}.zip".format(submission_id))
submission_zip_file = ZipFile(zip_file_name, 'w', ZIP_DEFLATED)
submission_zip_file.writestr("submission.csv", csv_str)
submission_zip_file.close()
log.info("Done")
def run(ex, data_folder, subject_id, n_chans, train_inds, test_inds,
sets_like_fbcsp_paper, clean_train, stop_chan, filt_order, low_cut_hz,
loss_expression, network, only_return_exp, run_after_early_stop):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
# trial ival in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
train_segment_ival = [1500, 4000]
test_segment_ival = [0, 4000]
if sets_like_fbcsp_paper:
if subject_id in [4, 5, 6, 7, 8, 9]:
train_inds = [3]
elif subject_id == 1:
train_inds = [1, 3]
else:
assert subject_id in [2, 3]
train_inds = [1, 2, 3]
train_loader = MultipleBCICompetition4Set2B(subject_id,
session_ids=train_inds,
data_folder=data_folder)
test_loader = MultipleBCICompetition4Set2B(subject_id,
session_ids=test_inds,
data_folder=data_folder)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}),
(bandpass_cnt, {
'low_cut_hz': low_cut_hz,
'high_cut_hz': 38,
'filt_order': filt_order,
}), (exponential_standardize_cnt, {})]
marker_def = {'1- Left Hand': [1], '2 - Right Hand': [2]}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=train_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=test_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
if clean_train:
train_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
else:
train_cleaner = NoCleaner()
test_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
lasagne.random.set_rng(RandomState(34734))
in_chans = train_set.get_topological_view().shape[1]
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
if network == 'deep':
final_layer = create_deep_net(in_chans, input_time_length)
else:
assert network == 'shallow'
final_layer = create_shallow_net(in_chans, input_time_length)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length, mode='max'),
RuntimeMonitor()
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
remember_best_chan = 'valid_' + stop_chan
stop_criterion = Or([
NoDecrease(remember_best_chan, num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
def _run_experiments_with_string(self, experiment_index, train_str):
assert experiment_index >= self._get_start_id()
assert experiment_index < self._get_stop_id()
lasagne.random.set_rng(RandomState(9859295))
# Save train string now, will be overwritten later after
# input dimensions determined, save now for debug in
# case of crash
if not self._dry_run:
self._save_train_string(train_str, experiment_index)
starttime = time.time()
train_dict = self._load_without_layers(train_str)
log.info("With params...")
if not self._quiet:
pprint(train_dict['original_params'])
if self._dry_run:
# Do not do the loading or training...
# Only go until here to show the train params
return
if self._batch_test:
# TODO: put into function
# load layers, load data with dimensions of the layer
# create experiment with max epochs 2, run
from braindecode.datasets.random import RandomSet
train_str = train_str.replace('in_cols', '1')
train_str = train_str.replace('in_sensors', '32')
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
n_chans = layers[0].shape[1]
n_classes = final_layer.output_shape[1]
n_samples = 500000
# set n sample perds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args'][
'iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# make at least batches
n_samples = int(n_sample_preds * 1.5 * 200)
dataset = RandomSet(topo_shape=[n_samples, n_chans, 1, 1],
y_shape=[n_samples, n_classes])
dataset.load()
splitter = FixedTrialSplitter(n_train_trials=int(n_samples * 0.8),
valid_set_fraction=0.1)
train_dict['exp_args']['preprocessor'] = None
train_dict['exp_args']['stop_criterion'] = MaxEpochs(1)
train_dict['exp_args']['iterator'].batch_size = 1
# TODO: set stop criterion to max epochs =1
# change batch_size in iterator
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run_until_early_stop()
datasets = exp.dataset_provider.get_train_valid_test(exp.dataset)
for batch_size in range(32, 200, 5):
train_dict['exp_args']['stop_criterion'].num_epochs += 2
log.info("Running with batch size {:d}".format(batch_size))
train_dict['exp_args']['iterator'].batch_size = batch_size
exp.run_until_stop(datasets, remember_best=False)
return
dataset = train_dict['dataset']
dataset.load()
iterator = train_dict['exp_args']['iterator']
splitter = train_dict['dataset_splitter']
if dataset.__class__.__name__ == 'EpilepsySet':
log.info("Reducing to float16 for epilepsy set...")
dataset.seizure_topo = np.float16(dataset.seizure_topo)
dataset.non_seizure_topo = np.float16(dataset.non_seizure_topo)
else:
# todo: remove this?
log.info(
"Determining dataset dimensions to set possible model params..."
)
train_set = splitter.split_into_train_valid_test(dataset)['train']
batch_gen = iterator.get_batches(train_set, shuffle=True)
dummy_batch_topo = batch_gen.next()[0]
del train_set
# not for ultrasound: assert 'in_sensors' in train_str
# not for cnt net assert 'in_rows' in train_str
# not for resnet: assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dummy_batch_topo.shape[1]))
train_str = train_str.replace('in_rows',
str(dummy_batch_topo.shape[2]))
train_str = train_str.replace('in_cols',
str(dummy_batch_topo.shape[3]))
self._save_train_string(train_str, experiment_index)
# reset rng for actual loading of layers, so you can reproduce it
# when you load the file later
lasagne.random.set_rng(RandomState(9859295))
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
assert len(
np.setdiff1d(
layers, lasagne.layers.get_all_layers(final_layer))) == 0, (
"All layers "
"should be used, unused {:s}".format(
str(
np.setdiff1d(
layers,
lasagne.layers.get_all_layers(final_layer)))))
# Set n sample preds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
# Can this be moved up and duplication in if clause( batch test,
# more above) be removed?
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
if not self._cross_validation:
# for now lets not do that, current models seem fine again.
# if (dataset.__class__.__name__ == 'EpilepsySet') and self._pred_loss_hack:
# from braindecode.epilepsy.experiment import EpilepsyExperiment
# log.info("Creating epilepsy experiment with the pred loss hack")
# exp = EpilepsyExperiment(final_layer, dataset, splitter,
# **train_dict['exp_args'])
# else:
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run()
endtime = time.time()
model = exp.final_layer
# dummy predictions targets
predictions = [0, 3, 1, 2, 3, 4]
targets = [3, 4, 1, 2, 3, 4]
result_or_results = Result(
parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp.monitor_chans,
predictions=predictions,
targets=targets)
else: # cross validation
assert False, (
"cross validation not used in long time, not up to date"
" for example targets predictions not added")
# default 5 folds for now
n_folds = train_dict['num_cv_folds']
exp_cv = ExperimentCrossValidation(final_layer,
dataset,
exp_args=train_dict['exp_args'],
n_folds=n_folds,
shuffle=self._shuffle)
exp_cv.run()
endtime = time.time()
result_or_results = []
for i_fold in xrange(n_folds):
res = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp_cv.all_monitor_chans[i_fold],
predictions=[0, 3, 1, 2, 3, 4],
targets=[3, 4, 1, 2, 3, 4])
result_or_results.append(res)
model = exp_cv.all_layers
if not os.path.exists(self._folder_paths[experiment_index]):
os.makedirs(self._folder_paths[experiment_index])
result_file_name = self._get_result_save_path(experiment_index)
log.info("Saving result to {:s}...".format(result_file_name))
with open(result_file_name, 'w') as resultfile:
pickle.dump(result_or_results, resultfile)
model_file_name = self._get_model_save_path(experiment_index)
param_file_name = model_file_name.replace('.pkl', '.npy')
np.save(param_file_name, lasagne.layers.get_all_param_values(model))
# Possibly make kaggle submission file
if isinstance(dataset,
KaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_one_subject(
self._folder_paths[experiment_index], exp.dataset, iterator,
train_dict['exp_args']['preprocessor'], final_layer,
experiment_save_id)
elif isinstance(
dataset,
AllSubjectsKaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_all_subject_model(
self._folder_paths[experiment_index], exp.dataset,
exp.dataset_provider, iterator, final_layer,
experiment_save_id)
elif isinstance(splitter, SeveralSetsSplitter):
pass # nothing to do in this case
# very hacky create predictions targets :)
# Not done earlier as there were weird theano crashes
if exp.monitors[2].__class__.__name__ == 'CntTrialMisclassMonitor':
del dataset
del exp
add_labels_to_cnt_exp_result(
self._base_save_paths[experiment_index])
def tied_neighbours_cnt_model(preds, targets, final_layer):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
return tied_neighbours(preds, n_sample_preds, n_classes)
def run_exp(i_fold):
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
d5net = Deep5Net(in_chans=in_chans,
input_time_length=input_time_length,
num_filters_time=num_filters_time,
filter_time_length=filter_time_length,
num_filters_spat=num_filters_spat,
pool_time_length=pool_time_length,
pool_time_stride=pool_time_stride,
num_filters_2=num_filters_2,
filter_length_2=filter_length_2,
num_filters_3=num_filters_3,
filter_length_3=filter_length_3,
num_filters_4=num_filters_4,
filter_length_4=filter_length_4,
final_dense_length=final_dense_length,
n_classes=n_classes,
final_nonlin=final_nonlin,
first_nonlin=first_nonlin,
first_pool_mode=first_pool_mode,
first_pool_nonlin=first_pool_nonlin,
later_nonlin=later_nonlin,
later_pool_mode=later_pool_mode,
later_pool_nonlin=later_pool_nonlin,
drop_in_prob=drop_in_prob,
drop_prob=drop_prob,
batch_norm_alpha=batch_norm_alpha,
double_time_convs=double_time_convs,
split_first_layer=split_first_layer,
batch_norm=batch_norm)
final_layer = d5net.get_layers()[-1]
final_layer = ClipLayer(final_layer, 1e-4, 1 - 1e-4)
dataset_splitter = CntTrialSingleFoldSplitter(n_folds=10,
i_test_fold=i_fold,
shuffle=True)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length,
mode='max'),
RuntimeMonitor()
]
#n_no_decrease_max_epochs = 2
#n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800
# real values for paper were 80 and 800
remember_best_chan = 'valid_' + stop_chan
stop_criterion = Or([
NoDecrease(remember_best_chan,
num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
preproc = None
exp = Experiment(final_layer,
dataset,
splitter,
preproc,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
return exp
def tied_losses_cnt_model(preds, targets, final_layer, n_pairs):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
return tied_losses(preds, n_sample_preds, n_classes, n_pairs)
def tied_neighbours_cnt_model_custom_loss(preds, targets, final_layer,
loss_fn):
n_sample_preds = get_n_sample_preds(final_layer)
n_classes = final_layer.output_shape[1]
return tied_neighbours_custom_loss(preds, n_sample_preds, n_classes,
loss_fn)
def run(
ex,
data_folder,
subject_id,
n_chans,
clean_train,
low_cut_hz,
train_start_ms,
kappa_mode,
loss_expression,
filt_order,
only_return_exp,
):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
load_sensor_names = None
train_filename = 'A{:02d}T.mat'.format(subject_id)
test_filename = 'A{:02d}E.mat'.format(subject_id)
train_filepath = os.path.join(data_folder, train_filename)
test_filepath = os.path.join(data_folder, test_filename)
# trial ivan in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
train_segment_ival = [train_start_ms, 4000]
test_segment_ival = [0, 4000]
train_loader = BCICompetition4Set2A(train_filepath,
load_sensor_names=load_sensor_names)
test_loader = BCICompetition4Set2A(test_filepath,
load_sensor_names=load_sensor_names)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}),
(bandpass_cnt, {
'low_cut_hz': low_cut_hz,
'high_cut_hz': 38,
'filt_order': filt_order,
}), (exponential_standardize_cnt, {})]
marker_def = {
'1- Right Hand': [1],
'2 - Left Hand': [2],
'3 - Rest': [3],
'4 - Feet': [4]
}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=train_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=test_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
if clean_train:
train_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
else:
train_cleaner = NoCleaner()
test_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
in_chans = train_set.get_topological_view().shape[1]
else:
in_chans = n_chans
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
final_layer = create_deep_net(in_chans, input_time_length)
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length,
mode=kappa_mode),
RuntimeMonitor(),
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
stop_criterion = Or([
NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop = True
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
save_npy_artifact(ex, lasagne.layers.get_all_param_values(exp.final_layer),
'model_params.npy')
