def compute_preds_per_trial(y, all_preds, all_batch_sizes, input_time_length):
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
y, check_trial_lengths_equal=False,
input_time_length=input_time_length)
i_pred_block = 0
n_sample_preds = all_preds[0].shape[0] / all_batch_sizes[0]
all_preds_arr = np.concatenate(all_preds, axis=0)
preds_per_forward_pass = np.reshape(all_preds_arr, (-1, n_sample_preds,
all_preds_arr.shape[1]))
preds_per_trial = []
for i_trial in xrange(len(i_trial_starts)):
# + 1 since end is inclusive
# so if trial end is 1 and trial start is 0
# need two samples (0 and 1)
needed_samples = (i_trial_ends[i_trial] - i_trial_starts[i_trial]) + 1
preds_this_trial = []
while needed_samples > 0:
# - needed_samples: only has an effect
# in case there are more samples thatn we actually still need
# in the block
# That can happen since final block of a trial can overlap
# with block before so we can have some redundant preds
pred_samples = preds_per_forward_pass[i_pred_block, -needed_samples:]
preds_this_trial.append(pred_samples)
needed_samples -= len(pred_samples)
i_pred_block += 1
preds_this_trial = np.concatenate(preds_this_trial, axis=0)
preds_per_trial.append(preds_this_trial)
assert i_pred_block == len(preds_per_forward_pass) , ("Expect that all "
"prediction forward passes are needed, used {:d}, existing {:d}".format(
i_pred_block, len(preds_per_forward_pass)))
return preds_per_trial
def compute_preds_per_trial(y, all_preds, all_batch_sizes, input_time_length):
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
y,
check_trial_lengths_equal=False,
input_time_length=input_time_length)
return compute_preds_per_trial_from_start_end(all_preds, all_batch_sizes,
i_trial_starts, i_trial_ends)
def compute_pred_and_target_labels(self, dataset, all_preds, all_batch_sizes):
all_target_labels = []
preds_per_trial = compute_preds_per_trial(dataset.y,
all_preds, all_batch_sizes, self.input_time_length)
all_pred_labels = [np.argmax(np.mean(p, axis=0))
for p in preds_per_trial]
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
dataset.y, check_trial_lengths_equal=False,
input_time_length=self.input_time_length)
for i_trial, (start, end) in enumerate(zip(i_trial_starts, i_trial_ends)):
targets = dataset.y[start:end+1] # end is not inclusive
assert len(targets) == len(preds_per_trial[i_trial])
# max would have several 1s for different classes
# if there are any two different classes with 1s
# in all samples
assert np.sum(np.max(targets, axis=0)) == 1, ("Trial should only "
"have one class")
assert np.sum(targets) == len(targets), ("Every sample should have "
"one positive marker")
target_label = np.argmax(np.max(targets, axis=0))
all_target_labels.append(target_label)
all_pred_labels = np.array(all_pred_labels)
all_target_labels = np.array(all_target_labels)
return all_pred_labels, all_target_labels
def load_markers_from_label_file(label_file_path, fs):
labelfile = loadmat(label_file_path)
y_signal = labelfile['true_y'].squeeze()
modified_y_signal = y_signal.copy()
modified_y_signal[np.isnan(modified_y_signal)] = 0
one_hot_y_signal = np.zeros((len(modified_y_signal), 2),
dtype=np.int32)
one_hot_y_signal[modified_y_signal == -1, 0] = 1
one_hot_y_signal[modified_y_signal == 1, 1] = 1
starts, ends = compute_trial_start_end_samples(
one_hot_y_signal, check_trial_lengths_equal=False)
mrk_codes = []
mrk_ms = []
for start, end in zip(starts, ends):
assert np.isnan(y_signal[start - 1])
assert not np.isnan(y_signal[start])
assert np.isnan(y_signal[end + 1])
assert not np.isnan(y_signal[end])
mrk_codes.append(((y_signal[start] + 1) / 2) + 1)
eval_start = start * 1000.0 / fs
mrk_ms.append(eval_start -
1000) # first 1000 ms removed from eval..
mrk_codes.append(
((y_signal[end] + 1) / 2) + 11) # -> 11,12 for end
mrk_ms.append(end * 1000.0 / fs)
return mrk_ms, mrk_codes
def compute_pred_and_target_labels(self, dataset, all_preds,
all_batch_sizes):
all_target_labels = []
preds_per_trial = compute_preds_per_trial(dataset.y, all_preds,
all_batch_sizes,
self.input_time_length)
all_pred_labels = [
np.argmax(np.mean(p, axis=0)) for p in preds_per_trial
]
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
dataset.y,
check_trial_lengths_equal=False,
input_time_length=self.input_time_length)
for i_trial, (start,
end) in enumerate(zip(i_trial_starts, i_trial_ends)):
targets = dataset.y[start:end + 1] # end is not inclusive
assert len(targets) == len(preds_per_trial[i_trial])
# max would have several 1s for different classes
# if there are any two different classes with 1s
# in all samples
assert np.sum(np.max(targets, axis=0)) == 1, ("Trial should only "
"have one class")
assert np.sum(targets) == len(targets), (
"Every sample should have "
"one positive marker")
target_label = np.argmax(np.max(targets, axis=0))
all_target_labels.append(target_label)
all_pred_labels = np.array(all_pred_labels)
all_target_labels = np.array(all_target_labels)
return all_pred_labels, all_target_labels
def split_into_train_valid_test(self, dataset):
"""Split into train valid test by splitting
dataset into num folds, test fold nr should be given,
valid fold will be the one immediately before the test fold,
train folds the remaining 8 folds
"""
# also works in case test fold nr is 0 as it will just take -1
# which is fine last fold)
i_valid_fold = self.i_test_fold - 1
starts, ends = compute_trial_start_end_samples(dataset.y, check_trial_lengths_equal=False)
n_trials = len(starts)
rng = RandomState(self.seed)
folds = get_balanced_batches(n_trials, rng, shuffle=self.shuffle,
n_batches=self.n_folds)
test_fold = folds[self.i_test_fold]
valid_fold = folds[i_valid_fold]
test_inds = convert_to_sample_inds(test_fold, ends)
valid_inds = convert_to_sample_inds(valid_fold, ends)
train_inds = np.setdiff1d(np.arange(len(dataset.y)),
np.concatenate((valid_inds, test_inds)))
datasets = split_set_by_indices(dataset, train_inds, valid_inds,
test_inds)
if hasattr(dataset, 'additional_set'):
datasets['train'] = concatenate_sets(dataset.additional_set,
datasets['train'],)
return datasets
def determine_trial_labels(dataset, iterator):
# determine trial labels
i_trial_starts, _ = compute_trial_start_end_samples(
dataset.y, check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
trial_labels = dataset.y[i_trial_starts]
assert np.all(np.sum(trial_labels, axis=1) == 1)
return trial_labels
def determine_trial_labels(dataset, iterator):
# determine trial labels
i_trial_starts, _ = compute_trial_start_end_samples(
dataset.y, check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
trial_labels = dataset.y[i_trial_starts]
assert np.all(np.sum(trial_labels, axis=1) == 1)
return trial_labels
def segment(self, cnt, y, class_names):
# dont modify original y
y = np.copy(y)
if (self.start_trial is not None) or (self.stop_trial is not None):
trial_starts, trial_ends = compute_trial_start_end_samples(
y, check_trial_lengths_equal=False)
if self.start_trial is not None:
y[:trial_starts[self.start_trial] - 1] = 0
if self.stop_trial is not None:
y[trial_starts[self.stop_trial] - 1:] = 0
return y, class_names
def segment(self, cnt, y, class_names):
# dont modify original y
y = np.copy(y)
if (self.start_trial is not None) or (self.stop_trial is not None):
trial_starts, trial_ends = compute_trial_start_end_samples(y,
check_trial_lengths_equal=False)
if self.start_trial is not None:
y[:trial_starts[self.start_trial] - 1] = 0
if self.stop_trial is not None:
y[trial_starts[self.stop_trial] - 1:] = 0
return y, class_names
def compute_trial_acts(model, i_layer, iterator, train_set):
"""Compute activations per trial per sample for given layer of the model.
Parameters
----------
model: Lasagne layer
Final layer of the model.
i_layer: int
Index of layer to compute activations for.
iterator: DatasetIterator
Iterator to get batches from.
train_set: Dataset (Cnt)
Dataset to use.
Returns
-------
trial_acts: 3darray of float
Activations per trial per sample. #trialx#channelx#sample
"""
# compute number of inputs per trial
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y,
check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
# +1 since trial ends is inclusive
n_trial_len = i_trial_ends[0] - i_trial_starts[0] + 1
n_inputs_per_trial = int(
np.ceil(n_trial_len / float(iterator.n_sample_preds)))
log.info("Create theano function...")
all_layers = lasagne.layers.get_all_layers(model)
all_out_fn = create_pred_fn(all_layers[i_layer])
assert (iterator.input_time_length == get_input_time_length(model))
assert (iterator.n_sample_preds == get_n_sample_preds(model))
log.info("Compute activations...")
all_outs_per_batch = [
all_out_fn(batch[0])
for batch in iterator.get_batches(train_set, False)
]
batch_sizes = [
len(batch[0]) for batch in iterator.get_batches(train_set, False)
]
all_outs_per_batch = np.array(all_outs_per_batch)
n_trials = len(i_trial_starts)
log.info("Transform to trial activations...")
trial_acts = get_trial_acts(all_outs_per_batch,
batch_sizes,
n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=iterator.n_sample_preds)
log.info("Done.")
return trial_acts
def get_trials_targets(train_set, n_sample_preds, iterator):
iterator = deepcopy(iterator) # will be modified..
starts, ends = compute_trial_start_end_samples(
train_set.y, check_trial_lengths_equal=True)
trial_len = ends[0] - starts[0]
iterator.n_sample_preds = n_sample_preds
# -> get one batch per trial by setting batch size
iterator.batch_size = int(np.ceil(trial_len / float(n_sample_preds)))
batches = list(iterator.get_batches(train_set, shuffle=False))
trials, targets = zip(*batches)
trials = np.array(trials)
targets = np.array(targets)
return trials, targets
def segment(self, cnt, y, class_names):
print("fs", cnt.fs)
n_min_samples = int(self.min_length_ms * cnt.fs / 1000.0)
starts, ends = compute_trial_start_end_samples(
y,
check_trial_lengths_equal=False,
)
n_removed_trials = 0
for i_sample_start, i_sample_end in zip(starts, ends):
n_samples_in_trial = i_sample_end - i_sample_start + 1
if n_samples_in_trial < n_min_samples:
y[i_sample_start:i_sample_end + 1] = 0
n_removed_trials += 1
return y, class_names
def compute_env_class_corr(exp, trial_env):
train_set = exp.dataset_provider.get_train_merged_valid_test(
exp.dataset)['train']
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y,
check_trial_lengths_equal=True,
input_time_length=exp.iterator.input_time_length)
assert len(i_trial_ends) == trial_env.shape[1]
# +1 as i_end is inclusive
y_signal = [exp.dataset.train_set.y[i_start:i_end+1]
for i_start, i_end in zip(i_trial_starts, i_trial_ends)]
y_signal = np.array(y_signal).transpose(0,2,1)
assert y_signal.shape[2] == trial_env.shape[3]
topo_corrs = compute_topo_corrs(trial_env, y_signal)
return topo_corrs
def compute_trial_acts(model, i_layer, iterator, train_set):
"""Compute activations per trial per sample for given layer of the model.
Parameters
----------
model: Lasagne layer
Final layer of the model.
i_layer: int
Index of layer to compute activations for.
iterator: DatasetIterator
Iterator to get batches from.
train_set: Dataset (Cnt)
Dataset to use.
Returns
-------
trial_acts: 3darray of float
Activations per trial per sample. #trialx#channelx#sample
"""
# compute number of inputs per trial
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y, check_trial_lengths_equal=True,
input_time_length=iterator.input_time_length)
# +1 since trial ends is inclusive
n_trial_len = i_trial_ends[0] - i_trial_starts[0] + 1
n_inputs_per_trial = int(np.ceil(n_trial_len / float(iterator.n_sample_preds)))
log.info("Create theano function...")
all_layers = lasagne.layers.get_all_layers(model)
all_out_fn = create_pred_fn(all_layers[i_layer])
assert(iterator.input_time_length == get_input_time_length(model))
assert(iterator.n_sample_preds == get_n_sample_preds(model))
log.info("Compute activations...")
all_outs_per_batch = [all_out_fn(batch[0])
for batch in iterator.get_batches(train_set, False)]
batch_sizes = [len(batch[0]) for batch in iterator.get_batches(train_set, False)]
all_outs_per_batch = np.array(all_outs_per_batch)
n_trials = len(i_trial_starts)
log.info("Transform to trial activations...")
trial_acts = get_trial_acts(all_outs_per_batch,
batch_sizes, n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=iterator.n_sample_preds)
log.info("Done.")
return trial_acts
def compute_env_class_corr(exp, trial_env):
train_set = exp.dataset_provider.get_train_merged_valid_test(
exp.dataset)['train']
i_trial_starts, i_trial_ends = compute_trial_start_end_samples(
train_set.y,
check_trial_lengths_equal=True,
input_time_length=exp.iterator.input_time_length)
assert len(i_trial_ends) == trial_env.shape[1]
# +1 as i_end is inclusive
y_signal = [
exp.dataset.train_set.y[i_start:i_end + 1]
for i_start, i_end in zip(i_trial_starts, i_trial_ends)
]
y_signal = np.array(y_signal).transpose(0, 2, 1)
assert y_signal.shape[2] == trial_env.shape[3]
topo_corrs = compute_topo_corrs(trial_env, y_signal)
return topo_corrs
def transform_to_meaned_trial_env(env, model, i_layer, train_set,
n_inputs_per_trial):
all_layers = lasagne.layers.get_all_layers(model)
layer = all_layers[i_layer]
field_size = get_receptive_field_size(layer)
trial_starts, trial_ends = compute_trial_start_end_samples(train_set.y)
assert len(np.unique(trial_ends - trial_starts)) == 1
n_trials = len(trial_starts)
# +1 as trial end is inclusive
n_trial_len = np.unique(trial_ends - trial_starts)[0] + 1
# Afterwards env is list empty lists(!!)
n_sample_preds = get_n_sample_preds(model)
trial_env = get_meaned_trial_env(env, field_size=field_size,
n_trials=n_trials,
n_inputs_per_trial=n_inputs_per_trial,
n_trial_len=n_trial_len,
n_sample_preds=n_sample_preds)
return trial_env
