def test_experiment_sample_windows():
data_rng = RandomState(398765905)
rand_topo = data_rng.rand(200,10,10,3).astype(np.float32)
rand_y = np.int32(data_rng.rand(200) > 0.5)
rand_topo[rand_y == 1] += 0.1
rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y)
lasagne.random.set_rng(RandomState(9859295))
in_layer = InputLayer(shape= [None, 10,5,3])
network = DenseLayer(incoming=in_layer, name='softmax',
num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
updates_modifier = MaxNormConstraint({'softmax': 0.5})
dataset = rand_set
dataset_iterator = WindowsIterator(n_samples_per_window=5,
batch_size=60)
preprocessor = OnlineAxiswiseStandardize(axis=['c', 1])
dataset_splitter=FixedTrialSplitter(n_train_trials=150, valid_set_fraction=0.1)
updates_var_func=lasagne.updates.adam
loss_var_func= lasagne.objectives.categorical_crossentropy
monitors=[braindecode.veganlasagne.monitors.LossMonitor (),
braindecode.veganlasagne.monitors.WindowMisclassMonitor(),
braindecode.veganlasagne.monitors.RuntimeMonitor()]
stop_criterion= braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=5)
exp = Experiment(network, dataset, dataset_splitter, preprocessor,
dataset_iterator, loss_var_func, updates_var_func,
updates_modifier, monitors, stop_criterion,
remember_best_chan='valid_misclass',
run_after_early_stop=True)
exp.setup()
exp.run()
assert np.allclose(
[0.629630,0.140741,0.029630,0.022222,0.000000,0.000000,0.000000],
exp.monitor_chans['train_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[0.400000,0.133333,0.066667,0.000000,0.000000,0.000000,0.000000],
exp.monitor_chans['valid_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[0.560000,0.060000,0.000000,0.000000,0.000000,0.000000,0.000000],
exp.monitor_chans['test_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[1.180485, 0.574264, 0.420023, 0.330909, 0.278569, 0.245692, 0.242845],
exp.monitor_chans['train_loss'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[1.016782, 0.514049, 0.370485, 0.288948, 0.240913, 0.211189, 0.215967],
exp.monitor_chans['valid_loss'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[1.031832, 0.504570, 0.352317, 0.269810, 0.223904, 0.196681, 0.197899],
exp.monitor_chans['test_loss'],
rtol=1e-4, atol=1e-4)
def run(
ex,
data_folder,
subject_id,
n_chans,
only_return_exp,
):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
load_sensor_names = None
train_filename = 'A{:02d}T.mat'.format(subject_id)
test_filename = 'A{:02d}E.mat'.format(subject_id)
train_filepath = os.path.join(data_folder, train_filename)
test_filepath = os.path.join(data_folder, test_filename)
# trial ivan in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
segment_ival = [1500, 4000]
train_loader = BCICompetition4Set2A(train_filepath,
load_sensor_names=load_sensor_names)
test_loader = BCICompetition4Set2A(test_filepath,
load_sensor_names=load_sensor_names)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}), (bandpass_cnt, {
'low_cut_hz': 0,
'high_cut_hz': 38,
}), (exponential_standardize_cnt, {})]
marker_def = {
'1- Right Hand': [1],
'2 - Left Hand': [2],
'3 - Rest': [3],
'4 - Feet': [4]
}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
from braindecode.mywyrm.clean import MaxAbsCleaner
train_cleaner = MaxAbsCleaner(segment_ival=[0, 4000],
threshold=800,
marker_def=marker_def)
test_cleaner = MaxAbsCleaner(segment_ival=[0, 4000],
threshold=800,
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
in_chans = train_set.get_topological_view().shape[1]
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
num_filters_time = 25
filter_time_length = 10
num_filters_spat = 25
pool_time_length = 3
pool_time_stride = 3
num_filters_2 = 50
filter_length_2 = 10
num_filters_3 = 100
filter_length_3 = 10
num_filters_4 = 200
filter_length_4 = 10
final_dense_length = 2
n_classes = 4
final_nonlin = softmax
first_nonlin = elu
first_pool_mode = 'max'
first_pool_nonlin = identity
later_nonlin = elu
later_pool_mode = 'max'
later_pool_nonlin = identity
drop_in_prob = 0.0
drop_prob = 0.5
batch_norm_alpha = 0.1
double_time_convs = False
split_first_layer = True
batch_norm = True
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
d5net = Deep5Net(in_chans=in_chans,
input_time_length=input_time_length,
num_filters_time=num_filters_time,
filter_time_length=filter_time_length,
num_filters_spat=num_filters_spat,
pool_time_length=pool_time_length,
pool_time_stride=pool_time_stride,
num_filters_2=num_filters_2,
filter_length_2=filter_length_2,
num_filters_3=num_filters_3,
filter_length_3=filter_length_3,
num_filters_4=num_filters_4,
filter_length_4=filter_length_4,
final_dense_length=final_dense_length,
n_classes=n_classes,
final_nonlin=final_nonlin,
first_nonlin=first_nonlin,
first_pool_mode=first_pool_mode,
first_pool_nonlin=first_pool_nonlin,
later_nonlin=later_nonlin,
later_pool_mode=later_pool_mode,
later_pool_nonlin=later_pool_nonlin,
drop_in_prob=drop_in_prob,
drop_prob=drop_prob,
batch_norm_alpha=batch_norm_alpha,
double_time_convs=double_time_convs,
split_first_layer=split_first_layer,
batch_norm=batch_norm)
final_layer = d5net.get_layers()[-1]
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
RuntimeMonitor()
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
stop_criterion = Or([
NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
loss_expression = categorical_crossentropy
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop = True
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
ex.info['runtime'] = run_time
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
def run(ex, data_folder, subject_id, n_chans, train_inds, test_inds,
sets_like_fbcsp_paper, clean_train, stop_chan, filt_order, low_cut_hz,
loss_expression, network, only_return_exp, run_after_early_stop):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
# trial ival in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
train_segment_ival = [1500, 4000]
test_segment_ival = [0, 4000]
if sets_like_fbcsp_paper:
if subject_id in [4, 5, 6, 7, 8, 9]:
train_inds = [3]
elif subject_id == 1:
train_inds = [1, 3]
else:
assert subject_id in [2, 3]
train_inds = [1, 2, 3]
train_loader = MultipleBCICompetition4Set2B(subject_id,
session_ids=train_inds,
data_folder=data_folder)
test_loader = MultipleBCICompetition4Set2B(subject_id,
session_ids=test_inds,
data_folder=data_folder)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}),
(bandpass_cnt, {
'low_cut_hz': low_cut_hz,
'high_cut_hz': 38,
'filt_order': filt_order,
}), (exponential_standardize_cnt, {})]
marker_def = {'1- Left Hand': [1], '2 - Right Hand': [2]}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=train_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=test_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
if clean_train:
train_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
else:
train_cleaner = NoCleaner()
test_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
lasagne.random.set_rng(RandomState(34734))
in_chans = train_set.get_topological_view().shape[1]
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
if network == 'deep':
final_layer = create_deep_net(in_chans, input_time_length)
else:
assert network == 'shallow'
final_layer = create_shallow_net(in_chans, input_time_length)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length, mode='max'),
RuntimeMonitor()
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
remember_best_chan = 'valid_' + stop_chan
stop_criterion = Or([
NoDecrease(remember_best_chan, num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
def _run_experiments_with_string(self, experiment_index, train_str):
assert experiment_index >= self._get_start_id()
assert experiment_index < self._get_stop_id()
lasagne.random.set_rng(RandomState(9859295))
# Save train string now, will be overwritten later after
# input dimensions determined, save now for debug in
# case of crash
if not self._dry_run:
self._save_train_string(train_str, experiment_index)
starttime = time.time()
train_dict = self._load_without_layers(train_str)
log.info("With params...")
if not self._quiet:
pprint(train_dict['original_params'])
if self._dry_run:
# Do not do the loading or training...
# Only go until here to show the train params
return
if self._batch_test:
# TODO: put into function
# load layers, load data with dimensions of the layer
# create experiment with max epochs 2, run
from braindecode.datasets.random import RandomSet
train_str = train_str.replace('in_cols', '1')
train_str = train_str.replace('in_sensors', '32')
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
n_chans = layers[0].shape[1]
n_classes = final_layer.output_shape[1]
n_samples = 500000
# set n sample perds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args'][
'iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# make at least batches
n_samples = int(n_sample_preds * 1.5 * 200)
dataset = RandomSet(topo_shape=[n_samples, n_chans, 1, 1],
y_shape=[n_samples, n_classes])
dataset.load()
splitter = FixedTrialSplitter(n_train_trials=int(n_samples * 0.8),
valid_set_fraction=0.1)
train_dict['exp_args']['preprocessor'] = None
train_dict['exp_args']['stop_criterion'] = MaxEpochs(1)
train_dict['exp_args']['iterator'].batch_size = 1
# TODO: set stop criterion to max epochs =1
# change batch_size in iterator
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run_until_early_stop()
datasets = exp.dataset_provider.get_train_valid_test(exp.dataset)
for batch_size in range(32, 200, 5):
train_dict['exp_args']['stop_criterion'].num_epochs += 2
log.info("Running with batch size {:d}".format(batch_size))
train_dict['exp_args']['iterator'].batch_size = batch_size
exp.run_until_stop(datasets, remember_best=False)
return
dataset = train_dict['dataset']
dataset.load()
iterator = train_dict['exp_args']['iterator']
splitter = train_dict['dataset_splitter']
if dataset.__class__.__name__ == 'EpilepsySet':
log.info("Reducing to float16 for epilepsy set...")
dataset.seizure_topo = np.float16(dataset.seizure_topo)
dataset.non_seizure_topo = np.float16(dataset.non_seizure_topo)
else:
# todo: remove this?
log.info(
"Determining dataset dimensions to set possible model params..."
)
train_set = splitter.split_into_train_valid_test(dataset)['train']
batch_gen = iterator.get_batches(train_set, shuffle=True)
dummy_batch_topo = batch_gen.next()[0]
del train_set
# not for ultrasound: assert 'in_sensors' in train_str
# not for cnt net assert 'in_rows' in train_str
# not for resnet: assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dummy_batch_topo.shape[1]))
train_str = train_str.replace('in_rows',
str(dummy_batch_topo.shape[2]))
train_str = train_str.replace('in_cols',
str(dummy_batch_topo.shape[3]))
self._save_train_string(train_str, experiment_index)
# reset rng for actual loading of layers, so you can reproduce it
# when you load the file later
lasagne.random.set_rng(RandomState(9859295))
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
assert len(
np.setdiff1d(
layers, lasagne.layers.get_all_layers(final_layer))) == 0, (
"All layers "
"should be used, unused {:s}".format(
str(
np.setdiff1d(
layers,
lasagne.layers.get_all_layers(final_layer)))))
# Set n sample preds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
# Can this be moved up and duplication in if clause( batch test,
# more above) be removed?
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
if not self._cross_validation:
# for now lets not do that, current models seem fine again.
# if (dataset.__class__.__name__ == 'EpilepsySet') and self._pred_loss_hack:
# from braindecode.epilepsy.experiment import EpilepsyExperiment
# log.info("Creating epilepsy experiment with the pred loss hack")
# exp = EpilepsyExperiment(final_layer, dataset, splitter,
# **train_dict['exp_args'])
# else:
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run()
endtime = time.time()
model = exp.final_layer
# dummy predictions targets
predictions = [0, 3, 1, 2, 3, 4]
targets = [3, 4, 1, 2, 3, 4]
result_or_results = Result(
parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp.monitor_chans,
predictions=predictions,
targets=targets)
else: # cross validation
assert False, (
"cross validation not used in long time, not up to date"
" for example targets predictions not added")
# default 5 folds for now
n_folds = train_dict['num_cv_folds']
exp_cv = ExperimentCrossValidation(final_layer,
dataset,
exp_args=train_dict['exp_args'],
n_folds=n_folds,
shuffle=self._shuffle)
exp_cv.run()
endtime = time.time()
result_or_results = []
for i_fold in xrange(n_folds):
res = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp_cv.all_monitor_chans[i_fold],
predictions=[0, 3, 1, 2, 3, 4],
targets=[3, 4, 1, 2, 3, 4])
result_or_results.append(res)
model = exp_cv.all_layers
if not os.path.exists(self._folder_paths[experiment_index]):
os.makedirs(self._folder_paths[experiment_index])
result_file_name = self._get_result_save_path(experiment_index)
log.info("Saving result to {:s}...".format(result_file_name))
with open(result_file_name, 'w') as resultfile:
pickle.dump(result_or_results, resultfile)
model_file_name = self._get_model_save_path(experiment_index)
param_file_name = model_file_name.replace('.pkl', '.npy')
np.save(param_file_name, lasagne.layers.get_all_param_values(model))
# Possibly make kaggle submission file
if isinstance(dataset,
KaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_one_subject(
self._folder_paths[experiment_index], exp.dataset, iterator,
train_dict['exp_args']['preprocessor'], final_layer,
experiment_save_id)
elif isinstance(
dataset,
AllSubjectsKaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_all_subject_model(
self._folder_paths[experiment_index], exp.dataset,
exp.dataset_provider, iterator, final_layer,
experiment_save_id)
elif isinstance(splitter, SeveralSetsSplitter):
pass # nothing to do in this case
# very hacky create predictions targets :)
# Not done earlier as there were weird theano crashes
if exp.monitors[2].__class__.__name__ == 'CntTrialMisclassMonitor':
del dataset
del exp
add_labels_to_cnt_exp_result(
self._base_save_paths[experiment_index])
def train_hyperopt(params):
""" Runs one fold with given parameters and returns test misclass."""
lasagne.random.set_rng(RandomState(9859295))
template_name = params.pop('template_name')
params = adjust_params_for_hyperopt(params)
config_strings = create_config_strings(template_name)
config_objects = create_config_objects(config_strings)
templates, _ = create_templates_variants_from_config_objects(
config_objects)
processed_templates, params_without_template_params = process_templates(
templates, params)
final_params = process_parameters_by_templates(
params_without_template_params, processed_templates)
# go to directory above this source-file
main_template_filename = os.path.dirname(
os.path.abspath(os.path.dirname(__file__)))
# then complete path to config
main_template_filename = os.path.join(main_template_filename, "configs",
"eegnet_template.yaml")
with (open(main_template_filename, 'r')) as main_template_file:
main_template_str = main_template_file.read()
final_params['original_params'] = 'dummy'
train_str = Template(main_template_str).substitute(final_params)
def do_not_load_constructor(loader, node):
return None
yaml.add_constructor(u'!DoNotLoad', do_not_load_constructor)
modified_train_str = train_str.replace('layers: ', 'layers: !DoNotLoad ')
train_dict = yaml_parse.load(modified_train_str)
dataset = train_dict['dataset']
dataset.load()
dataset_provider = train_dict['dataset_provider']
assert 'in_sensors' in train_str
assert 'in_rows' in train_str
assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dataset.get_topological_view().shape[1]))
train_str = train_str.replace('in_rows',
str(dataset.get_topological_view().shape[2]))
train_str = train_str.replace('in_cols',
str(dataset.get_topological_view().shape[3]))
train_dict = yaml_parse.load(train_str)
layers = train_dict['layers']
final_layer = layers[-1]
# turn off debug/info logging
logging.getLogger("pylearn2").setLevel(logging.WARN)
logging.getLogger("braindecode").setLevel(logging.WARN)
exp = Experiment()
exp.setup(final_layer, dataset_provider, **train_dict['exp_args'])
exp.run()
final_misclass = exp.monitor_chans['test_misclass'][-1]
print("Result for")
pprint(params)
print("Final Test misclass: {:5.4f}".format(float(final_misclass)))
return final_misclass
def test_experiment_fixed_split():
""" Regression test, checking that values have not changed from original run"""
data_rng = RandomState(398765905)
rand_topo = data_rng.rand(200,10,10,3).astype(np.float32)
rand_y = np.int32(data_rng.rand(200) > 0.5)
rand_topo[rand_y == 1] += 0.01
rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y)
lasagne.random.set_rng(RandomState(9859295))
in_layer = InputLayer(shape= [None, 10,10,3])
network = DenseLayer(incoming=in_layer, name="softmax",
num_units=2, nonlinearity=lasagne.nonlinearities.softmax)
updates_modifier = MaxNormConstraint({'softmax': 0.5})
dataset = rand_set
dataset_iterator = BalancedBatchIterator(batch_size=60)
preprocessor = OnlineAxiswiseStandardize (axis=['c', 1])
dataset_splitter=FixedTrialSplitter(n_train_trials=150, valid_set_fraction=0.1)
updates_var_func=lasagne.updates.adam
loss_var_func= lasagne.objectives.categorical_crossentropy
monitors=[braindecode.veganlasagne.monitors.LossMonitor (),
braindecode.veganlasagne.monitors.MisclassMonitor(),
braindecode.veganlasagne.monitors.RuntimeMonitor()]
stop_criterion= braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=30)
exp = Experiment(network, dataset, dataset_splitter, preprocessor,
dataset_iterator, loss_var_func, updates_var_func,
updates_modifier, monitors,
stop_criterion, remember_best_chan='valid_misclass',
run_after_early_stop=True)
exp.setup()
exp.run()
assert np.allclose(
[0.548148, 0.540741, 0.503704, 0.451852, 0.392593, 0.370370,
0.340741, 0.281481, 0.237037, 0.207407, 0.192593, 0.177778,
0.133333, 0.111111, 0.111111, 0.103704, 0.096296, 0.088889,
0.088889, 0.081481, 0.074074, 0.066667, 0.066667, 0.059259,
0.059259, 0.051852, 0.037037, 0.037037, 0.029630, 0.029630,
0.029630, 0.053333, 0.053333, 0.053333, 0.053333, 0.040000,
0.040000, 0.026667, 0.026667, 0.026667, 0.026667, 0.033333,
0.033333, 0.033333, 0.033333, 0.026667, 0.020000, 0.020000,
0.020000],
exp.monitor_chans['train_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[0.400000, 0.400000, 0.400000, 0.400000, 0.400000, 0.400000,
0.400000, 0.400000, 0.333333, 0.333333, 0.333333, 0.266667,
0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.266667,
0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.333333,
0.333333, 0.333333, 0.333333, 0.266667, 0.266667, 0.266667,
0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.200000,
0.200000, 0.133333, 0.133333, 0.133333, 0.133333, 0.133333,
0.133333, 0.133333, 0.133333, 0.066667, 0.000000, 0.000000,
0.000000],
exp.monitor_chans['valid_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[0.460000, 0.420000, 0.420000, 0.420000, 0.420000, 0.440000,
0.420000, 0.420000, 0.400000, 0.400000, 0.380000, 0.400000,
0.400000, 0.400000, 0.400000, 0.400000, 0.420000, 0.420000,
0.420000, 0.400000, 0.400000, 0.400000, 0.380000, 0.380000,
0.380000, 0.380000, 0.400000, 0.400000, 0.420000, 0.420000,
0.420000, 0.420000, 0.420000, 0.420000, 0.420000, 0.420000,
0.400000, 0.400000, 0.380000, 0.400000, 0.400000, 0.400000,
0.400000, 0.400000, 0.360000, 0.360000, 0.380000, 0.380000,
0.380000],
exp.monitor_chans['test_misclass'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[1.200389, 0.777420, 0.740212, 0.705151, 0.672329, 0.641764,
0.613245, 0.586423, 0.561397, 0.538399, 0.517073, 0.497741,
0.479949, 0.463601, 0.448505, 0.434583, 0.421652, 0.409739,
0.398721, 0.388490, 0.378988, 0.370121, 0.361965, 0.354295,
0.347159, 0.340496, 0.334237, 0.328328, 0.322803, 0.317624,
0.312765, 0.340091, 0.335658, 0.330868, 0.325923, 0.320895,
0.316027, 0.311290, 0.306683, 0.302364, 0.298264, 0.294475,
0.290957, 0.287673, 0.284664, 0.281860, 0.279309, 0.276918,
0.274709],
exp.monitor_chans['train_loss'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[0.766092, 0.642237, 0.636960, 0.629884, 0.623676, 0.618789,
0.613821, 0.609264, 0.605430, 0.601499, 0.598178, 0.594579,
0.591720, 0.589461, 0.587571, 0.585673, 0.583782, 0.581606,
0.580687, 0.579677, 0.579276, 0.578903, 0.578918, 0.578901,
0.579020, 0.579575, 0.580291, 0.581120, 0.581591, 0.582552,
0.583647, 0.585879, 0.582269, 0.571548, 0.555956, 0.536982,
0.517474, 0.496652, 0.474400, 0.453094, 0.432208, 0.412533,
0.394271, 0.377036, 0.361311, 0.346461, 0.333406, 0.321266,
0.310158],
exp.monitor_chans['valid_loss'],
rtol=1e-4, atol=1e-4)
assert np.allclose(
[1.069603, 0.751982, 0.746711, 0.742126, 0.738055, 0.734703,
0.731921, 0.729251, 0.727241, 0.724931, 0.723189, 0.721885,
0.720605, 0.719565, 0.718930, 0.718664, 0.718671, 0.718747,
0.719004, 0.718935, 0.719153, 0.719381, 0.719815, 0.720419,
0.721205, 0.721993, 0.722759, 0.723534, 0.724298, 0.724908,
0.725497, 0.725097, 0.725950, 0.726615, 0.726953, 0.727603,
0.728247, 0.728787, 0.729323, 0.729945, 0.730434, 0.731245,
0.732168, 0.732949, 0.734086, 0.735250, 0.736381, 0.737502,
0.738444],
exp.monitor_chans['test_loss'],
rtol=1e-4, atol=1e-4)
def test_experiment_fixed_split():
""" Regression test, checking that values have not changed from original run"""
data_rng = RandomState(398765905)
rand_topo = data_rng.rand(200, 10, 10, 3).astype(np.float32)
rand_y = np.int32(data_rng.rand(200) > 0.5)
rand_topo[rand_y == 1] += 0.01
rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y)
lasagne.random.set_rng(RandomState(9859295))
in_layer = InputLayer(shape=[None, 10, 10, 3])
network = DenseLayer(incoming=in_layer,
name="softmax",
num_units=2,
nonlinearity=lasagne.nonlinearities.softmax)
updates_modifier = MaxNormConstraint({'softmax': 0.5})
dataset = rand_set
dataset_iterator = BalancedBatchIterator(batch_size=60)
preprocessor = OnlineAxiswiseStandardize(axis=['c', 1])
dataset_splitter = FixedTrialSplitter(n_train_trials=150,
valid_set_fraction=0.1)
updates_var_func = lasagne.updates.adam
loss_var_func = lasagne.objectives.categorical_crossentropy
monitors = [
braindecode.veganlasagne.monitors.LossMonitor(),
braindecode.veganlasagne.monitors.MisclassMonitor(),
braindecode.veganlasagne.monitors.RuntimeMonitor()
]
stop_criterion = braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=30)
exp = Experiment(network,
dataset,
dataset_splitter,
preprocessor,
dataset_iterator,
loss_var_func,
updates_var_func,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan='valid_misclass',
run_after_early_stop=True)
exp.setup()
exp.run()
assert np.allclose([
0.548148, 0.540741, 0.503704, 0.451852, 0.392593, 0.370370, 0.340741,
0.281481, 0.237037, 0.207407, 0.192593, 0.177778, 0.133333, 0.111111,
0.111111, 0.103704, 0.096296, 0.088889, 0.088889, 0.081481, 0.074074,
0.066667, 0.066667, 0.059259, 0.059259, 0.051852, 0.037037, 0.037037,
0.029630, 0.029630, 0.029630, 0.053333, 0.053333, 0.053333, 0.053333,
0.040000, 0.040000, 0.026667, 0.026667, 0.026667, 0.026667, 0.033333,
0.033333, 0.033333, 0.033333, 0.026667, 0.020000, 0.020000, 0.020000
],
exp.monitor_chans['train_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose([
0.400000, 0.400000, 0.400000, 0.400000, 0.400000, 0.400000, 0.400000,
0.400000, 0.333333, 0.333333, 0.333333, 0.266667, 0.266667, 0.266667,
0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.266667,
0.266667, 0.266667, 0.333333, 0.333333, 0.333333, 0.333333, 0.266667,
0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.266667, 0.266667,
0.200000, 0.200000, 0.133333, 0.133333, 0.133333, 0.133333, 0.133333,
0.133333, 0.133333, 0.133333, 0.066667, 0.000000, 0.000000, 0.000000
],
exp.monitor_chans['valid_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose([
0.460000, 0.420000, 0.420000, 0.420000, 0.420000, 0.440000, 0.420000,
0.420000, 0.400000, 0.400000, 0.380000, 0.400000, 0.400000, 0.400000,
0.400000, 0.400000, 0.420000, 0.420000, 0.420000, 0.400000, 0.400000,
0.400000, 0.380000, 0.380000, 0.380000, 0.380000, 0.400000, 0.400000,
0.420000, 0.420000, 0.420000, 0.420000, 0.420000, 0.420000, 0.420000,
0.420000, 0.400000, 0.400000, 0.380000, 0.400000, 0.400000, 0.400000,
0.400000, 0.400000, 0.360000, 0.360000, 0.380000, 0.380000, 0.380000
],
exp.monitor_chans['test_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose([
1.200389, 0.777420, 0.740212, 0.705151, 0.672329, 0.641764, 0.613245,
0.586423, 0.561397, 0.538399, 0.517073, 0.497741, 0.479949, 0.463601,
0.448505, 0.434583, 0.421652, 0.409739, 0.398721, 0.388490, 0.378988,
0.370121, 0.361965, 0.354295, 0.347159, 0.340496, 0.334237, 0.328328,
0.322803, 0.317624, 0.312765, 0.340091, 0.335658, 0.330868, 0.325923,
0.320895, 0.316027, 0.311290, 0.306683, 0.302364, 0.298264, 0.294475,
0.290957, 0.287673, 0.284664, 0.281860, 0.279309, 0.276918, 0.274709
],
exp.monitor_chans['train_loss'],
rtol=1e-4,
atol=1e-4)
assert np.allclose([
0.766092, 0.642237, 0.636960, 0.629884, 0.623676, 0.618789, 0.613821,
0.609264, 0.605430, 0.601499, 0.598178, 0.594579, 0.591720, 0.589461,
0.587571, 0.585673, 0.583782, 0.581606, 0.580687, 0.579677, 0.579276,
0.578903, 0.578918, 0.578901, 0.579020, 0.579575, 0.580291, 0.581120,
0.581591, 0.582552, 0.583647, 0.585879, 0.582269, 0.571548, 0.555956,
0.536982, 0.517474, 0.496652, 0.474400, 0.453094, 0.432208, 0.412533,
0.394271, 0.377036, 0.361311, 0.346461, 0.333406, 0.321266, 0.310158
],
exp.monitor_chans['valid_loss'],
rtol=1e-4,
atol=1e-4)
assert np.allclose([
1.069603, 0.751982, 0.746711, 0.742126, 0.738055, 0.734703, 0.731921,
0.729251, 0.727241, 0.724931, 0.723189, 0.721885, 0.720605, 0.719565,
0.718930, 0.718664, 0.718671, 0.718747, 0.719004, 0.718935, 0.719153,
0.719381, 0.719815, 0.720419, 0.721205, 0.721993, 0.722759, 0.723534,
0.724298, 0.724908, 0.725497, 0.725097, 0.725950, 0.726615, 0.726953,
0.727603, 0.728247, 0.728787, 0.729323, 0.729945, 0.730434, 0.731245,
0.732168, 0.732949, 0.734086, 0.735250, 0.736381, 0.737502, 0.738444
],
exp.monitor_chans['test_loss'],
rtol=1e-4,
atol=1e-4)
def run(ex, data_folder, subject_id, n_chans, clean_train,
low_cut_hz, train_start_ms,kappa_mode, loss_expression,
network,
filt_order,
only_return_exp,):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
valid_subject_id = subject_
other_subject_ids = range(1,subject_id) + range(subject_id+1, 10)
other_sets = [create_dataset(
data_folder, other_sid, train_start_ms, low_cut_hz,
filt_order, clean_train) for other_sid in other_subject_ids]
test_set = create_dataset(
data_folder, subject_id, train_start_ms, low_cut_hz,
filt_order, clean_train)
combined_set = other_sets + [test_set]
def merge_train_test(single_combined_set):
return concatenate_sets(single_combined_set.train_set,
single_combined_set.test_set)
if not only_return_exp:
for i_set, this_set in enumerate(combined_set):
log.info("Loading {:d} of {:d}".format(i_set + 1,
len(combined_set)))
this_set.load()
merged_sets = [merge_train_test(s) for s in combined_set]
combined_set = CombinedSet(merged_sets)
in_chans = merged_sets[0].get_topological_view().shape[1]
else:
in_chans = n_chans
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
if network == 'deep':
final_layer = create_deep_net(in_chans, input_time_length)
else:
assert network == 'shallow'
final_layer = create_shallow_net(in_chans, input_time_length)
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.1, use_test_as_valid=False)
iterator = CntWindowTrialIterator(batch_size=45,input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length, mode=kappa_mode),
RuntimeMonitor(),]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800#100
# real values for paper were 80 and 800
stop_criterion = Or([NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop=True
exp = Experiment(final_layer, dataset,splitter,None,iterator, loss_expression,updates_expression, updates_modifier, monitors,
stop_criterion, remember_best_chan, run_after_early_stop, batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
save_npy_artifact(ex, lasagne.layers.get_all_param_values(exp.final_layer),
'model_params.npy')
def test_experiment_sample_windows():
data_rng = RandomState(398765905)
rand_topo = data_rng.rand(200, 10, 10, 3).astype(np.float32)
rand_y = np.int32(data_rng.rand(200) > 0.5)
rand_topo[rand_y == 1] += 0.1
rand_set = DenseDesignMatrixWrapper(topo_view=rand_topo, y=rand_y)
lasagne.random.set_rng(RandomState(9859295))
in_layer = InputLayer(shape=[None, 10, 5, 3])
network = DenseLayer(incoming=in_layer,
name='softmax',
num_units=2,
nonlinearity=lasagne.nonlinearities.softmax)
updates_modifier = MaxNormConstraint({'softmax': 0.5})
dataset = rand_set
dataset_iterator = WindowsIterator(n_samples_per_window=5, batch_size=60)
preprocessor = OnlineAxiswiseStandardize(axis=['c', 1])
dataset_splitter = FixedTrialSplitter(n_train_trials=150,
valid_set_fraction=0.1)
updates_var_func = lasagne.updates.adam
loss_var_func = lasagne.objectives.categorical_crossentropy
monitors = [
braindecode.veganlasagne.monitors.LossMonitor(),
braindecode.veganlasagne.monitors.WindowMisclassMonitor(),
braindecode.veganlasagne.monitors.RuntimeMonitor()
]
stop_criterion = braindecode.veganlasagne.stopping.MaxEpochs(num_epochs=5)
exp = Experiment(network,
dataset,
dataset_splitter,
preprocessor,
dataset_iterator,
loss_var_func,
updates_var_func,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan='valid_misclass',
run_after_early_stop=True)
exp.setup()
exp.run()
assert np.allclose(
[0.629630, 0.140741, 0.029630, 0.022222, 0.000000, 0.000000, 0.000000],
exp.monitor_chans['train_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose(
[0.400000, 0.133333, 0.066667, 0.000000, 0.000000, 0.000000, 0.000000],
exp.monitor_chans['valid_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose(
[0.560000, 0.060000, 0.000000, 0.000000, 0.000000, 0.000000, 0.000000],
exp.monitor_chans['test_misclass'],
rtol=1e-4,
atol=1e-4)
assert np.allclose(
[1.180485, 0.574264, 0.420023, 0.330909, 0.278569, 0.245692, 0.242845],
exp.monitor_chans['train_loss'],
rtol=1e-4,
atol=1e-4)
assert np.allclose(
[1.016782, 0.514049, 0.370485, 0.288948, 0.240913, 0.211189, 0.215967],
exp.monitor_chans['valid_loss'],
rtol=1e-4,
atol=1e-4)
assert np.allclose(
[1.031832, 0.504570, 0.352317, 0.269810, 0.223904, 0.196681, 0.197899],
exp.monitor_chans['test_loss'],
rtol=1e-4,
atol=1e-4)
def run(
ex,
data_folder,
subject_id,
n_chans,
clean_train,
low_cut_hz,
train_start_ms,
kappa_mode,
loss_expression,
filt_order,
only_return_exp,
):
start_time = time.time()
assert (only_return_exp is False) or (n_chans is not None)
ex.info['finished'] = False
load_sensor_names = None
train_filename = 'A{:02d}T.mat'.format(subject_id)
test_filename = 'A{:02d}E.mat'.format(subject_id)
train_filepath = os.path.join(data_folder, train_filename)
test_filepath = os.path.join(data_folder, test_filename)
# trial ivan in milliseconds
# these are the samples that will be predicted, so for a
# network with 2000ms receptive field
# 1500 means the first receptive field goes from -500 to 1500
train_segment_ival = [train_start_ms, 4000]
test_segment_ival = [0, 4000]
train_loader = BCICompetition4Set2A(train_filepath,
load_sensor_names=load_sensor_names)
test_loader = BCICompetition4Set2A(test_filepath,
load_sensor_names=load_sensor_names)
# Preprocessing pipeline in [(function, {args:values)] logic
cnt_preprocessors = [(resample_cnt, {
'newfs': 250.0
}),
(bandpass_cnt, {
'low_cut_hz': low_cut_hz,
'high_cut_hz': 38,
'filt_order': filt_order,
}), (exponential_standardize_cnt, {})]
marker_def = {
'1- Right Hand': [1],
'2 - Left Hand': [2],
'3 - Rest': [3],
'4 - Feet': [4]
}
train_signal_proc = SignalProcessor(set_loader=train_loader,
segment_ival=train_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
train_set = CntSignalMatrix(signal_processor=train_signal_proc,
sensor_names='all')
test_signal_proc = SignalProcessor(set_loader=test_loader,
segment_ival=test_segment_ival,
cnt_preprocessors=cnt_preprocessors,
marker_def=marker_def)
test_set = CntSignalMatrix(signal_processor=test_signal_proc,
sensor_names='all')
if clean_train:
train_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
else:
train_cleaner = NoCleaner()
test_cleaner = BCICompetitionIV2ABArtefactMaskCleaner(
marker_def=marker_def)
combined_set = CombinedCleanedSet(train_set, test_set, train_cleaner,
test_cleaner)
if not only_return_exp:
combined_set.load()
in_chans = train_set.get_topological_view().shape[1]
else:
in_chans = n_chans
input_time_length = 1000 # implies how many crops are processed in parallel, does _not_ determine receptive field size
# receptive field size is determined by model architecture
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
final_layer = create_deep_net(in_chans, input_time_length)
print_layers(final_layer)
dataset_splitter = SeveralSetsSplitter(valid_set_fraction=0.2,
use_test_as_valid=False)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length,
mode=kappa_mode),
RuntimeMonitor(),
]
#debug: n_no_decrease_max_epochs = 2
#debug: n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800 #100
# real values for paper were 80 and 800
stop_criterion = Or([
NoDecrease('valid_misclass', num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
remember_best_chan = 'valid_misclass'
run_after_early_stop = True
exp = Experiment(final_layer,
dataset,
splitter,
None,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
end_time = time.time()
run_time = end_time - start_time
ex.info['finished'] = True
for key in exp.monitor_chans:
ex.info[key] = exp.monitor_chans[key][-1]
ex.info['runtime'] = run_time
save_pkl_artifact(ex, exp.monitor_chans, 'monitor_chans.pkl')
save_npy_artifact(ex, lasagne.layers.get_all_param_values(exp.final_layer),
'model_params.npy')
def run_exp(i_fold):
# ensure reproducibility by resetting lasagne/theano random generator
lasagne.random.set_rng(RandomState(34734))
d5net = Deep5Net(in_chans=in_chans,
input_time_length=input_time_length,
num_filters_time=num_filters_time,
filter_time_length=filter_time_length,
num_filters_spat=num_filters_spat,
pool_time_length=pool_time_length,
pool_time_stride=pool_time_stride,
num_filters_2=num_filters_2,
filter_length_2=filter_length_2,
num_filters_3=num_filters_3,
filter_length_3=filter_length_3,
num_filters_4=num_filters_4,
filter_length_4=filter_length_4,
final_dense_length=final_dense_length,
n_classes=n_classes,
final_nonlin=final_nonlin,
first_nonlin=first_nonlin,
first_pool_mode=first_pool_mode,
first_pool_nonlin=first_pool_nonlin,
later_nonlin=later_nonlin,
later_pool_mode=later_pool_mode,
later_pool_nonlin=later_pool_nonlin,
drop_in_prob=drop_in_prob,
drop_prob=drop_prob,
batch_norm_alpha=batch_norm_alpha,
double_time_convs=double_time_convs,
split_first_layer=split_first_layer,
batch_norm=batch_norm)
final_layer = d5net.get_layers()[-1]
final_layer = ClipLayer(final_layer, 1e-4, 1 - 1e-4)
dataset_splitter = CntTrialSingleFoldSplitter(n_folds=10,
i_test_fold=i_fold,
shuffle=True)
iterator = CntWindowTrialIterator(
batch_size=45,
input_time_length=input_time_length,
n_sample_preds=get_n_sample_preds(final_layer))
monitors = [
LossMonitor(),
CntTrialMisclassMonitor(input_time_length=input_time_length),
KappaMonitor(input_time_length=iterator.input_time_length,
mode='max'),
RuntimeMonitor()
]
#n_no_decrease_max_epochs = 2
#n_max_epochs = 4
n_no_decrease_max_epochs = 80
n_max_epochs = 800
# real values for paper were 80 and 800
remember_best_chan = 'valid_' + stop_chan
stop_criterion = Or([
NoDecrease(remember_best_chan,
num_epochs=n_no_decrease_max_epochs),
MaxEpochs(num_epochs=n_max_epochs)
])
dataset = combined_set
splitter = dataset_splitter
updates_expression = adam
updates_modifier = MaxNormConstraintWithDefaults({})
preproc = None
exp = Experiment(final_layer,
dataset,
splitter,
preproc,
iterator,
loss_expression,
updates_expression,
updates_modifier,
monitors,
stop_criterion,
remember_best_chan,
run_after_early_stop,
batch_modifier=None)
if only_return_exp:
return exp
exp.setup()
exp.run()
return exp
def _run_experiments_with_string(self, experiment_index, train_str):
assert experiment_index >= self._get_start_id()
assert experiment_index < self._get_stop_id()
lasagne.random.set_rng(RandomState(9859295))
# Save train string now, will be overwritten later after
# input dimensions determined, save now for debug in
# case of crash
if not self._dry_run:
self._save_train_string(train_str, experiment_index)
starttime = time.time()
train_dict = self._load_without_layers(train_str)
log.info("With params...")
if not self._quiet:
pprint(train_dict['original_params'])
if self._dry_run:
# Do not do the loading or training...
# Only go until here to show the train params
return
if self._batch_test:
# TODO: put into function
# load layers, load data with dimensions of the layer
# create experiment with max epochs 2, run
from braindecode.datasets.random import RandomSet
train_str = train_str.replace('in_cols', '1')
train_str = train_str.replace('in_sensors', '32')
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
n_chans = layers[0].shape[1]
n_classes = final_layer.output_shape[1]
n_samples = 500000
# set n sample perds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
# make at least batches
n_samples = int(n_sample_preds * 1.5 * 200)
dataset = RandomSet(topo_shape=[n_samples, n_chans, 1, 1],
y_shape=[n_samples, n_classes])
dataset.load()
splitter = FixedTrialSplitter(n_train_trials=int(n_samples*0.8),
valid_set_fraction=0.1)
train_dict['exp_args']['preprocessor'] = None
train_dict['exp_args']['stop_criterion'] = MaxEpochs(1)
train_dict['exp_args']['iterator'].batch_size = 1
# TODO: set stop criterion to max epochs =1
# change batch_size in iterator
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run_until_early_stop()
datasets = exp.dataset_provider.get_train_valid_test(exp.dataset)
for batch_size in range(32,200,5):
train_dict['exp_args']['stop_criterion'].num_epochs += 2
log.info("Running with batch size {:d}".format(batch_size))
train_dict['exp_args']['iterator'].batch_size = batch_size
exp.run_until_stop(datasets, remember_best=False)
return
dataset = train_dict['dataset']
dataset.load()
iterator = train_dict['exp_args']['iterator']
splitter = train_dict['dataset_splitter']
if dataset.__class__.__name__ == 'EpilepsySet':
log.info("Reducing to float16 for epilepsy set...")
dataset.seizure_topo = np.float16(dataset.seizure_topo)
dataset.non_seizure_topo = np.float16(dataset.non_seizure_topo)
else:
# todo: remove this?
log.info("Determining dataset dimensions to set possible model params...")
train_set = splitter.split_into_train_valid_test(dataset)['train']
batch_gen = iterator.get_batches(train_set, shuffle=True)
dummy_batch_topo = batch_gen.next()[0]
del train_set
# not for ultrasound: assert 'in_sensors' in train_str
# not for cnt net assert 'in_rows' in train_str
# not for resnet: assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dummy_batch_topo.shape[1]))
train_str = train_str.replace('in_rows',
str(dummy_batch_topo.shape[2]))
train_str = train_str.replace('in_cols',
str(dummy_batch_topo.shape[3]))
self._save_train_string(train_str, experiment_index)
# reset rng for actual loading of layers, so you can reproduce it
# when you load the file later
lasagne.random.set_rng(RandomState(9859295))
train_dict = yaml_parse.load(train_str)
layers = load_layers_from_dict(train_dict)
final_layer = layers[-1]
assert len(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer))) == 0, ("All layers "
"should be used, unused {:s}".format(str(np.setdiff1d(layers,
lasagne.layers.get_all_layers(final_layer)))))
# Set n sample preds in case of cnt model
if (np.any([hasattr(l, 'n_stride') for l in layers])):
# Can this be moved up and duplication in if clause( batch test,
# more above) be removed?
n_sample_preds = get_n_sample_preds(final_layer)
log.info("Setting n_sample preds automatically to {:d}".format(
n_sample_preds))
for monitor in train_dict['exp_args']['monitors']:
if hasattr(monitor, 'n_sample_preds'):
monitor.n_sample_preds = n_sample_preds
train_dict['exp_args']['iterator'].n_sample_preds = n_sample_preds
log.info("Input window length is {:d}".format(
get_model_input_window(final_layer)))
if not self._cross_validation:
# for now lets not do that, current models seem fine again.
# if (dataset.__class__.__name__ == 'EpilepsySet') and self._pred_loss_hack:
# from braindecode.epilepsy.experiment import EpilepsyExperiment
# log.info("Creating epilepsy experiment with the pred loss hack")
# exp = EpilepsyExperiment(final_layer, dataset, splitter,
# **train_dict['exp_args'])
# else:
exp = Experiment(final_layer, dataset, splitter,
**train_dict['exp_args'])
exp.setup()
exp.run()
endtime = time.time()
model = exp.final_layer
# dummy predictions targets
predictions = [0,3,1,2,3,4]
targets = [3,4,1,2,3,4]
result_or_results = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp.monitor_chans,
predictions=predictions,
targets=targets)
else: # cross validation
assert False, ("cross validation not used in long time, not up to date"
" for example targets predictions not added")
# default 5 folds for now
n_folds = train_dict['num_cv_folds']
exp_cv = ExperimentCrossValidation(final_layer,
dataset, exp_args=train_dict['exp_args'], n_folds=n_folds,
shuffle=self._shuffle)
exp_cv.run()
endtime = time.time()
result_or_results = []
for i_fold in xrange(n_folds):
res = Result(parameters=train_dict['original_params'],
templates={},
training_time=endtime - starttime,
monitor_channels=exp_cv.all_monitor_chans[i_fold],
predictions=[0,3,1,2,3,4],
targets=[3,4,1,2,3,4])
result_or_results.append(res)
model = exp_cv.all_layers
if not os.path.exists(self._folder_paths[experiment_index]):
os.makedirs(self._folder_paths[experiment_index])
result_file_name = self._get_result_save_path(experiment_index)
log.info("Saving result...")
with open(result_file_name, 'w') as resultfile:
pickle.dump(result_or_results, resultfile)
model_file_name = self._get_model_save_path(experiment_index)
param_file_name = model_file_name.replace('.pkl', '.npy')
np.save(param_file_name, lasagne.layers.get_all_param_values(model))
# Possibly make kaggle submission file
if isinstance(dataset, KaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_one_subject(self._folder_paths[experiment_index],
exp.dataset, iterator,
train_dict['exp_args']['preprocessor'],
final_layer, experiment_save_id)
elif isinstance(dataset, AllSubjectsKaggleGraspLiftSet) and splitter.use_test_as_valid:
experiment_save_id = int(
self._base_save_paths[experiment_index].split("/")[-1])
create_submission_csv_for_all_subject_model(
self._folder_paths[experiment_index],
exp.dataset, exp.dataset_provider, iterator,
final_layer, experiment_save_id)
elif isinstance(splitter, SeveralSetsSplitter):
pass # nothing to do in this case
# very hacky create predictions targets :)
# Not done earlier as there were weird theano crashes
if exp.monitors[2].__class__.__name__ == 'CntTrialMisclassMonitor':
del dataset
del exp
add_labels_to_cnt_exp_result(self._base_save_paths[experiment_index])
def train_hyperopt(params):
""" Runs one fold with given parameters and returns test misclass."""
lasagne.random.set_rng(RandomState(9859295))
template_name = params.pop('template_name')
params = adjust_params_for_hyperopt(params)
config_strings = create_config_strings(template_name)
config_objects = create_config_objects(config_strings)
templates, _ = create_templates_variants_from_config_objects(
config_objects)
processed_templates, params_without_template_params = process_templates(
templates, params)
final_params = process_parameters_by_templates(params_without_template_params,
processed_templates)
# go to directory above this source-file
main_template_filename = os.path.dirname(os.path.abspath(os.path.dirname(
__file__)))
# then complete path to config
main_template_filename = os.path.join(main_template_filename, "configs",
"eegnet_template.yaml")
with (open(main_template_filename, 'r')) as main_template_file:
main_template_str = main_template_file.read()
final_params['original_params'] = 'dummy'
train_str = Template(main_template_str).substitute(final_params)
def do_not_load_constructor(loader, node):
return None
yaml.add_constructor(u'!DoNotLoad', do_not_load_constructor)
modified_train_str = train_str.replace('layers: ', 'layers: !DoNotLoad ')
train_dict = yaml_parse.load(modified_train_str)
dataset = train_dict['dataset']
dataset.load()
dataset_provider = train_dict['dataset_provider']
assert 'in_sensors' in train_str
assert 'in_rows' in train_str
assert 'in_cols' in train_str
train_str = train_str.replace('in_sensors',
str(dataset.get_topological_view().shape[1]))
train_str = train_str.replace('in_rows',
str(dataset.get_topological_view().shape[2]))
train_str = train_str.replace('in_cols',
str(dataset.get_topological_view().shape[3]))
train_dict = yaml_parse.load(train_str)
layers = train_dict['layers']
final_layer = layers[-1]
# turn off debug/info logging
logging.getLogger("pylearn2").setLevel(logging.WARN)
logging.getLogger("braindecode").setLevel(logging.WARN)
exp = Experiment()
exp.setup(final_layer, dataset_provider, **train_dict['exp_args'])
exp.run()
final_misclass = exp.monitor_chans['test_misclass'][-1]
print("Result for")
pprint(params)
print("Final Test misclass: {:5.4f}".format(float(final_misclass)))
return final_misclass
