def network_model(model, train_set, test_set, valid_set, n_chans, input_time_length, cuda): max_epochs = 30 max_increase_epochs = 10 batch_size = 64 init_block_size = 1000 set_random_seeds(seed=20190629, cuda=cuda) n_classes = 2 n_chans = n_chans input_time_length = input_time_length if model == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() elif model == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() if cuda: model.cuda() log.info("%s model: ".format(str(model))) optimizer = AdamW(model.parameters(), lr=0.00625, weight_decay=0) iterator = BalancedBatchSizeIterator(batch_size=batch_size) stop_criterion = Or([MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs)]) monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()] model_constraint = None print(train_set.X.shape[0]) model_test = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=F.nll_loss, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, cuda=cuda) model_test.run() return model_test
def create_model(n_classes, input_time_length, in_chans=22): set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=4, pool_time_length=20, pool_time_stride=5).create_network() to_dense_prediction_model(model) if cuda: model.cuda() return model
def run_exp(test_on_eval, sensor_types, n_chans, max_recording_mins, test_recording_mins, n_recordings, sec_to_cut_at_start, sec_to_cut_at_end, duration_recording_mins, max_abs_val, clip_before_resample, sampling_freq, divisor, n_folds, i_test_fold, shuffle, merge_train_valid, model_name, n_start_chans, n_chan_factor, input_time_length, final_conv_length, stride_before_pool, optimizer, learning_rate, weight_decay, scheduler, model_constraint, batch_size, max_epochs, log_dir, only_return_exp, np_th_seed): cuda = True if ('smac' in model_name) and (input_time_length is None): input_time_length = 12000 fix_input_length_for_smac = True else: fix_input_length_for_smac = False set_random_seeds(seed=np_th_seed, cuda=cuda) n_classes = 2 if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net( n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, stride_before_pool=stride_before_pool).create_network() elif (model_name == 'deep_smac') or (model_name == 'deep_smac_bnorm'): if model_name == 'deep_smac': do_batch_norm = False else: assert model_name == 'deep_smac_bnorm' do_batch_norm = True double_time_convs = False drop_prob = 0.244445 filter_length_2 = 12 filter_length_3 = 14 filter_length_4 = 12 filter_time_length = 21 final_conv_length = 1 first_nonlin = elu first_pool_mode = 'mean' first_pool_nonlin = identity later_nonlin = elu later_pool_mode = 'mean' later_pool_nonlin = identity n_filters_factor = 1.679066 n_filters_start = 32 pool_time_length = 1 pool_time_stride = 2 split_first_layer = True n_chan_factor = n_filters_factor n_start_chans = n_filters_start model = Deep4Net(n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, batch_norm=do_batch_norm, double_time_convs=double_time_convs, drop_prob=drop_prob, filter_length_2=filter_length_2, filter_length_3=filter_length_3, filter_length_4=filter_length_4, filter_time_length=filter_time_length, first_nonlin=first_nonlin, first_pool_mode=first_pool_mode, first_pool_nonlin=first_pool_nonlin, later_nonlin=later_nonlin, later_pool_mode=later_pool_mode, later_pool_nonlin=later_pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, stride_before_pool=True).create_network() elif model_name == 'shallow_smac': conv_nonlin = identity do_batch_norm = True drop_prob = 0.328794 filter_time_length = 56 final_conv_length = 22 n_filters_spat = 73 n_filters_time = 24 pool_mode = 'max' pool_nonlin = identity pool_time_length = 84 pool_time_stride = 3 split_first_layer = True model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_filters_time, n_filters_spat=n_filters_spat, input_time_length=input_time_length, final_conv_length=final_conv_length, conv_nonlin=conv_nonlin, batch_norm=do_batch_norm, drop_prob=drop_prob, filter_time_length=filter_time_length, pool_mode=pool_mode, pool_nonlin=pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, ).create_network() elif model_name == 'deep_smac_new': from torch.nn.functional import elu, relu, relu6, tanh from braindecode.torch_ext.functions import identity, square, safe_log n_filters_factor = 1.9532637176784269 n_filters_start = 61 deep_kwargs = { "batch_norm": False, "double_time_convs": False, "drop_prob": 0.3622676569047184, "filter_length_2": 9, "filter_length_3": 6, "filter_length_4": 10, "filter_time_length": 17, "final_conv_length": 5, "first_nonlin": elu, "first_pool_mode": "max", "first_pool_nonlin": identity, "later_nonlin": relu6, "later_pool_mode": "max", "later_pool_nonlin": identity, "n_filters_time": n_filters_start, "n_filters_spat": n_filters_start, "n_filters_2": int(n_filters_start * n_filters_factor), "n_filters_3": int(n_filters_start * (n_filters_factor**2.0)), "n_filters_4": int(n_filters_start * (n_filters_factor**3.0)), "pool_time_length": 1, "pool_time_stride": 4, "split_first_layer": True, "stride_before_pool": True, } model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, **deep_kwargs).create_network() elif model_name == 'shallow_smac_new': from torch.nn.functional import elu, relu, relu6, tanh from braindecode.torch_ext.functions import identity, square, safe_log shallow_kwargs = { "conv_nonlin": square, "batch_norm": True, "drop_prob": 0.10198630723385381, "filter_time_length": 51, "final_conv_length": 1, "n_filters_spat": 200, "n_filters_time": 76, "pool_mode": "max", "pool_nonlin": safe_log, "pool_time_length": 139, "pool_time_stride": 49, "split_first_layer": True, } model = ShallowFBCSPNet(in_chans=n_chans, n_classes=n_classes, input_time_length=input_time_length, **shallow_kwargs).create_network() elif model_name == 'linear': model = nn.Sequential() model.add_module("conv_classifier", nn.Conv2d(n_chans, n_classes, (600, 1))) model.add_module('softmax', nn.LogSoftmax()) model.add_module('squeeze', Expression(lambda x: x.squeeze(3))) elif model_name == '3path': virtual_chan_1x1_conv = True mean_across_features = False drop_prob = 0.5 n_start_filters = 10 early_bnorm = False n_classifier_filters = 100 later_kernel_len = 5 extra_conv_stride = 4 # dont forget to reset n_preds_per_blabla model = create_multi_start_path_net( in_chans=n_chans, virtual_chan_1x1_conv=virtual_chan_1x1_conv, n_start_filters=n_start_filters, early_bnorm=early_bnorm, later_kernel_len=later_kernel_len, extra_conv_stride=extra_conv_stride, mean_across_features=mean_across_features, n_classifier_filters=n_classifier_filters, drop_prob=drop_prob) else: assert False, "unknown model name {:s}".format(model_name) if not model_name == '3path': to_dense_prediction_model(model) log.info("Model:\n{:s}".format(str(model))) time_cut_off_sec = np.inf start_time = time.time() # fix input time length in case of smac models if fix_input_length_for_smac: assert ('smac' in model_name) and (input_time_length == 12000) if cuda: model.cuda() test_input = np_to_var( np.ones((2, n_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() try: out = model(test_input) except: raise ValueError("Model receptive field too large...") n_preds_per_input = out.cpu().data.numpy().shape[2] n_receptive_field = input_time_length - n_preds_per_input input_time_length = 2 * n_receptive_field exp = common.run_exp( max_recording_mins, n_recordings, sec_to_cut_at_start, sec_to_cut_at_end, duration_recording_mins, max_abs_val, clip_before_resample, sampling_freq, divisor, n_folds, i_test_fold, shuffle, merge_train_valid, model, input_time_length, optimizer, learning_rate, weight_decay, scheduler, model_constraint, batch_size, max_epochs, only_return_exp, time_cut_off_sec, start_time, test_on_eval, test_recording_mins, sensor_types, log_dir, np_th_seed, ) return exp
def run_exp( debug, subject_id, max_epochs, n_sensors, final_hz, half_before, start_ms, stop_ms, model, weight_decay, final_fft, add_bnorm, act_norm, ): model_name = model del model assert final_hz in [64, 256] car = not debug train_inputs, test_inputs = load_train_test( subject_id, car, n_sensors, final_hz, start_ms, stop_ms, half_before, only_load_given_sensors=debug, ) cuda = True if cuda: train_inputs = [i.cuda() for i in train_inputs] test_inputs = [i.cuda() for i in test_inputs] from braindecode.datautil.signal_target import SignalAndTarget sets = [] for inputs in (train_inputs, test_inputs): X = np.concatenate([var_to_np(ins) for ins in inputs]).astype( np.float32 ) y = np.concatenate( [np.ones(len(ins)) * i_class for i_class, ins in enumerate(inputs)] ) y = y.astype(np.int64) set = SignalAndTarget(X, y) sets.append(set) train_set = sets[0] valid_set = sets[1] from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from braindecode.models.deep4 import Deep4Net from torch import nn from braindecode.torch_ext.util import set_random_seeds set_random_seeds(2019011641, cuda) n_chans = train_inputs[0].shape[1] n_time = train_inputs[0].shape[2] n_classes = 2 input_time_length=train_set.X.shape[2] if model_name == 'shallow': # final_conv_length = auto ensures we only get a single output in the time dimension model = ShallowFBCSPNet(in_chans=n_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length='auto') elif model_name == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=train_set.X.shape[2], pool_time_length=2, pool_time_stride=2, final_conv_length='auto') elif model_name == 'invertible': model = InvertibleModel(n_chans, n_time, final_fft=final_fft, add_bnorm=add_bnorm) elif model_name == 'deep_invertible': n_chan_pad = 0 filter_length_time = 11 model = deep_invertible( n_chans, input_time_length, n_chan_pad, filter_length_time) model.add_module("select_dims", Expression(lambda x: x[:, :2, 0])) model.add_module("softmax", nn.LogSoftmax(dim=1)) model = WrappedModel(model) ## set scale if act_norm: model.cuda() for module in model.network.modules(): if hasattr(module, 'log_factor'): module._forward_hooks.clear() module.register_forward_hook(scale_to_unit_var) model.network(train_inputs[0].cuda()); for module in model.network.modules(): if hasattr(module, 'log_factor'): module._forward_hooks.clear() else: assert False if cuda: model.cuda() from braindecode.torch_ext.optimizers import AdamW import torch.nn.functional as F if model_name == 'shallow': assert weight_decay == 'hardcoded' optimizer = AdamW(model.parameters(), lr=0.0625 * 0.01, weight_decay=0) elif model_name == 'deep': assert weight_decay == 'hardcoded' optimizer = AdamW(model.parameters(), lr=1 * 0.01, weight_decay=0.5 * 0.001) # these are good values for the deep model elif model_name == 'invertible': optimizer = AdamW(model.parameters(), lr=1e-4, weight_decay=weight_decay) elif model_name == 'deep_invertible': optimizer = AdamW(model.parameters(), lr=1 * 0.001, weight_decay=weight_decay) else: assert False model.compile(loss=F.nll_loss, optimizer=optimizer, iterator_seed=1, ) model.fit(train_set.X, train_set.y, epochs=max_epochs, batch_size=64, scheduler='cosine', validation_data=(valid_set.X, valid_set.y), ) return model.epochs_df, model.network
def run_exp( data_folders, n_recordings, sensor_types, n_chans, max_recording_mins, sec_to_cut, duration_recording_mins, test_recording_mins, max_abs_val, sampling_freq, divisor, test_on_eval, n_folds, i_test_fold, shuffle, model_name, n_start_chans, n_chan_factor, input_time_length, final_conv_length, model_constraint, init_lr, batch_size, max_epochs, cuda, ): import torch.backends.cudnn as cudnn cudnn.benchmark = True preproc_functions = [] preproc_functions.append(lambda data, fs: ( data[:, int(sec_to_cut * fs):-int(sec_to_cut * fs)], fs)) preproc_functions.append(lambda data, fs: (data[:, :int( duration_recording_mins * 60 * fs)], fs)) if max_abs_val is not None: preproc_functions.append( lambda data, fs: (np.clip(data, -max_abs_val, max_abs_val), fs)) preproc_functions.append(lambda data, fs: (resampy.resample( data, fs, sampling_freq, axis=1, filter='kaiser_fast'), sampling_freq)) if divisor is not None: preproc_functions.append(lambda data, fs: (data / divisor, fs)) dataset = DiagnosisSet(n_recordings=n_recordings, max_recording_mins=max_recording_mins, preproc_functions=preproc_functions, data_folders=data_folders, train_or_eval='train', sensor_types=sensor_types) if test_on_eval: if test_recording_mins is None: test_recording_mins = duration_recording_mins test_preproc_functions = copy(preproc_functions) test_preproc_functions[1] = lambda data, fs: (data[:, :int( test_recording_mins * 60 * fs)], fs) test_dataset = DiagnosisSet(n_recordings=n_recordings, max_recording_mins=None, preproc_functions=test_preproc_functions, data_folders=data_folders, train_or_eval='eval', sensor_types=sensor_types) X, y = dataset.load() max_shape = np.max([list(x.shape) for x in X], axis=0) assert max_shape[1] == int(duration_recording_mins * sampling_freq * 60) if test_on_eval: test_X, test_y = test_dataset.load() max_shape = np.max([list(x.shape) for x in test_X], axis=0) assert max_shape[1] == int(test_recording_mins * sampling_freq * 60) if not test_on_eval: splitter = TrainValidTestSplitter(n_folds, i_test_fold, shuffle=shuffle) train_set, valid_set, test_set = splitter.split(X, y) else: splitter = TrainValidSplitter(n_folds, i_valid_fold=i_test_fold, shuffle=shuffle) train_set, valid_set = splitter.split(X, y) test_set = SignalAndTarget(test_X, test_y) del test_X, test_y del X, y # shouldn't be necessary, but just to make sure set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net(n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, stride_before_pool=True).create_network() elif (model_name == 'deep_smac'): if model_name == 'deep_smac': do_batch_norm = False else: assert model_name == 'deep_smac_bnorm' do_batch_norm = True double_time_convs = False drop_prob = 0.244445 filter_length_2 = 12 filter_length_3 = 14 filter_length_4 = 12 filter_time_length = 21 final_conv_length = 1 first_nonlin = elu first_pool_mode = 'mean' first_pool_nonlin = identity later_nonlin = elu later_pool_mode = 'mean' later_pool_nonlin = identity n_filters_factor = 1.679066 n_filters_start = 32 pool_time_length = 1 pool_time_stride = 2 split_first_layer = True n_chan_factor = n_filters_factor n_start_chans = n_filters_start model = Deep4Net(n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, batch_norm=do_batch_norm, double_time_convs=double_time_convs, drop_prob=drop_prob, filter_length_2=filter_length_2, filter_length_3=filter_length_3, filter_length_4=filter_length_4, filter_time_length=filter_time_length, first_nonlin=first_nonlin, first_pool_mode=first_pool_mode, first_pool_nonlin=first_pool_nonlin, later_nonlin=later_nonlin, later_pool_mode=later_pool_mode, later_pool_nonlin=later_pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, stride_before_pool=True).create_network() elif model_name == 'shallow_smac': conv_nonlin = identity do_batch_norm = True drop_prob = 0.328794 filter_time_length = 56 final_conv_length = 22 n_filters_spat = 73 n_filters_time = 24 pool_mode = 'max' pool_nonlin = identity pool_time_length = 84 pool_time_stride = 3 split_first_layer = True model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_filters_time, n_filters_spat=n_filters_spat, input_time_length=input_time_length, final_conv_length=final_conv_length, conv_nonlin=conv_nonlin, batch_norm=do_batch_norm, drop_prob=drop_prob, filter_time_length=filter_time_length, pool_mode=pool_mode, pool_nonlin=pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, ).create_network() elif model_name == 'linear': model = nn.Sequential() model.add_module("conv_classifier", nn.Conv2d(n_chans, n_classes, (600, 1))) model.add_module('softmax', nn.LogSoftmax()) model.add_module('squeeze', Expression(lambda x: x.squeeze(3))) else: assert False, "unknown model name {:s}".format(model_name) to_dense_prediction_model(model) log.info("Model:\n{:s}".format(str(model))) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, n_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() log.info("In shape: {:s}".format(str(test_input.cpu().data.numpy().shape))) out = model(test_input) log.info("Out shape: {:s}".format(str(out.cpu().data.numpy().shape))) n_preds_per_input = out.cpu().data.numpy().shape[2] log.info("{:d} predictions per input/trial".format(n_preds_per_input)) iterator = CropsFromTrialsIterator(batch_size=batch_size, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) optimizer = optim.Adam(model.parameters(), lr=init_lr) loss_function = lambda preds, targets: F.nll_loss( th.mean(preds, dim=2, keepdim=False), targets) if model_constraint is not None: assert model_constraint == 'defaultnorm' model_constraint = MaxNormDefaultConstraint() monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedDiagnosisMonitor(input_time_length, n_preds_per_input), RuntimeMonitor(), ] stop_criterion = MaxEpochs(max_epochs) batch_modifier = None run_after_early_stop = True exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=run_after_early_stop, batch_modifier=batch_modifier, cuda=cuda) exp.run() return exp
final_conv_length='auto') else: model = Deep4Net(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length='auto') else: # cropped if model_type == 'shallow': model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=None, final_conv_length=1) else: model = Deep4Net(in_chans=in_chans, n_classes=n_classes, input_time_length=None, final_conv_length=1) if cuda: model.cuda() from braindecode.torch_ext.optimizers import AdamW import torch.nn.functional as F if model_type == 'shallow': optimizer = AdamW(model.parameters(), lr=0.0625 * 0.01, weight_decay=0) else: optimizer = AdamW(model.parameters(), lr=1*0.01, weight_decay=0.5*0.001) # these are good values for the deep model if train_type == 'trialwise' : model.compile(loss=F.nll_loss, optimizer=optimizer, iterator_seed=1) else: # cropped model.compile(loss=F.nll_loss, optimizer=optimizer, iterator_seed=1, cropped=True) # Compile model exactly the same way as when you trained it
def test_cropped_decoding(): import mne from mne.io import concatenate_raws # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes) # Load each of the files parts = [mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events = mne.find_events(raw, shortest_event=0, stim_channel='STI 014') # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) import numpy as np from braindecode.datautil.signal_target import SignalAndTarget # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from torch import nn from braindecode.torch_ext.util import set_random_seeds from braindecode.models.util import to_dense_prediction_model # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12).create_network() to_dense_prediction_model(model) if cuda: model.cuda() from torch import optim optimizer = optim.Adam(model.parameters()) from braindecode.torch_ext.util import np_to_var # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] print("{:d} predictions per input/trial".format(n_preds_per_input)) from braindecode.datautil.iterators import CropsFromTrialsIterator iterator = CropsFromTrialsIterator(batch_size=32, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) from braindecode.torch_ext.util import np_to_var, var_to_np import torch.nn.functional as F from numpy.random import RandomState import torch as th from braindecode.experiments.monitors import compute_preds_per_trial_from_crops rng = RandomState((2017, 6, 30)) losses = [] accuracies = [] for i_epoch in range(4): # Set model to training mode model.train() for batch_X, batch_y in iterator.get_batches(train_set, shuffle=False): net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters optimizer.zero_grad() outputs = model(net_in) # Mean predictions across trial # Note that this will give identical gradients to computing # a per-prediction loss (at least for the combination of log softmax activation # and negative log likelihood loss which we are using here) outputs = th.mean(outputs, dim=2, keepdim=False) loss = F.nll_loss(outputs, net_target) loss.backward() optimizer.step() # Print some statistics each epoch model.eval() print("Epoch {:d}".format(i_epoch)) for setname, dataset in (('Train', train_set), ('Test', test_set)): # Collect all predictions and losses all_preds = [] all_losses = [] batch_sizes = [] for batch_X, batch_y in iterator.get_batches(dataset, shuffle=False): net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() outputs = model(net_in) all_preds.append(var_to_np(outputs)) outputs = th.mean(outputs, dim=2, keepdim=False) loss = F.nll_loss(outputs, net_target) loss = float(var_to_np(loss)) all_losses.append(loss) batch_sizes.append(len(batch_X)) # Compute mean per-input loss loss = np.mean(np.array(all_losses) * np.array(batch_sizes) / np.mean(batch_sizes)) print("{:6s} Loss: {:.5f}".format(setname, loss)) losses.append(loss) # Assign the predictions to the trials preds_per_trial = compute_preds_per_trial_from_crops(all_preds, input_time_length, dataset.X) # preds per trial are now trials x classes x timesteps/predictions # Now mean across timesteps for each trial to get per-trial predictions meaned_preds_per_trial = np.array( [np.mean(p, axis=1) for p in preds_per_trial]) predicted_labels = np.argmax(meaned_preds_per_trial, axis=1) accuracy = np.mean(predicted_labels == dataset.y) accuracies.append(accuracy * 100) print("{:6s} Accuracy: {:.1f}%".format( setname, accuracy * 100)) np.testing.assert_allclose( np.array(losses), np.array([1.703004002571106, 1.6295261979103088, 0.71168938279151917, 0.70825588703155518, 0.58231228590011597, 0.60176041722297668, 0.46629951894283295, 0.51184913516044617]), rtol=1e-4, atol=1e-5) np.testing.assert_allclose( np.array(accuracies), np.array( [50.0, 46.666666666666664, 60.0, 53.333333333333336, 68.333333333333329, 66.666666666666657, 88.333333333333329, 83.333333333333343]), rtol=1e-4, atol=1e-5)
def runModel(mode): cudnn.benchmark = True start = time.time() #mode = str(sys.argv[1]) #X,y,test_X,test_y = loadSubNormData(mode='all') #X,y,test_X,test_y = loadNEDCdata(mode=mode) #data = np.load('sessionsData/data%s-sessions.npy'%mode[:3]) #labels = np.load('sessionsData/labels%s-sessions.npy'%mode[:3]) data = np.load('data%s.npy' % mode[:3]) labels = np.load('labels%s.npy' % mode[:3]) X, y, test_X, test_y = splitDataRandom_Loaded(data, labels, mode) print('Mode - %s Total n: %d, Test n: %d' % (mode, len(y) + len(test_y), len(test_y))) #return 0 #X = addDataNoise(X,band=[1,4]) #test_X = addDataNoise(test_X,band=[1,4]) max_shape = np.max([list(x.shape) for x in X], axis=0) assert max_shape[1] == int(config.duration_recording_mins * config.sampling_freq * 60) n_classes = 2 n_recordings = None # set to an integer, if you want to restrict the set size sensor_types = ["EEG"] n_chans = 19 #21 max_recording_mins = 35 # exclude larger recordings from training set sec_to_cut = 60 # cut away at start of each recording duration_recording_mins = 5 #20 # how many minutes to use per recording test_recording_mins = 5 #20 max_abs_val = 800 # for clipping sampling_freq = 100 divisor = 10 # divide signal by this test_on_eval = True # teston evaluation set or on training set # in case of test on eval, n_folds and i_testfold determine # validation fold in training set for training until first stop n_folds = 10 i_test_fold = 9 shuffle = True model_name = 'linear' #'deep'#'shallow' 'linear' n_start_chans = 25 n_chan_factor = 2 # relevant for deep model only input_time_length = 6000 final_conv_length = 1 model_constraint = 'defaultnorm' init_lr = 1e-3 batch_size = 64 max_epochs = 35 # until first stop, the continue train on train+valid cuda = True # False if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net(n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, stride_before_pool=True).create_network() elif (model_name == 'deep_smac'): if model_name == 'deep_smac': do_batch_norm = False else: assert model_name == 'deep_smac_bnorm' do_batch_norm = True double_time_convs = False drop_prob = 0.244445 filter_length_2 = 12 filter_length_3 = 14 filter_length_4 = 12 filter_time_length = 21 final_conv_length = 1 first_nonlin = elu first_pool_mode = 'mean' first_pool_nonlin = identity later_nonlin = elu later_pool_mode = 'mean' later_pool_nonlin = identity n_filters_factor = 1.679066 n_filters_start = 32 pool_time_length = 1 pool_time_stride = 2 split_first_layer = True n_chan_factor = n_filters_factor n_start_chans = n_filters_start model = Deep4Net(n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, batch_norm=do_batch_norm, double_time_convs=double_time_convs, drop_prob=drop_prob, filter_length_2=filter_length_2, filter_length_3=filter_length_3, filter_length_4=filter_length_4, filter_time_length=filter_time_length, first_nonlin=first_nonlin, first_pool_mode=first_pool_mode, first_pool_nonlin=first_pool_nonlin, later_nonlin=later_nonlin, later_pool_mode=later_pool_mode, later_pool_nonlin=later_pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, stride_before_pool=True).create_network() elif model_name == 'shallow_smac': conv_nonlin = identity do_batch_norm = True drop_prob = 0.328794 filter_time_length = 56 final_conv_length = 22 n_filters_spat = 73 n_filters_time = 24 pool_mode = 'max' pool_nonlin = identity pool_time_length = 84 pool_time_stride = 3 split_first_layer = True model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_filters_time, n_filters_spat=n_filters_spat, input_time_length=input_time_length, final_conv_length=final_conv_length, conv_nonlin=conv_nonlin, batch_norm=do_batch_norm, drop_prob=drop_prob, filter_time_length=filter_time_length, pool_mode=pool_mode, pool_nonlin=pool_nonlin, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, split_first_layer=split_first_layer, ).create_network() elif model_name == 'linear': model = nn.Sequential() model.add_module("conv_classifier", nn.Conv2d(n_chans, n_classes, (600, 1))) model.add_module('softmax', nn.LogSoftmax(dim=1)) model.add_module('squeeze', Expression(lambda x: x.squeeze(3))) else: assert False, "unknown model name {:s}".format(model_name) to_dense_prediction_model(model) if config.cuda: model.cuda() test_input = np_to_var( np.ones((2, config.n_chans, config.input_time_length, 1), dtype=np.float32)) if config.cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] iterator = CropsFromTrialsIterator( batch_size=config.batch_size, input_time_length=config.input_time_length, n_preds_per_input=n_preds_per_input) #model.add_module('softmax', nn.LogSoftmax(dim=1)) model.eval() mode[2] = str(mode[2]) mode[3] = str(mode[3]) modelName = '-'.join(mode[:4]) #params = th.load('sessionsData/%sModel%s-sessions.pt'%(modelName,mode[4])) #params = th.load('%sModel%s.pt'%(modelName,mode[4])) params = th.load('linear/%sModel%s.pt' % (modelName, mode[4])) model.load_state_dict(params) if config.test_on_eval: #test_X, test_y = test_dataset.load() #test_X, test_y = loadNEDCdata(mode='eval') max_shape = np.max([list(x.shape) for x in test_X], axis=0) assert max_shape[1] == int(config.test_recording_mins * config.sampling_freq * 60) if not config.test_on_eval: splitter = TrainValidTestSplitter(config.n_folds, config.i_test_fold, shuffle=config.shuffle) train_set, valid_set, test_set = splitter.split(X, y) else: splitter = TrainValidSplitter(config.n_folds, i_valid_fold=config.i_test_fold, shuffle=config.shuffle) train_set, valid_set = splitter.split(X, y) test_set = SignalAndTarget(test_X, test_y) del test_X, test_y del X, y # shouldn't be necessary, but just to make sure datasets = OrderedDict( (('train', train_set), ('valid', valid_set), ('test', test_set))) for setname in ('train', 'valid', 'test'): #setname = 'test' #print("Compute predictions for {:s}...".format(setname)) dataset = datasets[setname] if config.cuda: preds_per_batch = [ var_to_np(model(np_to_var(b[0]).cuda())) for b in iterator.get_batches(dataset, shuffle=False) ] else: preds_per_batch = [ var_to_np(model(np_to_var(b[0]))) for b in iterator.get_batches(dataset, shuffle=False) ] preds_per_trial = compute_preds_per_trial( preds_per_batch, dataset, input_time_length=iterator.input_time_length, n_stride=iterator.n_preds_per_input) mean_preds_per_trial = [ np.mean(preds, axis=(0, 2)) for preds in preds_per_trial ] mean_preds_per_trial = np.array(mean_preds_per_trial) all_pred_labels = np.argmax(mean_preds_per_trial, axis=1).squeeze() all_target_labels = dataset.y acc_per_class = [] for i_class in range(n_classes): mask = all_target_labels == i_class acc = np.mean(all_pred_labels[mask] == all_target_labels[mask]) acc_per_class.append(acc) misclass = 1 - np.mean(acc_per_class) #print('Acc:{}, Class 0:{}, Class 1:{}'.format(np.mean(acc_per_class),acc_per_class[0],acc_per_class[1])) if setname == 'test': testResult = np.mean(acc_per_class) return testResult
def run_exp(max_recording_mins, n_recordings, sec_to_cut, duration_recording_mins, max_abs_val, max_min_threshold, max_min_expected, shrink_val, max_min_remove, batch_set_zero_val, batch_set_zero_test, sampling_freq, low_cut_hz, high_cut_hz, exp_demean, exp_standardize, moving_demean, moving_standardize, channel_demean, channel_standardize, divisor, n_folds, i_test_fold, model_name, input_time_length, final_conv_length, mean_before_softmax, batch_size, max_epochs, only_return_exp): cuda = True preproc_functions = [] preproc_functions.append(lambda data, fs: ( data[:, int(sec_to_cut * fs):-int(sec_to_cut * fs)], fs)) preproc_functions.append(lambda data, fs: (data[:, :int( duration_recording_mins * 60 * fs)], fs)) if max_abs_val is not None: preproc_functions.append( lambda data, fs: (np.clip(data, -max_abs_val, max_abs_val), fs)) if max_min_threshold is not None: preproc_functions.append(lambda data, fs: (clean_jumps( data, 200, max_min_threshold, max_min_expected, cuda), fs)) if max_min_remove is not None: window_len = 200 preproc_functions.append(lambda data, fs: (set_jumps_to_zero( data, window_len=window_len, threshold=max_min_remove, cuda=cuda, clip_min_max_to_zero=True), fs)) if shrink_val is not None: preproc_functions.append(lambda data, fs: (shrink_spikes( data, shrink_val, 1, 9, ), fs)) preproc_functions.append(lambda data, fs: (resampy.resample( data, fs, sampling_freq, axis=1, filter='kaiser_fast'), sampling_freq)) preproc_functions.append(lambda data, fs: (bandpass_cnt( data, low_cut_hz, high_cut_hz, fs, filt_order=4, axis=1), fs)) if exp_demean: preproc_functions.append(lambda data, fs: (exponential_running_demean( data.T, factor_new=0.001, init_block_size=100).T, fs)) if exp_standardize: preproc_functions.append( lambda data, fs: (exponential_running_standardize( data.T, factor_new=0.001, init_block_size=100).T, fs)) if moving_demean: preproc_functions.append(lambda data, fs: (padded_moving_demean( data, axis=1, n_window=201), fs)) if moving_standardize: preproc_functions.append(lambda data, fs: (padded_moving_standardize( data, axis=1, n_window=201), fs)) if channel_demean: preproc_functions.append(lambda data, fs: (demean(data, axis=1), fs)) if channel_standardize: preproc_functions.append(lambda data, fs: (standardize(data, axis=1), fs)) if divisor is not None: preproc_functions.append(lambda data, fs: (data / divisor, fs)) dataset = DiagnosisSet(n_recordings=n_recordings, max_recording_mins=max_recording_mins, preproc_functions=preproc_functions) if not only_return_exp: X, y = dataset.load() splitter = Splitter( n_folds, i_test_fold, ) if not only_return_exp: train_set, valid_set, test_set = splitter.split(X, y) del X, y # shouldn't be necessary, but just to make sure else: train_set = None valid_set = None test_set = None set_random_seeds(seed=20170629, cuda=cuda) in_chans = 21 n_classes = 2 if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net(in_chans, n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() optimizer = optim.Adam(model.parameters()) to_dense_prediction_model(model) log.info("Model:\n{:s}".format(str(model))) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] if mean_before_softmax: model = to_mean_before_softmax(model) else: model = to_mean_after_softmax(model) log.info("{:d} predictions per input/trial".format(n_preds_per_input)) iterator = CropsFromTrialsIterator(batch_size=batch_size, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) loss_function = F.nll_loss model_constraint = None monitors = [ LossMonitor(), MisclassMonitor(col_suffix='misclass'), RuntimeMonitor(), ] stop_criterion = MaxEpochs(max_epochs) batch_modifier = None if batch_set_zero_val is not None: batch_modifier = RemoveMinMaxDiff(batch_set_zero_val, clip_max_abs=True, set_zero=True) if (batch_set_zero_val is not None) and (batch_set_zero_test == True): iterator = ModifiedIterator( iterator, batch_modifier, ) batch_modifier = None exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, batch_modifier=batch_modifier, cuda=cuda) if not only_return_exp: exp.run() else: exp.dataset = dataset exp.splitter = splitter return exp
def test_experiment_class(): import mne from mne.io import concatenate_raws # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes) # Load each of the files parts = [mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) import numpy as np from braindecode.datautil.signal_target import SignalAndTarget from braindecode.datautil.splitters import split_into_two_sets # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) train_set, valid_set = split_into_two_sets(train_set, first_set_fraction=0.8) from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from torch import nn from braindecode.torch_ext.util import set_random_seeds from braindecode.models.util import to_dense_prediction_model # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12).create_network() to_dense_prediction_model(model) if cuda: model.cuda() from torch import optim optimizer = optim.Adam(model.parameters()) from braindecode.torch_ext.util import np_to_var # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] print("{:d} predictions per input/trial".format(n_preds_per_input)) from braindecode.experiments.experiment import Experiment from braindecode.datautil.iterators import CropsFromTrialsIterator from braindecode.experiments.monitors import RuntimeMonitor, LossMonitor, \ CroppedTrialMisclassMonitor, MisclassMonitor from braindecode.experiments.stopcriteria import MaxEpochs import torch.nn.functional as F import torch as th from braindecode.torch_ext.modules import Expression # Iterator is used to iterate over datasets both for training # and evaluation iterator = CropsFromTrialsIterator(batch_size=32, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) # Loss function takes predictions as they come out of the network and the targets # and returns a loss loss_function = lambda preds, targets: F.nll_loss( th.mean(preds, dim=2, keepdim=False), targets) # Could be used to apply some constraint on the models, then should be object # with apply method that accepts a module model_constraint = None # Monitors log the training progress monitors = [LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length), RuntimeMonitor(), ] # Stop criterion determines when the first stop happens stop_criterion = MaxEpochs(4) exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, batch_modifier=None, cuda=cuda) # need to setup python logging before to be able to see anything import logging import sys logging.basicConfig(format='%(asctime)s %(levelname)s : %(message)s', level=logging.DEBUG, stream=sys.stdout) exp.run() import pandas as pd from io import StringIO compare_df = pd.read_csv(StringIO( 'train_loss,valid_loss,test_loss,train_sample_misclass,valid_sample_misclass,' 'test_sample_misclass,train_misclass,valid_misclass,test_misclass\n' '14.167170524597168,13.910758018493652,15.945781707763672,0.5,0.5,' '0.5333333333333333,0.5,0.5,0.5333333333333333\n' '1.1735659837722778,1.4342904090881348,1.8664429187774658,0.4629567736185384,' '0.5120320855614973,0.5336007130124778,0.5,0.5,0.5333333333333333\n' '1.3168460130691528,1.60431969165802,1.9181344509124756,0.49298128342245995,' '0.5109180035650625,0.531729055258467,0.5,0.5,0.5333333333333333\n' '0.8465543389320374,1.280307412147522,1.439755916595459,0.4413435828877005,' '0.5461229946524064,0.5283422459893048,0.47916666666666663,0.5,' '0.5333333333333333\n0.6977059841156006,1.1762590408325195,1.2779350280761719,' '0.40290775401069523,0.588903743315508,0.5307486631016043,0.5,0.5,0.5\n' '0.7934166193008423,1.1762590408325195,1.2779350280761719,0.4401069518716577,' '0.588903743315508,0.5307486631016043,0.5,0.5,0.5\n0.5982189178466797,' '0.8581563830375671,0.9598925113677979,0.32032085561497325,0.47660427807486627,' '0.4672905525846702,0.31666666666666665,0.5,0.4666666666666667\n0.5044312477111816,' '0.7133197784423828,0.8164243102073669,0.2591354723707665,0.45699643493761144,' '0.4393048128342246,0.16666666666666663,0.41666666666666663,0.43333333333333335\n' '0.4815250039100647,0.6736412644386292,0.8016976714134216,0.23413547237076648,' '0.39505347593582885,0.42932263814616756,0.15000000000000002,0.41666666666666663,0.5\n')) for col in compare_df: np.testing.assert_allclose(np.array(compare_df[col]), exp.epochs_df[col], rtol=1e-3, atol=1e-4)
def run_experiment(train_set, valid_set, test_set, model_name, optimizer_name, init_lr, scheduler_name, use_norm_constraint, weight_decay, schedule_weight_decay, restarts, max_epochs, max_increase_epochs, np_th_seed): set_random_seeds(np_th_seed, cuda=True) #torch.backends.cudnn.benchmark = True# sometimes crashes? if valid_set is not None: assert max_increase_epochs is not None assert (max_epochs is None) != (restarts is None) if max_epochs is None: max_epochs = np.sum(restarts) n_classes = int(np.max(train_set.y) + 1) n_chans = int(train_set.X.shape[1]) input_time_length = 1000 if model_name == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=2).create_network() elif model_name == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=30).create_network() elif model_name in [ 'resnet-he-uniform', 'resnet-he-normal', 'resnet-xavier-normal', 'resnet-xavier-uniform' ]: init_name = model_name.lstrip('resnet-') from torch.nn import init init_fn = { 'he-uniform': lambda w: init.kaiming_uniform(w, a=0), 'he-normal': lambda w: init.kaiming_normal(w, a=0), 'xavier-uniform': lambda w: init.xavier_uniform(w, gain=1), 'xavier-normal': lambda w: init.xavier_normal(w, gain=1) }[init_name] model = EEGResNet(in_chans=n_chans, n_classes=n_classes, input_time_length=input_time_length, final_pool_length=10, n_first_filters=48, conv_weight_init_fn=init_fn).create_network() else: raise ValueError("Unknown model name {:s}".format(model_name)) if 'resnet' not in model_name: to_dense_prediction_model(model) model.cuda() model.eval() out = model(np_to_var(train_set.X[:1, :, :input_time_length, None]).cuda()) n_preds_per_input = out.cpu().data.numpy().shape[2] if optimizer_name == 'adam': optimizer = optim.Adam(model.parameters(), weight_decay=weight_decay, lr=init_lr) elif optimizer_name == 'adamw': optimizer = AdamW(model.parameters(), weight_decay=weight_decay, lr=init_lr) iterator = CropsFromTrialsIterator(batch_size=60, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input, seed=np_th_seed) if scheduler_name is not None: assert schedule_weight_decay == (optimizer_name == 'adamw') if scheduler_name == 'cosine': n_updates_per_epoch = sum( [1 for _ in iterator.get_batches(train_set, shuffle=True)]) if restarts is None: n_updates_per_period = n_updates_per_epoch * max_epochs else: n_updates_per_period = np.array(restarts) * n_updates_per_epoch scheduler = CosineAnnealing(n_updates_per_period) optimizer = ScheduledOptimizer( scheduler, optimizer, schedule_weight_decay=schedule_weight_decay) elif scheduler_name == 'cut_cosine': # TODO: integrate with if clause before, now just separate # to avoid messing with code n_updates_per_epoch = sum( [1 for _ in iterator.get_batches(train_set, shuffle=True)]) if restarts is None: n_updates_per_period = n_updates_per_epoch * max_epochs else: n_updates_per_period = np.array(restarts) * n_updates_per_epoch scheduler = CutCosineAnnealing(n_updates_per_period) optimizer = ScheduledOptimizer( scheduler, optimizer, schedule_weight_decay=schedule_weight_decay) else: raise ValueError("Unknown scheduler") monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length=input_time_length), RuntimeMonitor() ] if use_norm_constraint: model_constraint = MaxNormDefaultConstraint() else: model_constraint = None # change here this cell loss_function = lambda preds, targets: F.nll_loss(th.mean(preds, dim=2), targets) if valid_set is not None: run_after_early_stop = True do_early_stop = True remember_best_column = 'valid_misclass' stop_criterion = Or([ MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs) ]) else: run_after_early_stop = False do_early_stop = False remember_best_column = None stop_criterion = MaxEpochs(max_epochs) exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=loss_function, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column=remember_best_column, run_after_early_stop=run_after_early_stop, cuda=True, do_early_stop=do_early_stop) exp.run() return exp
class ShallowFBCSPNet_SpecializedTrainer(BaseEstimator, ClassifierMixin): model = None def __init__(self, network=None, filename=None): self.cuda = True if network is not None: self._decorateNetwork(network) elif filename is not None: self._loadFromFile(filename) else: print("unsupported option") sys.exit(-1) # set default parameters self.configure() def configure(self, nb_epoch=160, initial_lr=0.00006, trainTestRatio=(7 / 8)): self.nb_epoch = nb_epoch self.lr = initial_lr self.trainTestRatio = trainTestRatio def _decorateNetwork(self, network): self.model = network # TODO make a deep copy # deactivate training for all layers #for param in network.conv_classifier.parameters(): # param.requires_grad = False # replace last layer with a brand new one (for which training is true by default) self.model.conv_classifier = nn.Conv2d(5, 2, (116, 1), bias=True).cuda() # save/load only the model parameters(prefered solution) TODO: ask yannick torch.save(self.model.state_dict(), "myModel.pth") return def _loadFromFile(self, filename): # TODO: integrate this in saved file parameters somehow #n_filters_time=10 #filter_time_length=75 #n_filters_spat=5 #pool_time_length=60 #pool_time_stride=30 #in_chans = 15 #input_time_length = 3584 # final_conv_length = auto ensures we only get a single output in the time dimension self.model = ShallowFBCSPNet( in_chans=15, n_classes=2, input_time_length=3584, n_filters_time=10, filter_time_length=75, n_filters_spat=5, pool_time_length=60, pool_time_stride=30, final_conv_length='auto').create_network() # setup model for cuda if self.cuda: print("That's the new one") self.model.cuda() # load the saved network (makes it possible to run bottom form same starting point self.model.load_state_dict(torch.load("myModel.pth")) return """ Fit the network Params: X, data array in the format (...) y, labels ref: http://danielhnyk.cz/creating-your-own-estimator-scikit-learn/ """ def fit(self, X, y): self.nb_epoch = 160 # prepare an optimizer self.optimizer = optim.Adam(self.model.conv_classifier.parameters(), lr=self.lr) # define a number of train/test trials nb_train_trials = int(np.floor(self.trainTestRatio * X.shape[0])) # split the dataset train_set = SignalAndTarget(X[:nb_train_trials], y=y[:nb_train_trials]) test_set = SignalAndTarget(X[nb_train_trials:], y=y[nb_train_trials:]) # random generator self.rng = RandomState(None) # array that tracks results self.loss_rec = np.zeros((self.nb_epoch, 2)) self.accuracy_rec = np.zeros((self.nb_epoch, 2)) # run all epoch for i_epoch in range(self.nb_epoch): self._batchTrain(i_epoch, train_set) self._evalTraining(i_epoch, train_set, test_set) return self """ Training iteration, train the network on the train_set Params: i_epoch, current epoch iteration train_set, training set """ def _batchTrain(self, i_epoch, train_set): # get a set of balanced batches i_trials_in_batch = get_balanced_batches(len(train_set.X), self.rng, shuffle=True, batch_size=32) self.adjust_learning_rate(self.optimizer, i_epoch) # Set model to training mode self.model.train() # go through all batches for i_trials in i_trials_in_batch: # Have to add empty fourth dimension to X batch_X = train_set.X[i_trials][:, :, :, None] batch_y = train_set.y[i_trials] net_in = np_to_var(batch_X) net_target = np_to_var(batch_y) # if cuda, copy to cuda memory if self.cuda: net_in = net_in.cuda() net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters self.optimizer.zero_grad() # Compute outputs of the network outputs = self.model(net_in) # Compute the loss loss = F.nll_loss(outputs, net_target) # Do the backpropagation loss.backward() # Update parameters with the optimizer self.optimizer.step() return """ Evaluation iteration, computes the performance the network Params: i_epoch, current epoch iteration train_set, training set """ def _evalTraining(self, i_epoch, train_set, test_set): # Print some statistics each epoch self.model.eval() print("Epoch {:d}".format(i_epoch)) sets = {'Train': 0, 'Test': 1} # run evaluation on both train and test sets for setname, dataset in (('Train', train_set), ('Test', test_set)): # get balanced sets i_trials_in_batch = get_balanced_batches(len(dataset.X), self.rng, batch_size=32, shuffle=False) outputs = [] net_targets = [] # for all trials in set for i_trials in i_trials_in_batch: # adapt datasets batch_X = dataset.X[i_trials][:, :, :, None] batch_y = dataset.y[i_trials] # apply some conversion net_in = np_to_var(batch_X) net_target = np_to_var(batch_y) # convert if self.cuda: net_in = net_in.cuda() net_target = net_target.cuda() net_target = var_to_np(net_target) output = var_to_np(self.model(net_in)) outputs.append(output) net_targets.append(net_target) net_targets = np_to_var(np.concatenate(net_targets)) outputs = np_to_var(np.concatenate(outputs)) loss = F.nll_loss(outputs, net_targets) print("{:6s} Loss: {:.5f}".format(setname, float(var_to_np(loss)))) self.loss_rec[i_epoch, sets[setname]] = var_to_np(loss) predicted_labels = np.argmax(var_to_np(outputs), axis=1) accuracy = np.mean(dataset.y == predicted_labels) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) self.accuracy_rec[i_epoch, sets[setname]] = accuracy return def predict(self, X): self.model.eval() #i_trials_in_batch = get_balanced_batches(len(X), self.rng, batch_size=32, shuffle=False) outputs = [] for i_trials in i_trials_in_batch: batch_X = dataset.X[i_trials][:, :, :, None] net_in = np_to_var(batch_X) if self.cuda: net_in = net_in.cuda() output = var_to_np(self.model(net_in)) outputs.append(output) return outputs def adjust_learning_rate(self, optimizer, epoch): """Sets the learning rate to the initial LR decayed by 10% every 30 epochs""" lr = self.lr * (0.1**(epoch // 30)) for param_group in optimizer.param_groups: param_group['lr'] = lr
def test_experiment_class(): import mne from mne.io import concatenate_raws # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events = mne.find_events(raw, shortest_event=0, stim_channel='STI 014') # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) import numpy as np from braindecode.datautil.signal_target import SignalAndTarget from braindecode.datautil.splitters import split_into_two_sets # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) train_set, valid_set = split_into_two_sets(train_set, first_set_fraction=0.8) from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from torch import nn from braindecode.torch_ext.util import set_random_seeds from braindecode.models.util import to_dense_prediction_model # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12).create_network() to_dense_prediction_model(model) if cuda: model.cuda() from torch import optim optimizer = optim.Adam(model.parameters()) from braindecode.torch_ext.util import np_to_var # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] print("{:d} predictions per input/trial".format(n_preds_per_input)) from braindecode.experiments.experiment import Experiment from braindecode.datautil.iterators import CropsFromTrialsIterator from braindecode.experiments.monitors import RuntimeMonitor, LossMonitor, \ CroppedTrialMisclassMonitor, MisclassMonitor from braindecode.experiments.stopcriteria import MaxEpochs import torch.nn.functional as F import torch as th from braindecode.torch_ext.modules import Expression # Iterator is used to iterate over datasets both for training # and evaluation iterator = CropsFromTrialsIterator(batch_size=32, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) # Loss function takes predictions as they come out of the network and the targets # and returns a loss loss_function = lambda preds, targets: F.nll_loss( th.mean(preds, dim=2, keepdim=False), targets) # Could be used to apply some constraint on the models, then should be object # with apply method that accepts a module model_constraint = None # Monitors log the training progress monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length), RuntimeMonitor(), ] # Stop criterion determines when the first stop happens stop_criterion = MaxEpochs(4) exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, batch_modifier=None, cuda=cuda) # need to setup python logging before to be able to see anything import logging import sys logging.basicConfig(format='%(asctime)s %(levelname)s : %(message)s', level=logging.DEBUG, stream=sys.stdout) exp.run() import pandas as pd from io import StringIO compare_df = pd.read_csv( StringIO( u'train_loss,valid_loss,test_loss,train_sample_misclass,valid_sample_misclass,' 'test_sample_misclass,train_misclass,valid_misclass,test_misclass\n' '0,0.8692976435025532,0.7483791708946228,0.6975634694099426,' '0.5389371657754011,0.47103386809269165,0.4425133689839572,' '0.6041666666666667,0.5,0.4\n1,2.3362590074539185,' '2.317707061767578,2.1407743096351624,0.4827874331550802,' '0.5,0.4666666666666667,0.5,0.5,0.4666666666666667\n' '2,0.5981490015983582,0.785034716129303,0.7005959153175354,' '0.3391822638146168,0.47994652406417115,0.41996434937611404,' '0.22916666666666663,0.41666666666666663,0.43333333333333335\n' '3,0.6355261653661728,0.785034716129303,' '0.7005959153175354,0.3673351158645276,0.47994652406417115,' '0.41996434937611404,0.2666666666666667,0.41666666666666663,' '0.43333333333333335\n4,0.625280424952507,' '0.802731990814209,0.7048938572406769,0.3367201426024955,' '0.43137254901960786,0.4229946524064171,0.3666666666666667,' '0.5833333333333333,0.33333333333333337\n')) for col in compare_df: np.testing.assert_allclose(np.array(compare_df[col]), exp.epochs_df[col], rtol=1e-4, atol=1e-5)
def run_exp(max_recording_mins, n_recordings, sec_to_cut, duration_recording_mins, max_abs_val, max_min_threshold, max_min_expected, shrink_val, max_min_remove, batch_set_zero_val, batch_set_zero_test, sampling_freq, low_cut_hz, high_cut_hz, exp_demean, exp_standardize, moving_demean, moving_standardize, channel_demean, channel_standardize, divisor, n_folds, i_test_fold, model_name, input_time_length, final_conv_length, batch_size, max_epochs, only_return_exp): cuda = True preproc_functions = [] preproc_functions.append(lambda data, fs: ( data[:, int(sec_to_cut * fs):-int(sec_to_cut * fs)], fs)) preproc_functions.append(lambda data, fs: (data[:, :int( duration_recording_mins * 60 * fs)], fs)) if max_abs_val is not None: preproc_functions.append( lambda data, fs: (np.clip(data, -max_abs_val, max_abs_val), fs)) if max_min_threshold is not None: preproc_functions.append(lambda data, fs: (clean_jumps( data, 200, max_min_threshold, max_min_expected, cuda), fs)) if max_min_remove is not None: window_len = 200 preproc_functions.append(lambda data, fs: (set_jumps_to_zero( data, window_len=window_len, threshold=max_min_remove, cuda=cuda, clip_min_max_to_zero=True), fs)) if shrink_val is not None: preproc_functions.append(lambda data, fs: (shrink_spikes( data, shrink_val, 1, 9, ), fs)) preproc_functions.append(lambda data, fs: (resampy.resample( data, fs, sampling_freq, axis=1, filter='kaiser_fast'), sampling_freq)) preproc_functions.append(lambda data, fs: (bandpass_cnt( data, low_cut_hz, high_cut_hz, fs, filt_order=4, axis=1), fs)) if exp_demean: preproc_functions.append(lambda data, fs: (exponential_running_demean( data.T, factor_new=0.001, init_block_size=100).T, fs)) if exp_standardize: preproc_functions.append( lambda data, fs: (exponential_running_standardize( data.T, factor_new=0.001, init_block_size=100).T, fs)) if moving_demean: preproc_functions.append(lambda data, fs: (padded_moving_demean( data, axis=1, n_window=201), fs)) if moving_standardize: preproc_functions.append(lambda data, fs: (padded_moving_standardize( data, axis=1, n_window=201), fs)) if channel_demean: preproc_functions.append(lambda data, fs: (demean(data, axis=1), fs)) if channel_standardize: preproc_functions.append(lambda data, fs: (standardize(data, axis=1), fs)) if divisor is not None: preproc_functions.append(lambda data, fs: (data / divisor, fs)) all_file_names, labels = get_all_sorted_file_names_and_labels() lengths = np.load( '/home/schirrmr/code/auto-diagnosis/sorted-recording-lengths.npy') mask = lengths < max_recording_mins * 60 cleaned_file_names = np.array(all_file_names)[mask] cleaned_labels = labels[mask] diffs_per_rec = np.load( '/home/schirrmr/code/auto-diagnosis/diffs_per_recording.npy') def create_set(inds): X = [] for i in inds: log.info("Load {:s}".format(cleaned_file_names[i])) x = load_data(cleaned_file_names[i], preproc_functions) X.append(x) y = cleaned_labels[inds].astype(np.int64) return SignalAndTarget(X, y) if not only_return_exp: folds = get_balanced_batches(n_recordings, None, False, n_batches=n_folds) test_inds = folds[i_test_fold] valid_inds = folds[i_test_fold - 1] all_inds = list(range(n_recordings)) train_inds = np.setdiff1d(all_inds, np.union1d(test_inds, valid_inds)) rec_nr_sorted_by_diff = np.argsort(diffs_per_rec)[::-1] train_inds = rec_nr_sorted_by_diff[train_inds] valid_inds = rec_nr_sorted_by_diff[valid_inds] test_inds = rec_nr_sorted_by_diff[test_inds] train_set = create_set(train_inds) valid_set = create_set(valid_inds) test_set = create_set(test_inds) else: train_set = None valid_set = None test_set = None set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel n_classes = 2 in_chans = 21 if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net(in_chans, n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() optimizer = optim.Adam(model.parameters()) to_dense_prediction_model(model) log.info("Model:\n{:s}".format(str(model))) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] log.info("{:d} predictions per input/trial".format(n_preds_per_input)) iterator = CropsFromTrialsIterator(batch_size=batch_size, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) loss_function = lambda preds, targets: F.nll_loss( th.mean(preds, dim=2)[:, :, 0], targets) model_constraint = None monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length), RuntimeMonitor(), ] stop_criterion = MaxEpochs(max_epochs) batch_modifier = None if batch_set_zero_val is not None: batch_modifier = RemoveMinMaxDiff(batch_set_zero_val, clip_max_abs=True, set_zero=True) if (batch_set_zero_val is not None) and (batch_set_zero_test == True): iterator = ModifiedIterator( iterator, batch_modifier, ) batch_modifier = None exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, batch_modifier=batch_modifier, cuda=cuda) if not only_return_exp: exp.run() else: exp.dataset = None exp.splitter = None return exp
def run_exp(max_epochs, only_return_exp): from collections import OrderedDict filenames = [ 'data/robot-hall/NiRiNBD6.ds_1-1_500Hz.BBCI.mat', 'data/robot-hall/NiRiNBD8.ds_1-1_500Hz.BBCI.mat', 'data/robot-hall/NiRiNBD9.ds_1-1_500Hz.BBCI.mat', 'data/robot-hall/NiRiNBD10.ds_1-1_500Hz.BBCI.mat', 'data/robot-hall/NiRiNBD12_cursor_250Hz.BBCI.mat', 'data/robot-hall/NiRiNBD13_cursorS000R01_onlyFullRuns_250Hz.BBCI.mat', 'data/robot-hall/NiRiNBD14_cursor_250Hz.BBCI.mat', 'data/robot-hall/NiRiNBD15_cursor_250Hz.BBCI.mat' ] sensor_names = [ 'Fp1', 'Fpz', 'Fp2', 'AF7', 'AF3', 'AF4', 'AF8', 'F7', 'F5', 'F3', 'F1', 'Fz', 'F2', 'F4', 'F6', 'F8', 'FT7', 'FC5', 'FC3', 'FC1', 'FCz', 'FC2', 'FC4', 'FC6', 'FT8', 'M1', 'T7', 'C5', 'C3', 'C1', 'Cz', 'C2', 'C4', 'C6', 'T8', 'M2', 'TP7', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'TP8', 'P7', 'P5', 'P3', 'P1', 'Pz', 'P2', 'P4', 'P6', 'P8', 'PO7', 'PO5', 'PO3', 'POz', 'PO4', 'PO6', 'PO8', 'O1', 'Oz', 'O2' ] name_to_start_codes = OrderedDict([('Right Hand', [1]), ('Feet', [2]), ('Rotation', [3]), ('Words', [4])]) name_to_stop_codes = OrderedDict([('Right Hand', [10]), ('Feet', [20]), ('Rotation', [30]), ('Words', [40])]) trial_ival = [500, 0] min_break_length_ms = 6000 max_break_length_ms = 8000 break_ival = [1000, -500] input_time_length = 700 filename_to_extra_args = { 'data/robot-hall/NiRiNBD12_cursor_250Hz.BBCI.mat': dict( name_to_start_codes=OrderedDict([('Right Hand', [1]), ('Feet', [2]), ('Rotation', [3]), ('Words', [4]), ('Rest', [5])]), name_to_stop_codes=OrderedDict([('Right Hand', [10]), ('Feet', [20]), ('Rotation', [30]), ('Words', [40]), ('Rest', [50])]), min_break_length_ms=3700, max_break_length_ms=3900, ), 'data/robot-hall/NiRiNBD13_cursorS000R01_onlyFullRuns_250Hz.BBCI.mat': dict( name_to_start_codes=OrderedDict([('Right Hand', [1]), ('Feet', [2]), ('Rotation', [3]), ('Words', [4]), ('Rest', [5])]), name_to_stop_codes=OrderedDict([('Right Hand', [10]), ('Feet', [20]), ('Rotation', [30]), ('Words', [40]), ('Rest', [50])]), min_break_length_ms=3700, max_break_length_ms=3900, ), 'data/robot-hall/NiRiNBD14_cursor_250Hz.BBCI.mat': dict( name_to_start_codes=OrderedDict([('Right Hand', [1]), ('Feet', [2]), ('Rotation', [3]), ('Words', [4]), ('Rest', [5])]), name_to_stop_codes=OrderedDict([('Right Hand', [10]), ('Feet', [20]), ('Rotation', [30]), ('Words', [40]), ('Rest', [50])]), min_break_length_ms=3700, max_break_length_ms=3900, ), 'data/robot-hall/NiRiNBD15_cursor_250Hz.BBCI.mat': dict( name_to_start_codes=OrderedDict([('Right Hand', [1]), ('Feet', [2]), ('Rotation', [3]), ('Words', [4]), ('Rest', [5])]), name_to_stop_codes=OrderedDict([('Right Hand', [10]), ('Feet', [20]), ('Rotation', [30]), ('Words', [40]), ('Rest', [50])]), min_break_length_ms=3700, max_break_length_ms=3900, ), } from braindecode.datautil.trial_segment import \ create_signal_target_with_breaks_from_mne from copy import deepcopy def load_data(filenames, sensor_names, name_to_start_codes, name_to_stop_codes, trial_ival, break_ival, min_break_length_ms, max_break_length_ms, input_time_length, filename_to_extra_args): all_sets = [] original_args = locals() for filename in filenames: kwargs = deepcopy(original_args) if filename in filename_to_extra_args: kwargs.update(filename_to_extra_args[filename]) log.info("Loading {:s}...".format(filename)) cnt = BBCIDataset(filename, load_sensor_names=sensor_names).load() cnt = cnt.drop_channels(['STI 014']) log.info("Resampling...") cnt = resample_cnt(cnt, 100) log.info("Standardizing...") cnt = mne_apply( lambda a: exponential_running_standardize( a.T, init_block_size=50).T, cnt) log.info("Transform to set...") full_set = (create_signal_target_with_breaks_from_mne( cnt, kwargs['name_to_start_codes'], kwargs['trial_ival'], kwargs['name_to_stop_codes'], kwargs['min_break_length_ms'], kwargs['max_break_length_ms'], kwargs['break_ival'], prepad_trials_to_n_samples=kwargs['input_time_length'], )) all_sets.append(full_set) return all_sets sensor_names = [ 'Fp1', 'Fpz', 'Fp2', 'AF7', 'AF3', 'AF4', 'AF8', 'F7', 'F5', 'F3', 'F1', 'Fz', 'F2', 'F4', 'F6', 'F8', 'FT7', 'FC5', 'FC3', 'FC1', 'FCz', 'FC2', 'FC4', 'FC6', 'FT8', 'M1', 'T7', 'C5', 'C3', 'C1', 'Cz', 'C2', 'C4', 'C6', 'T8', 'M2', 'TP7', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'TP8', 'P7', 'P5', 'P3', 'P1', 'Pz', 'P2', 'P4', 'P6', 'P8', 'PO7', 'PO5', 'PO3', 'POz', 'PO4', 'PO6', 'PO8', 'O1', 'Oz', 'O2' ] #sensor_names = ['C3', 'C4'] n_chans = len(sensor_names) if not only_return_exp: all_sets = load_data(filenames, sensor_names, name_to_start_codes, name_to_stop_codes, trial_ival, break_ival, min_break_length_ms, max_break_length_ms, input_time_length, filename_to_extra_args) from braindecode.datautil.signal_target import SignalAndTarget from braindecode.datautil.splitters import concatenate_sets train_set = concatenate_sets(all_sets[:6]) valid_set = all_sets[6] test_set = all_sets[7] else: train_set = None valid_set = None test_set = None set_random_seeds(seed=20171017, cuda=True) n_classes = 5 # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=n_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=30).create_network() to_dense_prediction_model(model) model.cuda() from torch import optim import numpy as np optimizer = optim.Adam(model.parameters()) from braindecode.torch_ext.util import np_to_var # determine output size test_input = np_to_var( np.ones((2, n_chans, input_time_length, 1), dtype=np.float32)) test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] print("{:d} predictions per input/trial".format(n_preds_per_input)) from braindecode.datautil.iterators import CropsFromTrialsIterator iterator = CropsFromTrialsIterator(batch_size=32, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) from braindecode.experiments.experiment import Experiment from braindecode.experiments.monitors import RuntimeMonitor, LossMonitor, \ CroppedTrialMisclassMonitor, MisclassMonitor from braindecode.experiments.stopcriteria import MaxEpochs from braindecode.torch_ext.losses import log_categorical_crossentropy import torch.nn.functional as F import torch as th from braindecode.torch_ext.modules import Expression loss_function = log_categorical_crossentropy model_constraint = MaxNormDefaultConstraint() monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length), RuntimeMonitor(), ] stop_criterion = MaxEpochs(max_epochs) exp = Experiment(model, train_set, valid_set, test_set, iterator, loss_function, optimizer, model_constraint, monitors, stop_criterion, remember_best_column='valid_sample_misclass', run_after_early_stop=True, batch_modifier=None, cuda=True) if not only_return_exp: exp.run() return exp
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda): train_filename = 'A{:02d}T.gdf'.format(subject_id) test_filename = 'A{:02d}E.gdf'.format(subject_id) train_filepath = os.path.join(data_folder, train_filename) test_filepath = os.path.join(data_folder, test_filename) train_label_filepath = train_filepath.replace('.gdf', '.mat') test_label_filepath = test_filepath.replace('.gdf', '.mat') train_loader = BCICompetition4Set2A(train_filepath, labels_filename=train_label_filepath) test_loader = BCICompetition4Set2A(test_filepath, labels_filename=test_label_filepath) train_cnt = train_loader.load() test_cnt = test_loader.load() # Preprocessing train_cnt = train_cnt.drop_channels( ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right']) assert len(train_cnt.ch_names) == 22 # lets convert to millvolt for numerical stability of next operations train_cnt = mne_apply(lambda a: a * 1e6, train_cnt) train_cnt = mne_apply( lambda a: bandpass_cnt( a, low_cut_hz, 38, train_cnt.info['sfreq'], filt_order=3, axis=1), train_cnt) train_cnt = mne_apply( lambda a: exponential_running_standardize( a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, train_cnt) test_cnt = test_cnt.drop_channels( ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right']) assert len(test_cnt.ch_names) == 22 test_cnt = mne_apply(lambda a: a * 1e6, test_cnt) test_cnt = mne_apply( lambda a: bandpass_cnt( a, low_cut_hz, 38, test_cnt.info['sfreq'], filt_order=3, axis=1), test_cnt) test_cnt = mne_apply( lambda a: exponential_running_standardize( a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, test_cnt) marker_def = OrderedDict([('Left Hand', [1]), ( 'Right Hand', [2], ), ('Foot', [3]), ('Tongue', [4])]) ival = [-500, 4000] train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival) test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival) train_set, valid_set = split_into_two_sets(train_set, first_set_fraction=0.8) set_random_seeds(seed=20190706, cuda=cuda) n_classes = 4 n_chans = int(train_set.X.shape[1]) input_time_length = train_set.X.shape[2] if model == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() elif model == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() if cuda: model.cuda() log.info("Model: \n{:s}".format(str(model))) optimizer = optim.Adam(model.parameters()) iterator = BalancedBatchSizeIterator(batch_size=60) stop_criterion = Or([MaxEpochs(1600), NoDecrease('valid_misclass', 160)]) monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()] model_constraint = MaxNormDefaultConstraint() exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=F.nll_loss, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, cuda=cuda) exp.run() return exp
class ShallowFBCSPNet_GeneralTrainer(BaseEstimator, ClassifierMixin): """ Initialize the parameters of the network Full list of parameters described in ref: https://robintibor.github.io/braindecode/source/braindecode.models.html """ def __init__(self, n_filters_time=10, filter_time_length=75, n_filters_spat=5, pool_time_length=60, pool_time_stride=30, nb_epoch=160): # init meta info self.cuda = torch.cuda.is_available() #set_random_seeds(seed=20180505, cuda=self.cuda) # TODO: Fix random seed set_random_seeds(seed=randint(1, 20180505), cuda=self.cuda) # TODO: Fix random seed # copy all network parameters self.n_filters_time = n_filters_time self.filter_time_length = filter_time_length self.n_filters_spat = n_filters_spat self.pool_time_length = pool_time_length self.pool_time_stride = pool_time_stride self.nb_epoch = nb_epoch return """ Fit the network Params: X, data array in the format (...) y, labels ref: http://danielhnyk.cz/creating-your-own-estimator-scikit-learn/ """ def fit(self, X, y): # define a number of train/test trials nb_train_trials = int(np.floor(7 / 8 * X.shape[0])) # split the dataset train_set = SignalAndTarget(X[:nb_train_trials], y=y[:nb_train_trials]) test_set = SignalAndTarget(X[nb_train_trials:], y=y[nb_train_trials:]) # number of classes and input channels n_classes = np.unique(y).size in_chans = train_set.X.shape[1] # final_conv_length = auto ensures we only get a single output in the time dimension self.model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], n_filters_time=self.n_filters_time, filter_time_length=self.filter_time_length, n_filters_spat=self.n_filters_spat, pool_time_length=self.pool_time_length, pool_time_stride=self.pool_time_stride, final_conv_length='auto').create_network() # setup model for cuda if self.cuda: self.model.cuda() # setup optimizer self.optimizer = optim.Adam(self.model.parameters()) # random generator self.rng = RandomState(None) # array that tracks results self.loss_rec = np.zeros((self.nb_epoch, 2)) self.accuracy_rec = np.zeros((self.nb_epoch, 2)) # run all epoch for i_epoch in range(self.nb_epoch): self._batchTrain(i_epoch, train_set) self._evalTraining(i_epoch, train_set, test_set) return self """ Training iteration, train the network on the train_set Params: i_epoch, current epoch iteration train_set, training set """ def _batchTrain(self, i_epoch, train_set): # get a set of balanced batches i_trials_in_batch = get_balanced_batches(len(train_set.X), self.rng, shuffle=True, batch_size=32) # Set model to training mode self.model.train() # go through all batches for i_trials in i_trials_in_batch: # Have to add empty fourth dimension to X batch_X = train_set.X[i_trials][:, :, :, None] batch_y = train_set.y[i_trials] net_in = np_to_var(batch_X) net_target = np_to_var(batch_y) # if cuda, copy to cuda memory if self.cuda: net_in = net_in.cuda() net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters self.optimizer.zero_grad() # Compute outputs of the network outputs = self.model(net_in) # Compute the loss loss = F.nll_loss(outputs, net_target) # Do the backpropagation loss.backward() # Update parameters with the optimizer self.optimizer.step() return """ Evaluation iteration, computes the performance the network Params: i_epoch, current epoch iteration train_set, training set """ def _evalTraining(self, i_epoch, train_set, test_set): # Print some statistics each epoch self.model.eval() print("Epoch {:d}".format(i_epoch)) sets = {'Train': 0, 'Test': 1} # run evaluation on both train and test sets for setname, dataset in (('Train', train_set), ('Test', test_set)): # get balanced sets i_trials_in_batch = get_balanced_batches(len(dataset.X), self.rng, batch_size=32, shuffle=False) outputs = [] net_targets = [] # for all trials in set for i_trials in i_trials_in_batch: # adapt datasets batch_X = dataset.X[i_trials][:, :, :, None] batch_y = dataset.y[i_trials] # apply some conversion net_in = np_to_var(batch_X) net_target = np_to_var(batch_y) # convert if self.cuda: net_in = net_in.cuda() net_target = net_target.cuda() net_target = var_to_np(net_target) output = var_to_np(self.model(net_in)) outputs.append(output) net_targets.append(net_target) net_targets = np_to_var(np.concatenate(net_targets)) outputs = np_to_var(np.concatenate(outputs)) loss = F.nll_loss(outputs, net_targets) print("{:6s} Loss: {:.5f}".format(setname, float(var_to_np(loss)))) self.loss_rec[i_epoch, sets[setname]] = var_to_np(loss) predicted_labels = np.argmax(var_to_np(outputs), axis=1) accuracy = np.mean(dataset.y == predicted_labels) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) self.accuracy_rec[i_epoch, sets[setname]] = accuracy return def predict(self, X): self.model.eval() #i_trials_in_batch = get_balanced_batches(len(X), self.rng, batch_size=32, shuffle=False) outputs = [] for i_trials in i_trials_in_batch: batch_X = dataset.X[i_trials][:, :, :, None] net_in = np_to_var(batch_X) if self.cuda: net_in = net_in.cuda() output = var_to_np(self.model(net_in)) outputs.append(output) return outputs
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda): ival = [-500, 4000] max_epochs = 1600 max_increase_epochs = 160 batch_size = 60 high_cut_hz = 38 factor_new = 1e-3 init_block_size = 1000 valid_set_fraction = 0.2 train_filename = "A{:02d}T.gdf".format(subject_id) test_filename = "A{:02d}E.gdf".format(subject_id) train_filepath = os.path.join(data_folder, train_filename) test_filepath = os.path.join(data_folder, test_filename) train_label_filepath = train_filepath.replace(".gdf", ".mat") test_label_filepath = test_filepath.replace(".gdf", ".mat") train_loader = BCICompetition4Set2A( train_filepath, labels_filename=train_label_filepath ) test_loader = BCICompetition4Set2A( test_filepath, labels_filename=test_label_filepath ) train_cnt = train_loader.load() test_cnt = test_loader.load() # Preprocessing train_cnt = train_cnt.drop_channels( ["EOG-left", "EOG-central", "EOG-right"] ) assert len(train_cnt.ch_names) == 22 # lets convert to millvolt for numerical stability of next operations train_cnt = mne_apply(lambda a: a * 1e6, train_cnt) train_cnt = mne_apply( lambda a: bandpass_cnt( a, low_cut_hz, high_cut_hz, train_cnt.info["sfreq"], filt_order=3, axis=1, ), train_cnt, ) train_cnt = mne_apply( lambda a: exponential_running_standardize( a.T, factor_new=factor_new, init_block_size=init_block_size, eps=1e-4, ).T, train_cnt, ) test_cnt = test_cnt.drop_channels(["EOG-left", "EOG-central", "EOG-right"]) assert len(test_cnt.ch_names) == 22 test_cnt = mne_apply(lambda a: a * 1e6, test_cnt) test_cnt = mne_apply( lambda a: bandpass_cnt( a, low_cut_hz, high_cut_hz, test_cnt.info["sfreq"], filt_order=3, axis=1, ), test_cnt, ) test_cnt = mne_apply( lambda a: exponential_running_standardize( a.T, factor_new=factor_new, init_block_size=init_block_size, eps=1e-4, ).T, test_cnt, ) marker_def = OrderedDict( [ ("Left Hand", [1]), ("Right Hand", [2]), ("Foot", [3]), ("Tongue", [4]), ] ) train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival) test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival) train_set, valid_set = split_into_two_sets( train_set, first_set_fraction=1 - valid_set_fraction ) set_random_seeds(seed=20190706, cuda=cuda) n_classes = 4 n_chans = int(train_set.X.shape[1]) input_time_length = train_set.X.shape[2] if model == "shallow": model = ShallowFBCSPNet( n_chans, n_classes, input_time_length=input_time_length, final_conv_length="auto", ).create_network() elif model == "deep": model = Deep4Net( n_chans, n_classes, input_time_length=input_time_length, final_conv_length="auto", ).create_network() if cuda: model.cuda() log.info("Model: \n{:s}".format(str(model))) optimizer = optim.Adam(model.parameters()) iterator = BalancedBatchSizeIterator(batch_size=batch_size) stop_criterion = Or( [ MaxEpochs(max_epochs), NoDecrease("valid_misclass", max_increase_epochs), ] ) monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()] model_constraint = MaxNormDefaultConstraint() exp = Experiment( model, train_set, valid_set, test_set, iterator=iterator, loss_function=F.nll_loss, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column="valid_misclass", run_after_early_stop=True, cuda=cuda, ) exp.run() return exp
def run_exp(epoches, batch_size, subject_num, model_type, cuda, single_subject, single_subject_num): # ival = [-500, 4000] max_increase_epochs = 160 # Preprocessing X, y = loadSubjects(subject_num, single_subject, single_subject_num) X = X.astype(np.float32) y = y.astype(np.int64) X, y = shuffle(X, y) trial_length = X.shape[2] print("trial_length " + str(trial_length)) print("trying to run with {} sec trials ".format((trial_length - 1) / 256)) print("y") print(y) trainingSampleSize = int(len(X) * 0.6) valudationSampleSize = int(len(X) * 0.2) testSampleSize = int(len(X) * 0.2) print("INFO : Training sample size: {}".format(trainingSampleSize)) print("INFO : Validation sample size: {}".format(valudationSampleSize)) print("INFO : Test sample size: {}".format(testSampleSize)) train_set = SignalAndTarget(X[:trainingSampleSize], y=y[:trainingSampleSize]) valid_set = SignalAndTarget( X[trainingSampleSize:(trainingSampleSize + valudationSampleSize)], y=y[trainingSampleSize:(trainingSampleSize + valudationSampleSize)]) test_set = SignalAndTarget(X[(trainingSampleSize + valudationSampleSize):], y=y[(trainingSampleSize + valudationSampleSize):]) set_random_seeds(seed=20190706, cuda=cuda) n_classes = 3 n_chans = int(train_set.X.shape[1]) input_time_length = train_set.X.shape[2] if model_type == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() elif model_type == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() elif model_type == 'eegnet': model = EEGNetv4(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() if cuda: model.cuda() log.info("Model: \n{:s}".format(str(model))) optimizer = optim.Adam(model.parameters()) iterator = BalancedBatchSizeIterator(batch_size=batch_size) stop_criterion = Or([ MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs) ]) monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()] model_constraint = MaxNormDefaultConstraint() exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=F.nll_loss, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, cuda=cuda) exp.run() # th.save(model, "models\{}-cropped-singleSubjectNum{}-{}sec-{}epoches-torch_model".format(model_type, single_subject_num, ((trial_length - 1) / 256), epoches)) return exp
def run_exp(max_recording_mins, n_recordings, sec_to_cut, duration_recording_mins, max_abs_val, shrink_val, sampling_freq, divisor, n_folds, i_test_fold, final_conv_length, model_constraint, batch_size, max_epochs, n_filters_time, n_filters_spat, filter_time_length, conv_nonlin, pool_time_length, pool_time_stride, pool_mode, pool_nonlin, split_first_layer, do_batch_norm, drop_prob, time_cut_off_sec, start_time, input_time_length, only_return_exp): kwargs = locals() for model_param in [ 'final_conv_length', 'n_filters_time', 'n_filters_spat', 'filter_time_length', 'conv_nonlin', 'pool_time_length', 'pool_time_stride', 'pool_mode', 'pool_nonlin', 'split_first_layer', 'do_batch_norm', 'drop_prob', ]: kwargs.pop(model_param) nonlin_dict = { 'elu': elu, 'relu': relu, 'relu6': relu6, 'tanh': tanh, 'square': square, 'identity': identity, 'log': safe_log, } assert input_time_length == 6000 # copy over from early seizure # make proper n_classes = 2 in_chans = 21 cuda = True set_random_seeds(seed=20170629, cuda=cuda) model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length, n_filters_time=n_filters_time, filter_time_length=filter_time_length, n_filters_spat=n_filters_spat, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, conv_nonlin=nonlin_dict[conv_nonlin], pool_mode=pool_mode, pool_nonlin=nonlin_dict[pool_nonlin], split_first_layer=split_first_layer, batch_norm=do_batch_norm, batch_norm_alpha=0.1, drop_prob=drop_prob).create_network() to_dense_prediction_model(model) if cuda: model.cuda() model.eval() test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() try: out = model(test_input) except RuntimeError: raise ValueError("Model receptive field too large...") n_preds_per_input = out.cpu().data.numpy().shape[2] n_receptive_field = input_time_length - n_preds_per_input if n_receptive_field > 6000: raise ValueError("Model receptive field ({:d}) too large...".format( n_receptive_field)) # For future, here optionally add input time length instead model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length, n_filters_time=n_filters_time, filter_time_length=filter_time_length, n_filters_spat=n_filters_spat, pool_time_length=pool_time_length, pool_time_stride=pool_time_stride, conv_nonlin=nonlin_dict[conv_nonlin], pool_mode=pool_mode, pool_nonlin=nonlin_dict[pool_nonlin], split_first_layer=split_first_layer, batch_norm=do_batch_norm, batch_norm_alpha=0.1, drop_prob=drop_prob).create_network() return common.run_exp(model=model, **kwargs)
def test_trialwise_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # event_codes = [6] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset # events = mne.find_events(raw, shortest_event=0, stim_channel='STI 014') events, _ = mne.events_from_annotations(raw) # Extract trials, only using EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. # X:[90,64,497] X = (epoched.get_data() * 1e6).astype(np.float32) # y:[90] y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # X_train:[60,64,497], y_train:[60] train_set = SignalAndTarget(X[:60], y=y[:60]) # X_test:[30,64,497], y_test:[30] test_set = SignalAndTarget(X[60:], y=y[60:]) # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length = auto ensures we only get a single output in the time dimension # def __init__(self, in_chans=64, n_classes=2, input_time_length=497, n_filters_time=40, filter_time_length=25, n_filters_spat=40, pool_time_length=75, pool_time_stride=15, final_conv_length='auto, conv_nonlin=square, pool_mode="mean", pool_nonlin=safe_log, split_first_layer=True, batch_norm=True, batch_norm_alpha=0.1, drop_prob=0.5, ): # 感觉create_network()就是__init__的一部分, 现在改成用self.model调用了, 还是感觉不优雅, 主要是forward集成在nn.Sequential里面了 # 然后这个model的实际__init__不是ShallowFBCSPNet, 而是nn.Sequential, 感觉我更喜欢原来的定义方式, 这种方式看不到中间输出 # model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], final_conv_length='auto').create_network() #原来的 model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], final_conv_length='auto').model if cuda: model.cuda() optimizer = optim.Adam(model.parameters()) rng = RandomState((2017, 6, 30)) losses = [] accuracies = [] for i_epoch in range(6): i_trials_in_batch = get_balanced_batches(len(train_set.X), rng, shuffle=True, batch_size=10) # Set model to training mode model.train() for i_trials in i_trials_in_batch: # Have to add empty fourth dimension to X batch_X = train_set.X[i_trials][:, :, :, None] batch_y = train_set.y[i_trials] net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters optimizer.zero_grad() # Compute outputs of the network #net_in: [10, 64, 497, 1]=[bsz, H_im, W_im, C_im] # outputs = model.forward(net_in) # model=Sequential( # (dimshuffle): Expression(expression=_transpose_time_to_spat) # (conv_time): Conv2d(1, 40, kernel_size=(25, 1), stride=(1, 1)) # (conv_spat): Conv2d(40, 40, kernel_size=(1, 64), stride=(1, 1), bias=False) # (bnorm): BatchNorm2d(40, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) # (conv_nonlin): Expression(expression=square) # (pool): AvgPool2d(kernel_size=(75, 1), stride=(15, 1), padding=0) # (pool_nonlin): Expression(expression=safe_log) # (drop): Dropout(p=0.5) # (conv_classifier): Conv2d(40, 2, kernel_size=(27, 1), stride=(1, 1)) # (softmax): LogSoftmax() # (squeeze): Expression(expression=_squeeze_final_output) # ) # Compute the loss loss = F.nll_loss(outputs, net_target) # Do the backpropagation loss.backward() # Update parameters with the optimizer optimizer.step() # Print some statistics each epoch model.eval() print("Epoch {:d}".format(i_epoch)) for setname, dataset in (('Train', train_set), ('Test', test_set)): # Here, we will use the entire dataset at once, which is still possible # for such smaller datasets. Otherwise we would have to use batches. net_in = np_to_var(dataset.X[:, :, :, None]) if cuda: net_in = net_in.cuda() net_target = np_to_var(dataset.y) if cuda: net_target = net_target.cuda() outputs = model(net_in) loss = F.nll_loss(outputs, net_target) losses.append(float(var_to_np(loss))) print("{:6s} Loss: {:.5f}".format(setname, float(var_to_np(loss)))) predicted_labels = np.argmax(var_to_np(outputs), axis=1) accuracy = np.mean(dataset.y == predicted_labels) accuracies.append(accuracy * 100) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) np.testing.assert_allclose(np.array(losses), np.array([ 1.1775966882705688, 1.2602351903915405, 0.7068756818771362, 0.9367912411689758, 0.394258975982666, 0.6598362326622009, 0.3359280526638031, 0.656258761882782, 0.2790488004684448, 0.6104397177696228, 0.27319177985191345, 0.5949864983558655 ]), rtol=1e-4, atol=1e-5) np.testing.assert_allclose(np.array(accuracies), np.array([ 51.666666666666671, 53.333333333333336, 63.333333333333329, 56.666666666666664, 86.666666666666671, 66.666666666666657, 90.0, 63.333333333333329, 96.666666666666671, 56.666666666666664, 96.666666666666671, 66.666666666666657 ]), rtol=1e-4, atol=1e-5)
def test_trialwise_decoding(): import mne from mne.io import concatenate_raws # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events = mne.find_events(raw, shortest_event=0, stim_channel='STI 014') # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) import numpy as np # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 from braindecode.datautil.signal_target import SignalAndTarget train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from torch import nn from braindecode.torch_ext.util import set_random_seeds # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length = auto ensures we only get a single output in the time dimension model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], final_conv_length='auto').create_network() if cuda: model.cuda() from torch import optim optimizer = optim.Adam(model.parameters()) from braindecode.torch_ext.util import np_to_var, var_to_np from braindecode.datautil.iterators import get_balanced_batches import torch.nn.functional as F from numpy.random import RandomState rng = RandomState((2017, 6, 30)) losses = [] accuracies = [] for i_epoch in range(6): i_trials_in_batch = get_balanced_batches(len(train_set.X), rng, shuffle=True, batch_size=30) # Set model to training mode model.train() for i_trials in i_trials_in_batch: # Have to add empty fourth dimension to X batch_X = train_set.X[i_trials][:, :, :, None] batch_y = train_set.y[i_trials] net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters optimizer.zero_grad() # Compute outputs of the network outputs = model(net_in) # Compute the loss loss = F.nll_loss(outputs, net_target) # Do the backpropagation loss.backward() # Update parameters with the optimizer optimizer.step() # Print some statistics each epoch model.eval() print("Epoch {:d}".format(i_epoch)) for setname, dataset in (('Train', train_set), ('Test', test_set)): # Here, we will use the entire dataset at once, which is still possible # for such smaller datasets. Otherwise we would have to use batches. net_in = np_to_var(dataset.X[:, :, :, None]) if cuda: net_in = net_in.cuda() net_target = np_to_var(dataset.y) if cuda: net_target = net_target.cuda() outputs = model(net_in) loss = F.nll_loss(outputs, net_target) losses.append(float(var_to_np(loss))) print("{:6s} Loss: {:.5f}".format(setname, float(var_to_np(loss)))) predicted_labels = np.argmax(var_to_np(outputs), axis=1) accuracy = np.mean(dataset.y == predicted_labels) accuracies.append(accuracy * 100) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) np.testing.assert_allclose(np.array(losses), np.array([ 1.1775966882705688, 1.2602351903915405, 0.7068756818771362, 0.9367912411689758, 0.394258975982666, 0.6598362326622009, 0.3359280526638031, 0.656258761882782, 0.2790488004684448, 0.6104397177696228, 0.27319177985191345, 0.5949864983558655 ]), rtol=1e-4, atol=1e-5) np.testing.assert_allclose(np.array(accuracies), np.array([ 51.666666666666671, 53.333333333333336, 63.333333333333329, 56.666666666666664, 86.666666666666671, 66.666666666666657, 90.0, 63.333333333333329, 96.666666666666671, 56.666666666666664, 96.666666666666671, 66.666666666666657 ]), rtol=1e-4, atol=1e-5)
def run_exp(data_folder, session_id, subject_id, low_cut_hz, model, cuda): ival = [-500, 4000] max_epochs = 1600 max_increase_epochs = 160 batch_size = 10 high_cut_hz = 38 factor_new = 1e-3 init_block_size = 1000 valid_set_fraction = .2 ''' # BCIcompetition train_filename = 'A{:02d}T.gdf'.format(subject_id) test_filename = 'A{:02d}E.gdf'.format(subject_id) train_filepath = os.path.join(data_folder, train_filename) test_filepath = os.path.join(data_folder, test_filename) train_label_filepath = train_filepath.replace('.gdf', '.mat') test_label_filepath = test_filepath.replace('.gdf', '.mat') train_loader = BCICompetition4Set2A( train_filepath, labels_filename=train_label_filepath) test_loader = BCICompetition4Set2A( test_filepath, labels_filename=test_label_filepath) train_cnt = train_loader.load() test_cnt = test_loader.load() ''' # GIGAscience filename = 'sess{:02d}_subj{:02d}_EEG_MI.mat'.format( session_id, subject_id) filepath = os.path.join(data_folder, filename) train_variable = 'EEG_MI_train' test_variable = 'EEG_MI_test' train_loader = GIGAscience(filepath, train_variable) test_loader = GIGAscience(filepath, test_variable) train_cnt = train_loader.load() test_cnt = test_loader.load() # Preprocessing ''' channel ['Fp1', 'Fp2', 'F7', 'F3', 'Fz', 'F4', 'F8', 'FC5', 'FC1', 'FC2', 'FC6', 'T7', 'C3', 'Cz', 'C4', 'T8', 'TP9', 'CP5', 'CP1', 'CP2', 'CP6', 'TP10', 'P7', 'P3', 'Pz', 'P4', 'P8', 'PO9', 'O1', 'Oz', 'O2', 'PO10', 'FC3', 'FC4', 'C5', 'C1', 'C2', 'C6', 'CP3', 'CPz', 'CP4', 'P1', 'P2', 'POz', 'FT9', 'FTT9h', 'TTP7h', 'TP7', 'TPP9h', 'FT10', 'FTT10h', 'TPP8h', 'TP8', 'TPP10h', 'F9', 'F10', 'AF7', 'AF3', 'AF4', 'AF8', 'PO3', 'PO4'] ''' train_cnt = train_cnt.pick_channels([ 'FC5', 'FC3', 'FC1', 'Fz', 'FC2', 'FC4', 'FC6', 'C5', 'C3', 'C1', 'Cz', 'C2', 'C4', 'C6', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'Pz' ]) train_cnt, train_cnt.info['events'] = train_cnt.copy().resample( 250, npad='auto', events=train_cnt.info['events']) assert len(train_cnt.ch_names) == 22 # lets convert to millvolt for numerical stability of next operations train_cnt = mne_apply(lambda a: a * 1e6, train_cnt) train_cnt = mne_apply( lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz, train_cnt.info['sfreq'], filt_order=3, axis=1), train_cnt) train_cnt = mne_apply( lambda a: exponential_running_standardize(a.T, factor_new=factor_new, init_block_size= init_block_size, eps=1e-4).T, train_cnt) test_cnt = test_cnt.pick_channels([ 'FC5', 'FC3', 'FC1', 'Fz', 'FC2', 'FC4', 'FC6', 'C5', 'C3', 'C1', 'Cz', 'C2', 'C4', 'C6', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'Pz' ]) test_cnt, test_cnt.info['events'] = test_cnt.copy().resample( 250, npad='auto', events=test_cnt.info['events']) assert len(test_cnt.ch_names) == 22 test_cnt = mne_apply(lambda a: a * 1e6, test_cnt) test_cnt = mne_apply( lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz, test_cnt.info['sfreq'], filt_order=3, axis=1), test_cnt) test_cnt = mne_apply( lambda a: exponential_running_standardize(a.T, factor_new=factor_new, init_block_size= init_block_size, eps=1e-4).T, test_cnt) marker_def = OrderedDict([('Right Hand', [1]), ('Left Hand', [2])]) train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival) test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival) train_set, valid_set = split_into_two_sets(train_set, first_set_fraction=1 - valid_set_fraction) set_random_seeds(seed=20190706, cuda=cuda) n_classes = 2 n_chans = int(train_set.X.shape[1]) input_time_length = train_set.X.shape[2] if model == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() elif model == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length='auto').create_network() if cuda: model.cuda() log.info("Model: \n{:s}".format(str(model))) optimizer = optim.Adam(model.parameters()) iterator = BalancedBatchSizeIterator(batch_size=batch_size) stop_criterion = Or([ MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs) ]) monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()] model_constraint = MaxNormDefaultConstraint() exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=F.nll_loss, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, cuda=cuda) exp.run() return exp
def run_exp_on_high_gamma_dataset(train_filename, test_filename, low_cut_hz, model_name, max_epochs, max_increase_epochs, np_th_seed, debug): train_set, valid_set, test_set = load_train_valid_test( train_filename=train_filename, test_filename=test_filename, low_cut_hz=low_cut_hz, debug=debug) if debug: max_epochs = 4 set_random_seeds(np_th_seed, cuda=True) #torch.backends.cudnn.benchmark = True# sometimes crashes? n_classes = int(np.max(train_set.y) + 1) n_chans = int(train_set.X.shape[1]) input_time_length = 1000 if model_name == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=2).create_network() elif model_name == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=30).create_network() to_dense_prediction_model(model) model.cuda() model.eval() out = model(np_to_var(train_set.X[:1, :, :input_time_length, None]).cuda()) n_preds_per_input = out.cpu().data.numpy().shape[2] optimizer = optim.Adam(model.parameters(), weight_decay=0, lr=1e-3) iterator = CropsFromTrialsIterator(batch_size=60, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input, seed=np_th_seed) monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length=input_time_length), RuntimeMonitor() ] model_constraint = MaxNormDefaultConstraint() loss_function = lambda preds, targets: F.nll_loss(th.mean(preds, dim=2), targets) run_after_early_stop = True do_early_stop = True remember_best_column = 'valid_misclass' stop_criterion = Or([ MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs) ]) exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=loss_function, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column=remember_best_column, run_after_early_stop=run_after_early_stop, cuda=True, do_early_stop=do_early_stop) exp.run() return exp
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda): ival = [-500, 4000] input_time_length = 1000 max_epochs = 800 max_increase_epochs = 80 batch_size = 60 high_cut_hz = 38 factor_new = 1e-3 init_block_size = 1000 valid_set_fraction = 0.2 train_filename = 'A{:02d}T.gdf'.format(subject_id) test_filename = 'A{:02d}E.gdf'.format(subject_id) train_filepath = os.path.join(data_folder, train_filename) test_filepath = os.path.join(data_folder, test_filename) train_label_filepath = train_filepath.replace('.gdf', '.mat') test_label_filepath = test_filepath.replace('.gdf', '.mat') train_loader = BCICompetition4Set2A(train_filepath, labels_filename=train_label_filepath) test_loader = BCICompetition4Set2A(test_filepath, labels_filename=test_label_filepath) train_cnt = train_loader.load() test_cnt = test_loader.load() # Preprocessing train_cnt = train_cnt.drop_channels( ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right']) assert len(train_cnt.ch_names) == 22 # lets convert to millvolt for numerical stability of next operations train_cnt = mne_apply(lambda a: a * 1e6, train_cnt) train_cnt = mne_apply( lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz, train_cnt.info['sfreq'], filt_order=3, axis=1), train_cnt) train_cnt = mne_apply( lambda a: exponential_running_standardize(a.T, factor_new=factor_new, init_block_size= init_block_size, eps=1e-4).T, train_cnt) test_cnt = test_cnt.drop_channels( ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right']) assert len(test_cnt.ch_names) == 22 test_cnt = mne_apply(lambda a: a * 1e6, test_cnt) test_cnt = mne_apply( lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz, test_cnt.info['sfreq'], filt_order=3, axis=1), test_cnt) test_cnt = mne_apply( lambda a: exponential_running_standardize(a.T, factor_new=factor_new, init_block_size= init_block_size, eps=1e-4).T, test_cnt) marker_def = OrderedDict([('Left Hand', [1]), ( 'Right Hand', [2], ), ('Foot', [3]), ('Tongue', [4])]) train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival) test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival) train_set, valid_set = split_into_two_sets(train_set, first_set_fraction=1 - valid_set_fraction) set_random_seeds(seed=20190706, cuda=cuda) n_classes = 4 n_chans = int(train_set.X.shape[1]) if model == 'shallow': model = ShallowFBCSPNet(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=30).create_network() elif model == 'deep': model = Deep4Net(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=2).create_network() to_dense_prediction_model(model) if cuda: model.cuda() log.info("Model: \n{:s}".format(str(model))) dummy_input = np_to_var(train_set.X[:1, :, :, None]) if cuda: dummy_input = dummy_input.cuda() out = model(dummy_input) n_preds_per_input = out.cpu().data.numpy().shape[2] optimizer = optim.Adam(model.parameters()) iterator = CropsFromTrialsIterator(batch_size=batch_size, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) stop_criterion = Or([ MaxEpochs(max_epochs), NoDecrease('valid_misclass', max_increase_epochs) ]) monitors = [ LossMonitor(), MisclassMonitor(col_suffix='sample_misclass'), CroppedTrialMisclassMonitor(input_time_length=input_time_length), RuntimeMonitor() ] model_constraint = MaxNormDefaultConstraint() loss_function = lambda preds, targets: F.nll_loss( th.mean(preds, dim=2, keepdim=False), targets) exp = Experiment(model, train_set, valid_set, test_set, iterator=iterator, loss_function=loss_function, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column='valid_misclass', run_after_early_stop=True, cuda=cuda) exp.run() return exp
def setup_exp( train_folder, n_recordings, n_chans, model_name, n_start_chans, n_chan_factor, input_time_length, final_conv_length, model_constraint, stride_before_pool, init_lr, batch_size, max_epochs, cuda, num_workers, task, weight_decay, n_folds, shuffle_folds, lazy_loading, eval_folder, result_folder, run_on_normals, run_on_abnormals, seed, l2_decay, gradient_clip, ): info_msg = "using {}, {}".format( os.environ["SLURM_JOB_PARTITION"], os.environ["SLURMD_NODENAME"], ) info_msg += ", gpu {}".format(os.environ["CUDA_VISIBLE_DEVICES"]) logging.info(info_msg) logging.info("Targets for this task: <{}>".format(task)) import torch.backends.cudnn as cudnn cudnn.benchmark = True if task == "age": loss_function = mse_loss_on_mean remember_best_column = "valid_rmse" n_classes = 1 else: loss_function = nll_loss_on_mean remember_best_column = "valid_misclass" n_classes = 2 if model_constraint is not None: assert model_constraint == 'defaultnorm' model_constraint = MaxNormDefaultConstraint() stop_criterion = MaxEpochs(max_epochs) set_random_seeds(seed=seed, cuda=cuda) if model_name == 'shallow': model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == 'deep': model = Deep4Net( n_chans, n_classes, n_filters_time=n_start_chans, n_filters_spat=n_start_chans, input_time_length=input_time_length, n_filters_2=int(n_start_chans * n_chan_factor), n_filters_3=int(n_start_chans * (n_chan_factor**2.0)), n_filters_4=int(n_start_chans * (n_chan_factor**3.0)), final_conv_length=final_conv_length, stride_before_pool=stride_before_pool).create_network() elif model_name == 'eegnet': model = EEGNetv4(n_chans, n_classes, input_time_length=input_time_length, final_conv_length=final_conv_length).create_network() elif model_name == "tcn": assert task != "age", "what to change to do regression with tcn?!" model = TemporalConvNet(input_size=n_chans, output_size=n_classes, context_size=0, num_channels=55, num_levels=5, kernel_size=16, dropout=0.05270154233150525, skip_mode=None, use_context=0, lasso_selection=0.0, rnn_normalization=None) else: assert False, "unknown model name {:s}".format(model_name) if task == "age": # remove softmax layer, set n_classes to 1 model.n_classes = 1 new_model = nn.Sequential() for name, module_ in model.named_children(): if name == "softmax": continue new_model.add_module(name, module_) model = new_model # maybe check if this works and wait / re-try after some time? # in case of all cuda devices are busy if cuda: model.cuda() if model_name != "tcn": to_dense_prediction_model(model) logging.info("Model:\n{:s}".format(str(model))) test_input = np_to_var( np.ones((2, n_chans, input_time_length, 1), dtype=np.float32)) if list(model.parameters())[0].is_cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] if eval_folder is None: logging.info("will do validation") if lazy_loading: logging.info( "using lazy loading to load {} recs".format(n_recordings)) dataset = TuhLazy(train_folder, target=task, n_recordings=n_recordings) else: logging.info("using traditional loading to load {} recs".format( n_recordings)) dataset = Tuh(train_folder, n_recordings=n_recordings, target=task) assert not (run_on_normals and run_on_abnormals), ( "decide whether to run on normal or abnormal subjects") # only run on normal subjects if run_on_normals: ids = [ i for i in range(len(dataset)) if dataset.pathologicals[i] == 0 ] # 0 is non-pathological dataset = TuhSubset(dataset, ids) logging.info("only using {} normal subjects".format(len(dataset))) if run_on_abnormals: ids = [ i for i in range(len(dataset)) if dataset.pathologicals[i] == 1 ] # 1 is pathological dataset = TuhSubset(dataset, ids) logging.info("only using {} abnormal subjects".format( len(dataset))) indices = np.arange(len(dataset)) kf = KFold(n_splits=n_folds, shuffle=shuffle_folds) for i, (train_ind, test_ind) in enumerate(kf.split(indices)): assert len(np.intersect1d( train_ind, test_ind)) == 0, ("train and test set overlap!") # seed is in range of number of folds and was set by submit script if i == seed: break if lazy_loading: test_subset = TuhLazySubset(dataset, test_ind) train_subset = TuhLazySubset(dataset, train_ind) else: test_subset = TuhSubset(dataset, test_ind) train_subset = TuhSubset(dataset, train_ind) else: logging.info("will do final evaluation") if lazy_loading: train_subset = TuhLazy(train_folder, target=task) test_subset = TuhLazy(eval_folder, target=task) else: train_subset = Tuh(train_folder, target=task) test_subset = Tuh(eval_folder, target=task) # remove rec: # train/abnormal/01_tcp_ar/081/00008184/s001_2011_09_21/00008184_s001_t001 # since it contains no crop without outliers (channels A1, A2 broken) subjects = [f.split("/")[-3] for f in train_subset.file_paths] if "00008184" in subjects: bad_id = subjects.index("00008184") train_subset = remove_file_from_dataset( train_subset, file_id=bad_id, file=("train/abnormal/01_tcp_ar/081/00008184/s001_2011_09_21/" "00008184_s001_t001")) subjects = [f.split("/")[-3] for f in test_subset.file_paths] if "00008184" in subjects: bad_id = subjects.index("00008184") test_subset = remove_file_from_dataset( test_subset, file_id=bad_id, file=("train/abnormal/01_tcp_ar/081/00008184/s001_2011_09_21/" "00008184_s001_t001")) if task == "age": # standardize ages based on train set y_train = train_subset.y y_train_mean = np.mean(y_train) y_train_std = np.std(y_train) train_subset.y = (y_train - y_train_mean) / y_train_std y_test = test_subset.y test_subset.y = (y_test - y_train_mean) / y_train_std if lazy_loading: iterator = LazyCropsFromTrialsIterator( input_time_length, n_preds_per_input, batch_size, seed=seed, num_workers=num_workers, reset_rng_after_each_batch=False, check_preds_smaller_trial_len=False) # True! else: iterator = CropsFromTrialsIterator(batch_size, input_time_length, n_preds_per_input, seed) monitors = [] monitors.append(LossMonitor()) monitors.append(RAMMonitor()) monitors.append(RuntimeMonitor()) if task == "age": monitors.append( RMSEMonitor(input_time_length, n_preds_per_input, mean=y_train_mean, std=y_train_std)) else: monitors.append( CroppedDiagnosisMonitor(input_time_length, n_preds_per_input)) monitors.append(LazyMisclassMonitor(col_suffix='sample_misclass')) if lazy_loading: n_updates_per_epoch = len( iterator.get_batches(train_subset, shuffle=False)) else: n_updates_per_epoch = sum( [1 for _ in iterator.get_batches(train_subset, shuffle=False)]) n_updates_per_period = n_updates_per_epoch * max_epochs logging.info("there are {} updates per epoch".format(n_updates_per_epoch)) if model_name == "tcn": adamw = ExtendedAdam(model.parameters(), lr=init_lr, weight_decay=weight_decay, l2_decay=l2_decay, gradient_clip=gradient_clip) else: adamw = AdamWWithTracking(model.parameters(), init_lr, weight_decay=weight_decay) scheduler = CosineAnnealing(n_updates_per_period) optimizer = ScheduledOptimizer(scheduler, adamw, schedule_weight_decay=True) exp = Experiment(model=model, train_set=train_subset, valid_set=None, test_set=test_subset, iterator=iterator, loss_function=loss_function, optimizer=optimizer, model_constraint=model_constraint, monitors=monitors, stop_criterion=stop_criterion, remember_best_column=remember_best_column, run_after_early_stop=False, batch_modifier=None, cuda=cuda, do_early_stop=False, reset_after_second_run=False) return exp