def test_set_random_seeds_warning(): torch.backends.cudnn.benchmark = True with pytest.warns(UserWarning, match="torch.backends.cudnn.benchmark was set to True which may results in " "lack of reproducibility. In some cases to ensure reproducibility you " "may need to set torch.backends.cudnn.benchmark to False."): set_random_seeds(100, True)
def get_deep_learning_model(model_args, valid_dataset): cuda = torch.cuda.is_available() device = model_args["device"] if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = model_args["seed"] # = 20200220 random seed to make results reproducible # Set random seed to be able to reproduce results if seed: set_random_seeds(seed=seed, cuda=cuda) if model_args["model_type"] == "ShallowFBCSPNet": model = ShallowFBCSPNet( model_args["n_chans"], model_args["n_classes"] + 1, input_window_samples=model_args["input_window_samples"], final_conv_length='auto', ) elif model_args["model_type"] == "SleepStager": model = model = SleepStager( n_channels=model_args["n_chans"], sfreq=model_args["sfreq"], n_classes=model_args["n_classes"] + 1, input_size_s=model_args["input_window_samples"] / model_args["sfreq"], ) else: raise ValueError("Boom !") if cuda: model.cuda() clf = EEGClassifier( model, criterion=model_args["criterion"], optimizer=torch.optim.AdamW, # using test_sample for validation train_split=predefined_split(valid_dataset), optimizer__lr=model_args["lr"], optimizer__weight_decay=model_args["weight_decay"], batch_size=model_args["batch_size"], callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=model_args["n_epochs"] - 1)), ("early_stopping", EarlyStopping(monitor='valid_loss', patience=model_args["patience"])) ], device=device, iterator_train__num_workers=20, iterator_train__pin_memory=True) # torch.in torch.out return clf
def test_compute_amplitude_gradients_for_X(): # If the weights are initalized with a sine function # gradient of amplitude should be only in one frequency bin set_random_seeds(948, False) model = nn.Conv1d(1, 1, 16) # torch.linspace(,,n)[:n-1] is same as np.linspace(,,n,endpoint=False) model.weight.data[:, :, :] = torch.sin( torch.linspace(0, 2 * np.pi, 17)[:16]) model.bias.data[:] = 0 grads = compute_amplitude_gradients_for_X(model, torch.randn(1, 1, 16)) grads = grads.squeeze() assert np.abs(grads[1]) / np.sum(np.abs(grads)) > 0.99
def test_tcn(): set_random_seeds(0, False) tcn = TCN( n_in_chans=21, n_outputs=2, n_filters=55, n_blocks=5, kernel_size=16, drop_prob=0.05270154233150525, add_log_softmax=True ) # braindecode models are always in eval mode after initialization # original model implementation was not tcn.train() x = torch.rand(1, 21, 1000, 1) out = tcn(x) # this is the output of the original model implementation using the same # initialization arguments as above expected = np.array( [[[-0.5504, -0.5304, -0.6023, -0.5231, -0.5387, -0.5522, -0.5323, -0.5540, -0.5297, -0.5333, -0.5743, -0.5330, -0.5117, -0.5051, -0.5523, -0.5507, -0.5724, -0.5380, -0.5697, -0.4871, -0.5400, -0.4986, -0.5502, -0.5524, -0.5263, -0.5440, -0.5464, -0.5005, -0.5404, -0.5098, -0.5197, -0.5578, -0.5419, -0.5601, -0.5031, -0.5616, -0.5205, -0.5378, -0.5472, -0.4897, -0.5216, -0.5560, -0.5480, -0.5488, -0.5258, -0.5637, -0.5318, -0.5134, -0.5460, -0.5294, -0.5513, -0.5310, -0.5307, -0.5326, -0.5270, -0.5156, -0.5569, -0.5416, -0.5279, -0.5553, -0.5589, -0.5166, -0.5108, -0.5076, -0.5279, -0.5208, -0.5367, -0.5557, -0.5690, -0.5494], [-0.8597, -0.8877, -0.7931, -0.8982, -0.8758, -0.8573, -0.8849, -0.8549, -0.8887, -0.8834, -0.8280, -0.8839, -0.9150, -0.9250, -0.8572, -0.8593, -0.8305, -0.8769, -0.8340, -0.9530, -0.8741, -0.9350, -0.8600, -0.8570, -0.8935, -0.8685, -0.8652, -0.9319, -0.8735, -0.9179, -0.9031, -0.8497, -0.8714, -0.8466, -0.9280, -0.8447, -0.9019, -0.8771, -0.8640, -0.9489, -0.9003, -0.8521, -0.8630, -0.8619, -0.8942, -0.8419, -0.8856, -0.9124, -0.8658, -0.8890, -0.8585, -0.8867, -0.8872, -0.8845, -0.8924, -0.9092, -0.8509, -0.8718, -0.8912, -0.8531, -0.8482, -0.9077, -0.9163, -0.9212, -0.8912, -0.9016, -0.8787, -0.8525, -0.8349, -0.8611]]]) np.testing.assert_allclose( out.detach().numpy(), expected, rtol=1e-3, atol=1e-3)
def check_forward_pass(model, input_sizes, only_check_until_dim=None): # Test 4d Input set_random_seeds(0, False) rng = np.random.RandomState(42) X = rng.randn(input_sizes['n_samples'], input_sizes['n_channels'], input_sizes['n_in_times'], 1) X = torch.Tensor(X.astype(np.float32)) y_pred = model(X) assert y_pred.shape[:only_check_until_dim] == (input_sizes['n_samples'], input_sizes['n_classes']) # Test 3d input set_random_seeds(0, False) X = X.squeeze(-1) assert len(X.shape) == 3 y_pred_new = model(X) assert y_pred_new.shape[:only_check_until_dim] == ( input_sizes['n_samples'], input_sizes['n_classes']) np.testing.assert_allclose(y_pred.detach().cpu().numpy(), y_pred_new.detach().cpu().numpy())
def test_trialwise_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) raw.apply_function(lambda x: x * 1000000) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) ds = EpochsDataset(epoched) train_set = Subset(ds, np.arange(60)) valid_set = Subset(ds, np.arange(60, len(ds))) train_valid_split = predefined_split(valid_set) cuda = False if cuda: device = 'cuda' else: device = 'cpu' set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set[0][0].shape[0] input_window_samples = train_set[0][0].shape[1] model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_window_samples=input_window_samples, final_conv_length="auto", ) if cuda: model.cuda() clf = EEGClassifier( model, cropped=False, criterion=torch.nn.NLLLoss, optimizer=torch.optim.Adam, train_split=train_valid_split, optimizer__lr=0.001, batch_size=30, callbacks=["accuracy"], device=device, ) clf.fit(train_set, y=None, epochs=6) np.testing.assert_allclose( clf.history[:, 'train_loss'], np.array([ 1.1114967465400696, 1.0180627405643463, 0.8020123243331909, 0.8934760391712189, 0.8401200771331787, 0.5898805856704712 ]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_loss'], np.array([ 0.8467752933502197, 1.0855580568313599, 0.873993992805481, 0.8403236865997314, 0.8534432053565979, 0.8854812383651733 ]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'train_accuracy'], np.array( [0.7166666666666667, 0.6666666666666666, 0.8, 0.9, 0.95, 0.95]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_accuracy'], np.array([ 0.6, 0.5666666666666667, 0.5666666666666667, 0.5, 0.5333333333333333, 0.6333333333333333 ]), rtol=1e-4, atol=1e-5, )
def test_trialwise_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data( subject_id, event_codes, update_path=False ) # Load each of the files parts = [ mne.io.read_raw_edf( path, preload=True, stim_channel="auto", verbose="WARNING" ) for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types( raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads" ) # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 SignalAndTarget = namedtuple("SignalAndTarget", "X y") train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length = auto ensures we only get a single output in the time dimension model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], final_conv_length="auto", ) if cuda: model.cuda() optimizer = optim.Adam(model.parameters()) rng = RandomState((2017, 6, 30)) losses = [] accuracies = [] for i_epoch in range(6): i_trials_in_batch = get_balanced_batches( len(train_set.X), rng, shuffle=True, batch_size=30 ) # Set model to training mode model.train() for i_trials in i_trials_in_batch: # Have to add empty fourth dimension to X batch_X = train_set.X[i_trials][:, :, :, None] batch_y = train_set.y[i_trials] net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters optimizer.zero_grad() # Compute outputs of the network outputs = model(net_in) # Compute the loss loss = F.nll_loss(outputs, net_target) # Do the backpropagation loss.backward() # Update parameters with the optimizer optimizer.step() # Print some statistics each epoch model.eval() print("Epoch {:d}".format(i_epoch)) for setname, dataset in (("Train", train_set), ("Test", test_set)): # Here, we will use the entire dataset at once, which is still possible # for such smaller datasets. Otherwise we would have to use batches. net_in = np_to_var(dataset.X[:, :, :, None]) if cuda: net_in = net_in.cuda() net_target = np_to_var(dataset.y) if cuda: net_target = net_target.cuda() outputs = model(net_in) loss = F.nll_loss(outputs, net_target) losses.append(float(var_to_np(loss))) print("{:6s} Loss: {:.5f}".format(setname, float(var_to_np(loss)))) predicted_labels = np.argmax(var_to_np(outputs), axis=1) accuracy = np.mean(dataset.y == predicted_labels) accuracies.append(accuracy * 100) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) np.testing.assert_allclose( np.array(losses), np.array( [ 0.91796708, 1.2714895, 0.4999536, 0.94365239, 0.39268905, 0.89928466, 0.37648854, 0.8940345, 0.35774994, 0.86749417, 0.35080773, 0.80767328, ] ), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( np.array(accuracies), np.array( [ 55.0, 63.33333333, 71.66666667, 63.33333333, 81.66666667, 60.0, 78.33333333, 63.33333333, 83.33333333, 66.66666667, 80.0, 66.66666667, ] ), rtol=1e-4, atol=1e-5, )
def exp(subject_id): import torch test_subj = np.r_[subject_id] print('test subj:' + str(test_subj)) # train_subj = np.setdiff1d(np.r_[1:10], test_subj) train_subj = np.setdiff1d(np.r_[1, 3, 7, 8], test_subj) tr = [] val = [] for ids in train_subj: train_size = int(0.99 * len(splitted[ids])) test_size = len(splitted[ids]) - train_size tr_i, val_i = torch.utils.data.random_split(splitted[ids], [train_size, test_size]) tr.append(tr_i) val.append(val_i) train_set = torch.utils.data.ConcatDataset(tr) valid_set = torch.utils.data.ConcatDataset(val) valid_set = BaseConcatDataset([splitted[ids] for ids in test_subj]) ###################################################################### # Create model # ------------ # ###################################################################### # Now we create the deep learning model! Braindecode comes with some # predefined convolutional neural network architectures for raw # time-domain EEG. Here, we use the shallow ConvNet model from `Deep # learning with convolutional neural networks for EEG decoding and # visualization <https://arxiv.org/abs/1703.05051>`__. These models are # pure `PyTorch <https://pytorch.org>`__ deep learning models, therefore # to use your own model, it just has to be a normal PyTorch # `nn.Module <https://pytorch.org/docs/stable/nn.html#torch.nn.Module>`__. # import torch from braindecode.util import set_random_seeds from braindecode.models import ShallowFBCSPNet, Deep4Net cuda = torch.cuda.is_available( ) # check if GPU is available, if True chooses to use it device = 'cuda:0' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = 20200220 # random seed to make results reproducible # Set random seed to be able to reproduce results set_random_seeds(seed=seed, cuda=cuda) n_classes = 3 # Extract number of chans and time steps from dataset n_chans = train_set[0][0].shape[0] input_window_samples = train_set[0][0].shape[1] # # model = ShallowFBCSPNet( # n_chans, # n_classes, # input_window_samples=input_window_samples, # final_conv_length='auto', # ) from mynetworks import Deep4Net_origin, ConvClfNet, FcClfNet model = Deep4Net( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length="auto", ) # # embedding_net = Deep4Net_origin(4, 22, input_window_samples) # model = FcClfNet(embedding_net) # # print(model) # Send model to GPU if cuda: model.cuda() ###################################################################### # Training # -------- # ###################################################################### # Now we train the network! EEGClassifier is a Braindecode object # responsible for managing the training of neural networks. It inherits # from skorch.NeuralNetClassifier, so the training logic is the same as in # `Skorch <https://skorch.readthedocs.io/en/stable/>`__. # ###################################################################### # **Note**: In this tutorial, we use some default parameters that we # have found to work well for motor decoding, however we strongly # encourage you to perform your own hyperparameter optimization using # cross validation on your training data. # from skorch.callbacks import LRScheduler from skorch.helper import predefined_split from braindecode import EEGClassifier # # These values we found good for shallow network: lr = 0.0625 * 0.01 weight_decay = 0 # For deep4 they should be: # lr = 1 * 0.01 # weight_decay = 0.5 * 0.001 batch_size = 8 n_epochs = 100 clf = EEGClassifier( model, criterion=torch.nn.NLLLoss, optimizer=torch.optim.AdamW, train_split=predefined_split( valid_set), # using valid_set for validation optimizer__lr=lr, optimizer__weight_decay=weight_decay, batch_size=batch_size, callbacks=[ "accuracy", ("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)), ], device=device, ) # Model training for a specified number of epochs. `y` is None as it is already supplied # in the dataset. clf.fit(train_set, y=None, epochs=n_epochs) ###################################################################### # Plot Results # ------------ # ###################################################################### # Now we use the history stored by Skorch throughout training to plot # accuracy and loss curves. # import matplotlib.pyplot as plt from matplotlib.lines import Line2D import pandas as pd # Extract loss and accuracy values for plotting from history object results_columns = [ 'train_loss', 'valid_loss', 'train_accuracy', 'valid_accuracy' ] df = pd.DataFrame(clf.history[:, results_columns], columns=results_columns, index=clf.history[:, 'epoch']) # get percent of misclass for better visual comparison to loss df = df.assign(train_misclass=100 - 100 * df.train_accuracy, valid_misclass=100 - 100 * df.valid_accuracy) plt.style.use('seaborn') fig, ax1 = plt.subplots(figsize=(8, 3)) df.loc[:, ['train_loss', 'valid_loss']].plot(ax=ax1, style=['-', ':'], marker='o', color='tab:blue', legend=False, fontsize=14) ax1.tick_params(axis='y', labelcolor='tab:blue', labelsize=14) ax1.set_ylabel("Loss", color='tab:blue', fontsize=14) ax2 = ax1.twinx() # instantiate a second axes that shares the same x-axis df.loc[:, ['train_misclass', 'valid_misclass']].plot(ax=ax2, style=['-', ':'], marker='o', color='tab:red', legend=False) ax2.tick_params(axis='y', labelcolor='tab:red', labelsize=14) ax2.set_ylabel("Misclassification Rate [%]", color='tab:red', fontsize=14) ax2.set_ylim(ax2.get_ylim()[0], 85) # make some room for legend ax1.set_xlabel("Epoch", fontsize=14) # where some data has already been plotted to ax handles = [] handles.append( Line2D([0], [0], color='black', linewidth=1, linestyle='-', label='Train')) handles.append( Line2D([0], [0], color='black', linewidth=1, linestyle=':', label='Valid')) plt.legend(handles, [h.get_label() for h in handles], fontsize=14) plt.tight_layout() # plt.show() return df
def test_eeg_classifier(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_window_samples = 450 n_classes = 2 in_chans = X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_window_samples=input_window_samples, final_conv_length=12, ) to_dense_prediction_model(model) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_window_samples, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] train_set = create_from_X_y(X[:48], y[:48], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) valid_set = create_from_X_y(X[48:60], y[48:60], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) cropped_cb_train = CroppedTrialEpochScoring( "accuracy", name="train_trial_accuracy", lower_is_better=False, on_train=True, ) cropped_cb_valid = CroppedTrialEpochScoring( "accuracy", on_train=False, name="valid_trial_accuracy", lower_is_better=False, ) clf = EEGClassifier( model, criterion=CroppedLoss, criterion__loss_function=nll_loss, optimizer=optim.Adam, train_split=predefined_split(valid_set), batch_size=32, callbacks=[ ("train_trial_accuracy", cropped_cb_train), ("valid_trial_accuracy", cropped_cb_valid), ], ) clf.fit(train_set, y=None, epochs=4) expected = [{ 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.9391239881515503 }, { 'train_batch_size': 32, 'train_loss': 2.895704507827759 }, { 'train_batch_size': 32, 'train_loss': 1.0713887214660645 }, { 'valid_batch_size': 24, 'valid_loss': 1.18110191822052 }], 'epoch': 1, 'train_batch_count': 3, 'train_loss': 1.9687390724817913, 'train_loss_best': True, 'train_trial_accuracy': 0.4791666666666667, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 1.18110191822052, 'valid_loss_best': True, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': True }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.6741573810577393 }, { 'train_batch_size': 32, 'train_loss': 0.9984264373779297 }, { 'train_batch_size': 32, 'train_loss': 1.1340471506118774 }, { 'valid_batch_size': 24, 'valid_loss': 2.5375664234161377 }], 'epoch': 2, 'train_batch_count': 3, 'train_loss': 1.2688769896825154, 'train_loss_best': True, 'train_trial_accuracy': 0.5, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 2.5375664234161377, 'valid_loss_best': False, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': False }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 0.8795645833015442 }, { 'train_batch_size': 32, 'train_loss': 1.0339491367340088 }, { 'train_batch_size': 32, 'train_loss': 1.19275963306427 }, { 'valid_batch_size': 24, 'valid_loss': 1.655737042427063 }], 'epoch': 3, 'train_batch_count': 3, 'train_loss': 1.0354244510332744, 'train_loss_best': True, 'train_trial_accuracy': 0.5, 'train_trial_accuracy_best': False, 'valid_batch_count': 1, 'valid_loss': 1.655737042427063, 'valid_loss_best': False, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': False }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.1963350772857666 }, { 'train_batch_size': 32, 'train_loss': 0.8621770143508911 }, { 'train_batch_size': 32, 'train_loss': 1.099318265914917 }, { 'valid_batch_size': 24, 'valid_loss': 1.0293445587158203 }], 'epoch': 4, 'train_batch_count': 3, 'train_loss': 1.0526101191838582, 'train_loss_best': False, 'train_trial_accuracy': 0.625, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 1.0293445587158203, 'valid_loss_best': True, 'valid_trial_accuracy': 0.25, 'valid_trial_accuracy_best': False }] history_without_dur = [{k: v for k, v in h.items() if k != "dur"} for h in clf.history] assert_deep_allclose(expected, history_without_dur, atol=1e-3, rtol=1e-3)
tmin=1, tmax=4.1, proj=False, picks=picks, baseline=None, preload=True, ) X = (epochs.get_data() * 1e6).astype(np.float32) y = (epochs.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 del epochs # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = X.shape[1] class EEGDataSet(Dataset): def __init__(self, X, y): self.X = X if self.X.ndim == 3: self.X = self.X[:, :, :, None] self.y = y def __len__(self): return len(self.X) def __getitem__(self, idx):
def test_eeg_classifier(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12, ) to_dense_prediction_model(model) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] train_set = CroppedXyDataset(X[:60], y=y[:60], input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) cropped_cb_train = CroppedTrialEpochScoring( "accuracy", name="train_trial_accuracy", lower_is_better=False, on_train=True, ) cropped_cb_valid = CroppedTrialEpochScoring( "accuracy", on_train=False, name="valid_trial_accuracy", lower_is_better=False, ) clf = EEGClassifier( model, criterion=CroppedNLLLoss, optimizer=optim.Adam, train_split=TrainTestSplit( train_size=0.8, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input, ), batch_size=32, callbacks=[ ("train_trial_accuracy", cropped_cb_train), ("valid_trial_accuracy", cropped_cb_valid), ], ) clf.fit(train_set.X, train_set.y, epochs=4) expected = [ { "batches": [ { "train_loss": 2.0750882625579834, "train_batch_size": 32 }, { "train_loss": 3.09424090385437, "train_batch_size": 32 }, { "train_loss": 1.079931378364563, "train_batch_size": 32 }, { "valid_loss": 2.3208131790161133, "valid_batch_size": 24 }, ], "epoch": 1, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 2.083086848258972, "train_loss_best": True, "valid_loss": 2.3208131790161133, "valid_loss_best": True, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": True, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": True, }, { "batches": [ { "train_loss": 1.827332615852356, "train_batch_size": 32 }, { "train_loss": 1.4135494232177734, "train_batch_size": 32 }, { "train_loss": 1.1295170783996582, "train_batch_size": 32 }, { "valid_loss": 1.4291356801986694, "valid_batch_size": 24 }, ], "epoch": 2, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.4567997058232625, "train_loss_best": True, "valid_loss": 1.4291356801986694, "valid_loss_best": True, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": False, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, { "batches": [ { "train_loss": 1.1495535373687744, "train_batch_size": 32 }, { "train_loss": 2.356320381164551, "train_batch_size": 32 }, { "train_loss": 0.9548418521881104, "train_batch_size": 32 }, { "valid_loss": 2.248246908187866, "valid_batch_size": 24 }, ], "epoch": 3, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.4869052569071453, "train_loss_best": False, "valid_loss": 2.248246908187866, "valid_loss_best": False, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": False, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, { "batches": [ { "train_loss": 1.2157528400421143, "train_batch_size": 32 }, { "train_loss": 1.1182057857513428, "train_batch_size": 32 }, { "train_loss": 0.9163083434104919, "train_batch_size": 32 }, { "valid_loss": 0.9732739925384521, "valid_batch_size": 24 }, ], "epoch": 4, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.083422323067983, "train_loss_best": True, "valid_loss": 0.9732739925384521, "valid_loss_best": True, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": False, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, ] history_without_dur = [{k: v for k, v in h.items() if k != "dur"} for h in clf.history] assert_deep_allclose(history_without_dur, expected, atol=1e-3, rtol=1e-3)
def exp(subject_id): dataset = MOABBDataset(dataset_name="BNCI2014001", subject_ids=subject_id) from braindecode.datautil.preprocess import exponential_moving_standardize from braindecode.datautil.preprocess import MNEPreproc, NumpyPreproc, preprocess low_cut_hz = 0. # low cut frequency for filtering high_cut_hz = 49. # high cut frequency for filtering # Parameters for exponential moving standardization factor_new = 1e-3 init_block_size = 1000 preprocessors = [ # keep only EEG sensors MNEPreproc(fn='pick_types', eeg=True, meg=False, stim=False), # convert from volt to microvolt, directly modifying the numpy array NumpyPreproc(fn=lambda x: x * 1e6), # bandpass filter MNEPreproc(fn='filter', l_freq=low_cut_hz, h_freq=high_cut_hz), # exponential moving standardization # NumpyPreproc(fn=exponential_moving_standardize, factor_new=factor_new, # init_block_size=init_block_size) ] # Transform the data preprocess(dataset, preprocessors) ###################################################################### # Create model and compute windowing parameters # --------------------------------------------- # ###################################################################### # In contrast to trialwise decoding, we first have to create the model # before we can cut the dataset into windows. This is because we need to # know the receptive field of the network to know how large the window # stride should be. # ###################################################################### # We first choose the compute/input window size that will be fed to the # network during training This has to be larger than the networks # receptive field size and can otherwise be chosen for computational # efficiency (see explanations in the beginning of this tutorial). Here we # choose 1000 samples, which are 4 seconds for the 250 Hz sampling rate. # input_window_samples = 1000 ###################################################################### # Now we create the model. To enable it to be used in cropped decoding # efficiently, we manually set the length of the final convolution layer # to some length that makes the receptive field of the ConvNet smaller # than ``input_window_samples`` (see ``final_conv_length=30`` in the model # definition). # import torch from braindecode.util import set_random_seeds from braindecode.models import ShallowFBCSPNet, Deep4Net cuda = torch.cuda.is_available( ) # check if GPU is available, if True chooses to use it device = 'cuda:1' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = 20190706 # random seed to make results reproducible # Set random seed to be able to reproduce results set_random_seeds(seed=seed, cuda=cuda) n_classes = 4 # Extract number of chans from dataset n_chans = dataset[0][0].shape[0] # model = Deep4Net( # n_chans, # n_classes, # input_window_samples=input_window_samples, # final_conv_length="auto", # ) # # # # embedding_net = Deep4Net_origin(4, 22, input_window_samples) # model = FcClfNet(embedding_net) model = ShallowFBCSPNet( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length=30, ) print(model) # Send model to GPU if cuda: model.cuda(device) ###################################################################### # And now we transform model with strides to a model that outputs dense # prediction, so we can use it to obtain predictions for all # crops. # from braindecode.models.util import to_dense_prediction_model, get_output_shape to_dense_prediction_model(model) n_preds_per_input = get_output_shape(model, n_chans, input_window_samples)[2] print("n_preds_per_input : ", n_preds_per_input) print(model) ###################################################################### # Cut the data into windows # ------------------------- # ###################################################################### # In contrast to trialwise decoding, we have to supply an explicit window size and window stride to the # ``create_windows_from_events`` function. # import numpy as np from braindecode.datautil.windowers import create_windows_from_events trial_start_offset_seconds = -0.5 # Extract sampling frequency, check that they are same in all datasets sfreq = dataset.datasets[0].raw.info['sfreq'] assert all([ds.raw.info['sfreq'] == sfreq for ds in dataset.datasets]) # Calculate the trial start offset in samples. trial_start_offset_samples = int(trial_start_offset_seconds * sfreq) # Create windows using braindecode function for this. It needs parameters to define how # trials should be used. windows_dataset = create_windows_from_events( dataset, trial_start_offset_samples=trial_start_offset_samples, trial_stop_offset_samples=0, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input, drop_last_window=False, preload=True, ) ###################################################################### # Split the dataset # ----------------- # # This code is the same as in trialwise decoding. # from braindecode.datasets.base import BaseConcatDataset splitted = windows_dataset.split('session') train_set = splitted['session_T'] valid_set = splitted['session_E'] lr = 0.0625 * 0.01 weight_decay = 0 batch_size = 8 n_epochs = 100 train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) # valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False) test_loader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False) # Send model to GPU if cuda: model.cuda(device) from torch.optim import lr_scheduler import torch.optim as optim import argparse parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=50, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=50, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') args = parser.parse_args() args.gpuidx = 0 args.seed = 0 args.use_tensorboard = False args.save_model = False optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay) # scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=10, T_mult=1) scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=n_epochs - 1) import pandas as pd results_columns = ['test_loss', 'test_accuracy'] df = pd.DataFrame(columns=results_columns) for epochidx in range(1, n_epochs): print(epochidx) train_crop(10, model, device, train_loader, optimizer, scheduler, cuda, args.gpuidx) test_loss, test_score = eval_crop(model, device, test_loader) results = {'test_loss': test_loss, 'test_accuracy': test_score} df = df.append(results, ignore_index=True) print(results) return df
###################################################################### # We can now create the deep learning model. In this tutorial, we use the sleep # staging architecture introduced in [1]_, which is a four-layer convolutional # neural network. # import torch from braindecode.util import set_random_seeds from braindecode.models import SleepStagerChambon2018 cuda = torch.cuda.is_available() # check if GPU is available device = 'cuda' if torch.cuda.is_available() else 'cpu' if cuda: torch.backends.cudnn.benchmark = True # Set random seed to be able to reproduce results set_random_seeds(seed=87, cuda=cuda) n_classes = 5 # Extract number of channels and time steps from dataset n_channels = train_set[0][0].shape[0] input_size_samples = train_set[0][0].shape[1] model = SleepStagerChambon2018(n_channels, sfreq, n_classes=n_classes, input_size_s=input_size_samples / sfreq) # Send model to GPU if cuda: model.cuda()
def test_cropped_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=True) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel='auto', verbose='WARNING') for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude='bads') # Extract trials, only using EEG channels epoched = mne.Epochs(raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 train_set = SignalAndTarget(X[:60], y=y[:60]) test_set = SignalAndTarget(X[60:], y=y[60:]) # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12).create_network() to_dense_prediction_model(model) if cuda: model.cuda() optimizer = optim.Adam(model.parameters()) # determine output size test_input = np_to_var( np.ones((2, in_chans, input_time_length, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] print("{:d} predictions per input/trial".format(n_preds_per_input)) iterator = CropsFromTrialsIterator(batch_size=32, input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) losses = [] accuracies = [] for i_epoch in range(4): # Set model to training mode model.train() for batch_X, batch_y in iterator.get_batches(train_set, shuffle=False): net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() # Remove gradients of last backward pass from all parameters optimizer.zero_grad() outputs = model(net_in) # Mean predictions across trial # Note that this will give identical gradients to computing # a per-prediction loss (at least for the combination of log softmax activation # and negative log likelihood loss which we are using here) outputs = th.mean(outputs, dim=2, keepdim=False) loss = F.nll_loss(outputs, net_target) loss.backward() optimizer.step() # Print some statistics each epoch model.eval() print("Epoch {:d}".format(i_epoch)) for setname, dataset in (('Train', train_set), ('Test', test_set)): # Collect all predictions and losses all_preds = [] all_losses = [] batch_sizes = [] for batch_X, batch_y in iterator.get_batches(dataset, shuffle=False): net_in = np_to_var(batch_X) if cuda: net_in = net_in.cuda() net_target = np_to_var(batch_y) if cuda: net_target = net_target.cuda() outputs = model(net_in) all_preds.append(var_to_np(outputs)) outputs = th.mean(outputs, dim=2, keepdim=False) loss = F.nll_loss(outputs, net_target) loss = float(var_to_np(loss)) all_losses.append(loss) batch_sizes.append(len(batch_X)) # Compute mean per-input loss loss = np.mean( np.array(all_losses) * np.array(batch_sizes) / np.mean(batch_sizes)) print("{:6s} Loss: {:.5f}".format(setname, loss)) losses.append(loss) # Assign the predictions to the trials preds_per_trial = compute_preds_per_trial_from_crops( all_preds, input_time_length, dataset.X) # preds per trial are now trials x classes x timesteps/predictions # Now mean across timesteps for each trial to get per-trial predictions meaned_preds_per_trial = np.array( [np.mean(p, axis=1) for p in preds_per_trial]) predicted_labels = np.argmax(meaned_preds_per_trial, axis=1) accuracy = np.mean(predicted_labels == dataset.y) accuracies.append(accuracy * 100) print("{:6s} Accuracy: {:.1f}%".format(setname, accuracy * 100)) np.testing.assert_allclose(np.array(losses), np.array([ 1.31657708, 1.73548156, 1.02950428, 1.43932164, 0.78677772, 1.12382019, 0.55920881, 0.87277424 ]), rtol=1e-4, atol=1e-5) np.testing.assert_allclose(np.array(accuracies), np.array([ 50., 46.66666667, 50., 46.66666667, 50., 46.66666667, 66.66666667, 50. ]), rtol=1e-4, atol=1e-5)
def test_variable_length_trials_cropped_decoding(): cuda = False set_random_seeds(seed=20210726, cuda=cuda) # create fake tuh abnormal dataset tuh = _TUHAbnormalMock(path='') # fake variable length trials by cropping first recording splits = tuh.split([[i] for i in range(len(tuh.datasets))]) preprocess( concat_ds=splits['0'], preprocessors=[ Preprocessor('crop', tmax=300), ], ) variable_tuh = BaseConcatDataset( [splits[str(i)] for i in range(len(tuh.datasets))]) # make sure we actually have different length trials assert any(np.diff([ds.raw.n_times for ds in variable_tuh.datasets]) != 0) # create windows variable_tuh_windows = create_fixed_length_windows( concat_ds=variable_tuh, window_size_samples=1000, window_stride_samples=1000, drop_last_window=False, mapping={ True: 1, False: 0 }, ) # create train and valid set splits = variable_tuh_windows.split( [[i] for i in range(len(variable_tuh_windows.datasets))]) variable_tuh_windows_train = BaseConcatDataset( [splits[str(i)] for i in range(len(tuh.datasets) - 1)]) variable_tuh_windows_valid = BaseConcatDataset( [splits[str(len(tuh.datasets) - 1)]]) for x, y, ind in variable_tuh_windows_train: break train_split = predefined_split(variable_tuh_windows_valid) # initialize a model model = ShallowFBCSPNet( in_chans=x.shape[0], n_classes=len(tuh.description.pathological.unique()), ) to_dense_prediction_model(model) if cuda: model.cuda() # create and train a classifier clf = EEGClassifier( model, cropped=True, criterion=CroppedLoss, criterion__loss_function=torch.nn.functional.nll_loss, optimizer=torch.optim.Adam, batch_size=32, callbacks=['accuracy'], train_split=train_split, ) clf.fit(variable_tuh_windows_train, y=None, epochs=3) # make sure it does what we expect np.testing.assert_allclose( clf.history[:, 'train_loss'], np.array([ 0.689495325088501, 0.1353449523448944, 0.006638816092163324, ]), rtol=1e-1, atol=1e-1, ) np.testing.assert_allclose( clf.history[:, 'valid_loss'], np.array([ 2.925871, 3.611423, 4.23494, ]), rtol=1e-1, atol=1e-1, )
############################################################################## # Create the model # ---------------- # Braindecode comes with some predefined convolutional neural network # architectures for raw time-domain EEG. Here, we use the shallow ConvNet # model from [Deep learning with convolutional neural networks for EEG # decoding and visualization](https://arxiv.org/abs/1703.05051). from braindecode.models import ShallowFBCSPNet from braindecode.util import set_random_seeds # XXX : move to braindecode.util # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set.X.shape[1] # final_conv_length = auto ensures we only get a single output in the time dimension model = ShallowFBCSPNet(in_chans=in_chans, n_classes=n_classes, input_time_length=train_set.X.shape[2], final_conv_length='auto') if cuda: model.cuda() # We use [AdamW](https://arxiv.org/abs/1711.05101) to optimize the parameters of our network together with [Cosine Annealing](https://arxiv.org/abs/1608.03983) of the learning rate. We supply some default parameters that we have found to work well for motor decoding, however we strongly encourage you to perform your own hyperparameter optimization using cross validation on your training data. # <div class="alert alert-info"> #
from braindecode.losses import CroppedNLLLoss from braindecode.models.deep4 import Deep4Net from braindecode.models.shallow_fbcsp import ShallowFBCSPNet from braindecode.models.util import to_dense_prediction_model from braindecode.scoring import CroppedTrialEpochScoring from braindecode.util import set_random_seeds model_name = "shallow" # 'shallow' or 'deep' cuda = torch.cuda.is_available() # cuda = False if cuda: device = "cuda" else: device = "cpu" set_random_seeds(seed=20190706, cuda=cuda) input_time_length = 1000 n_classes = 4 n_chans = 26 # TODO: should be 22 of course if model_name == "shallow": model = ShallowFBCSPNet( n_chans, n_classes, input_time_length=input_time_length, final_conv_length=30, ) elif model_name == "deep": model = Deep4Net( n_chans, n_classes,
# Printing the arguments print('\n\n\n>>> CNN BCI decoding <<<') print('\nInput parameters:') for key, val in vars(args).items(): print('{:16} {}'.format(key, val)) # ============================================================================= # GPU/CPU and random seed # ============================================================================= # Check for GPU and set random seed to make results reproducible cuda = torch.cuda.is_available() args.device = 'cuda' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True set_random_seeds(seed=args.seed, cuda=cuda) # ============================================================================= # Loading and preprocessing the data # ============================================================================= # For intra-subject decoding, 10 trials per condition are used for validation, # 10 trials for testing, and the remaining trials are used for training. # For inter-subject decoding 75 trials per condition of the subject of interest # are used for validation and 75 for testing. All the data from the other # subjects is used for training. dataset = load_5f_halt(args) # Getting EEG data info args.n_classes = len(np.unique(dataset.datasets[0].raw.annotations.description)) args.l_freq = dataset.datasets[0].raw.info['highpass']
def test_eeg_classifier(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_window_samples = 450 n_classes = 2 in_chans = X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_window_samples=input_window_samples, final_conv_length=12, ) to_dense_prediction_model(model) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_window_samples, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] train_set = create_from_X_y(X[:48], y[:48], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) valid_set = create_from_X_y(X[48:60], y[48:60], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) cropped_cb_train = CroppedTrialEpochScoring( "accuracy", name="train_trial_accuracy", lower_is_better=False, on_train=True, ) cropped_cb_valid = CroppedTrialEpochScoring( "accuracy", on_train=False, name="valid_trial_accuracy", lower_is_better=False, ) clf = EEGClassifier( model, cropped=True, criterion=CroppedLoss, criterion__loss_function=nll_loss, optimizer=optim.Adam, train_split=predefined_split(valid_set), batch_size=32, callbacks=[ ("train_trial_accuracy", cropped_cb_train), ("valid_trial_accuracy", cropped_cb_valid), ], ) clf.fit(train_set, y=None, epochs=4) expected = [{ 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.6639312505722046 }, { 'train_batch_size': 32, 'train_loss': 2.6161606311798096 }, { 'train_batch_size': 32, 'train_loss': 1.627132773399353 }, { 'valid_batch_size': 24, 'valid_loss': 0.9677614569664001 }], 'epoch': 1, 'train_batch_count': 3, 'train_loss': 1.9690748850504558, 'train_loss_best': True, 'train_trial_accuracy': 0.4791666666666667, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 0.9677614569664001, 'valid_loss_best': True, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': True }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.3829222917556763 }, { 'train_batch_size': 32, 'train_loss': 1.3123714923858643 }, { 'train_batch_size': 32, 'train_loss': 1.0109959840774536 }, { 'valid_batch_size': 24, 'valid_loss': 1.9435862302780151 }], 'epoch': 2, 'train_batch_count': 3, 'train_loss': 1.2354299227396648, 'train_loss_best': True, 'train_trial_accuracy': 0.5, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 1.9435862302780151, 'valid_loss_best': False, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': False }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 1.172208547592163 }, { 'train_batch_size': 32, 'train_loss': 0.8899562954902649 }, { 'train_batch_size': 32, 'train_loss': 1.0232216119766235 }, { 'valid_batch_size': 24, 'valid_loss': 0.9585554599761963 }], 'epoch': 3, 'train_batch_count': 3, 'train_loss': 1.0284621516863506, 'train_loss_best': True, 'train_trial_accuracy': 0.5, 'train_trial_accuracy_best': False, 'valid_batch_count': 1, 'valid_loss': 0.9585554599761963, 'valid_loss_best': True, 'valid_trial_accuracy': 0.5, 'valid_trial_accuracy_best': False }, { 'batches': [{ 'train_batch_size': 32, 'train_loss': 0.9693693518638611 }, { 'train_batch_size': 32, 'train_loss': 0.900641918182373 }, { 'train_batch_size': 32, 'train_loss': 0.8839665651321411 }, { 'valid_batch_size': 24, 'valid_loss': 0.873468816280365 }], 'epoch': 4, 'train_batch_count': 3, 'train_loss': 0.9179926117261251, 'train_loss_best': True, 'train_trial_accuracy': 0.625, 'train_trial_accuracy_best': True, 'valid_batch_count': 1, 'valid_loss': 0.873468816280365, 'valid_loss_best': True, 'valid_trial_accuracy': 0.4166666666666667, 'valid_trial_accuracy_best': False }] history_without_dur = [{k: v for k, v in h.items() if k != "dur"} for h in clf.history] assert_deep_allclose(expected, history_without_dur, atol=1e-3, rtol=1e-3)
def run_exp(first_n, lr, weight_decay, cross_ent_weight, batch_size, np_th_seed, debug, n_epochs, n_mixes, output_dir, scale_2_cross_ent, mask_for_cross_ent, nll_weight, linear_classifier, flow_gmm, flow_coupling): hparams = {k: v for k, v in locals().items() if v is not None} noise_factor = 1 / 256.0 if debug: first_n = 512 batch_size = 10 n_epochs = 5 set_random_seeds(np_th_seed, True) writer = SummaryWriter(output_dir) writer.add_hparams(hparams, metric_dict={}, name=output_dir) writer.flush() model = create_glow_model(hidden_channels=512, K=32, L=3, flow_permutation='invconv', flow_coupling=flow_coupling, LU_decomposed=True, n_chans=3, block_type='conv', use_act_norm=True) if flow_coupling == 'additive': state_dict = th.load( '/home/schirrmr/data/exps/invertible/additive/7/state_dicts_model_250.pth' ) else: assert flow_coupling == 'affine' state_dict = th.load( '/home/schirrmr/data/exps/invertible/finetune//12/state_dicts_model_76.pth' ) for key in state_dict.keys(): if 'loc' in key or 'log_scale' in key: state_dict[key].squeeze_() model.load_state_dict(state_dict) del state_dict pre_dist_model = convert_glow_to_pre_dist_model(model, as_list=True) del model if flow_gmm: dist0 = NClassIndependentDist(10, n_dims=3072 // 2, optimize_mean=False, optimize_std=False) dist1 = NClassIndependentDist(10, n_dims=3072 // 4, optimize_mean=False, optimize_std=False) dist2 = NClassIndependentDist(10, n_dims=3072 // 4, optimize_mean=False, optimize_std=False) dist0.class_means.normal_(mean=0, std=1) dist1.class_means.normal_(mean=0, std=1) dist2.class_means.normal_(mean=0, std=1) else: init_dist_std = 1e-1 dist0 = PerDimWeightedMix(10, n_mixes=n_mixes, n_dims=3072 // 2, optimize_mean=True, optimize_std=True, init_std=init_dist_std) dist1 = PerDimWeightedMix(10, n_mixes=n_mixes, n_dims=3072 // 4, optimize_mean=True, optimize_std=True, init_std=init_dist_std) dist2 = PerDimWeightedMix(10, n_mixes=n_mixes, n_dims=3072 // 4, optimize_mean=True, optimize_std=True, init_std=init_dist_std) model = Node(pre_dist_model, ApplyToList(dist0, dist1, dist2)) net = model.cuda() init_all_modules(net, None) if mask_for_cross_ent: alphas_mask = th.zeros(768, requires_grad=True, device='cuda') if linear_classifier: clf = th.nn.Linear(768, 10).cuda() train_loader, valid_loader = load_train_test( 'cifar10', shuffle_train=True, drop_last_train=True, batch_size=batch_size, eval_batch_size=256, n_workers=8, first_n=first_n, augment=True, exclude_cifar_from_tiny=False, ) optim = th.optim.Adam(net.parameters(), lr=lr, weight_decay=weight_decay) if mask_for_cross_ent: optim.add_param_group( dict(params=[alphas_mask], lr=5e-2, weight_decay=0)) if linear_classifier: optim.add_param_group( dict(params=clf.parameters(), lr=lr, weight_decay=weight_decay)) def get_lp_for_cross_ent(z, lp, net, scale_2_cross_ent, mask_for_cross_ent): dists = list(net.module.module_list.children()) if linear_classifier: lp_for_cross_ent = clf(z[2]) else: if scale_2_cross_ent: lp_for_cross_ent = dists[2](z[2], fixed=dict(sum_dims=False))[1] if mask_for_cross_ent: mask = th.sigmoid(alphas_mask) lp_for_cross_ent = lp_for_cross_ent * mask.unsqueeze( 0).unsqueeze(0) lp_for_cross_ent = lp_for_cross_ent.sum(dim=-1) else: lp_for_cross_ent = lp return lp_for_cross_ent for i_epoch in range(n_epochs + 1): if i_epoch > 0: for X, y in train_loader: y = y.cuda() noise = th.rand_like(X) * 1 / 256.0 noised = X + noise z, lp = net(noised.cuda(), fixed=dict(y=None)) lp_for_cross_ent = get_lp_for_cross_ent( z, lp, net, scale_2_cross_ent, mask_for_cross_ent) cross_ent = th.nn.functional.cross_entropy( lp_for_cross_ent, y.argmax(dim=1), ) nll = -th.mean(th.sum(lp * y, dim=1)) loss = cross_ent_weight * cross_ent + nll_weight * nll optim.zero_grad() loss.backward() optim.step() optim.zero_grad() del y, noise, noised, lp, cross_ent, nll, loss print(i_epoch) results = {} with th.no_grad(): for name, loader in (('Train', train_loader), ('Valid', valid_loader)): all_lps = [] all_corrects = [] for X, y in loader: y = y.cuda() # First with noise to get nll for bpd, # then without noise for accâuracy noise = th.rand_like(X) * 1 / 256.0 noised = X + noise noise_log_prob = np.log(256) * np.prod(X.shape[1:]) z, lp = net(noised.cuda()) lps = to_numpy(th.sum(lp * y, dim=1) - noise_log_prob) all_lps.extend(lps) z, lp = net(X.cuda() + (1 / (2 * 256.0))) lp_for_cross_ent = get_lp_for_cross_ent( z, lp, net, scale_2_cross_ent, mask_for_cross_ent) corrects = to_numpy( y.argmax(dim=1) == lp_for_cross_ent.argmax(dim=1)) all_corrects.extend(corrects) acc = np.mean(all_corrects) nll = -(np.mean(all_lps) / (np.prod(X.shape[1:]) * np.log(2))) print(f"{name} NLL: {nll:.2f}") print(f"{name} Acc: {acc:.1%}") results[f"{name.lower()}_nll"] = nll results[f"{name.lower()}_acc"] = acc writer.add_scalar(f"{name.lower()}_nll", nll, i_epoch) writer.add_scalar(f"{name.lower()}_acc", acc * 100, i_epoch) del noise, noised, z, lp, lps writer.flush() sys.stdout.flush() if not debug: dict_path = os.path.join(output_dir, "model_dict.th") th.save(net.state_dict(), open(dict_path, 'wb')) if mask_for_cross_ent: mask_path = os.path.join(output_dir, "alphas_mask.th") th.save(alphas_mask, open(mask_path, 'wb')) model_path = os.path.join(output_dir, "model.th") th.save(net, open(model_path, 'wb')) return results
def test_post_epoch_train_scoring(): cuda = False set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 class EEGDataSet(Dataset): def __init__(self, X, y): self.X = X if self.X.ndim == 3: self.X = self.X[:, :, :, None] self.y = y def __len__(self): return len(self.X) def __getitem__(self, idx): return self.X[idx], self.y[idx] X, y = sklearn.datasets.make_classification( 40, (3 * 100), n_informative=3 * 50, n_classes=2 ) X = X.reshape(40, 3, 100).astype(np.float32) in_chans = X.shape[1] train_set = EEGDataSet(X, y) class TestCallback(Callback): def on_epoch_end(self, net, *args, **kwargs): preds = net.predict(train_set.X) y_true = train_set.y np.testing.assert_allclose( clf.history[-1]["train_f1"], f1_score(y_true, preds), rtol=1e-4, atol=1e-4, ) np.testing.assert_allclose( clf.history[-1]["train_acc"], accuracy_score(y_true, preds), rtol=1e-4, atol=1e-4, ) set_random_seeds(20200114, cuda) # final_conv_length = auto ensures # we only get a single output in the time dimension model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_window_samples=train_set.X.shape[2], pool_time_stride=1, pool_time_length=2, final_conv_length="auto", ) if cuda: model.cuda() clf = EEGClassifier( model, criterion=torch.nn.NLLLoss, optimizer=optim.AdamW, train_split=None, optimizer__lr=0.0625 * 0.01, optimizer__weight_decay=0, batch_size=64, callbacks=[ ( "train_accuracy", PostEpochTrainScoring( "accuracy", lower_is_better=False, name="train_acc" ), ), ( "train_f1_score", PostEpochTrainScoring( "f1", lower_is_better=False, name="train_f1" ), ), ("test_callback", TestCallback()), ], ) clf.fit(train_set, y=None, epochs=4)
def test_trialwise_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) raw.apply_function(lambda x: x * 1000000) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) ds = EpochsDataset(epoched) train_set = Subset(ds, np.arange(60)) valid_set = Subset(ds, np.arange(60, len(ds))) train_valid_split = predefined_split(valid_set) cuda = False if cuda: device = 'cuda' else: device = 'cpu' set_random_seeds(seed=20170629, cuda=cuda) n_classes = 2 in_chans = train_set[0][0].shape[0] input_time_length = train_set[0][0].shape[1] model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length="auto", ) if cuda: model.cuda() clf = EEGClassifier( model, cropped=False, criterion=torch.nn.NLLLoss, optimizer=torch.optim.Adam, train_split=train_valid_split, optimizer__lr=0.001, batch_size=30, callbacks=["accuracy"], device=device, ) clf.fit(train_set, y=None, epochs=6) np.testing.assert_allclose( clf.history[:, 'train_loss'], np.array([ 1.1114974617958069, 1.0976492166519165, 0.668171226978302, 0.5880511999130249, 0.7054798305034637, 0.5272344648838043 ]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_loss'], np.array([ 0.8467752933502197, 0.9804958701133728, 0.9134824872016907, 0.8305345773696899, 0.8263336420059204, 0.8535978198051453 ]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'train_accuracy'], np.array([ 0.7166666666666667, 0.6666666666666666, 0.85, 0.9333333333333333, 0.9166666666666666, 0.9 ]), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_accuracy'], np.array([ 0.6, 0.5666666666666667, 0.5333333333333333, 0.5333333333333333, 0.6, 0.6666666666666666 ]), rtol=1e-4, atol=1e-5, )
def test_cropped_decoding(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data( subject_id, event_codes, update_path=False ) # Load each of the files parts = [ mne.io.read_raw_edf( path, preload=True, stim_channel="auto", verbose="WARNING" ) for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types( raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads" ) # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_time_length = 450 n_classes = 2 in_chans = X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_time_length=input_time_length, final_conv_length=12, ) to_dense_prediction_model(model) if cuda: model.cuda() # Perform forward pass to determine how many outputs per input n_preds_per_input = get_output_shape(model, in_chans, input_time_length)[2] train_set = CroppedXyDataset(X[:60], y[:60], input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) valid_set = CroppedXyDataset(X[60:], y=y[60:], input_time_length=input_time_length, n_preds_per_input=n_preds_per_input) train_split = predefined_split(valid_set) clf = EEGClassifier( model, cropped=True, criterion=CroppedLoss, criterion__loss_function=torch.nn.functional.nll_loss, optimizer=optim.Adam, train_split=train_split, batch_size=32, callbacks=['accuracy'], ) clf.fit(train_set, y=None, epochs=4) np.testing.assert_allclose( clf.history[:, 'train_loss'], np.array( [ 1.455306, 1.455934, 1.210563, 1.065806 ] ), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_loss'], np.array( [ 2.547288, 1.51785, 1.394036, 1.064355 ] ), rtol=1e-4, atol=1e-4, ) np.testing.assert_allclose( clf.history[:, 'train_accuracy'], np.array( [ 0.5, 0.5, 0.5, 0.533333 ] ), rtol=1e-4, atol=1e-5, ) np.testing.assert_allclose( clf.history[:, 'valid_accuracy'], np.array( [ 0.533333, 0.466667, 0.533333, 0.5 ] ), rtol=1e-4, atol=1e-5, )
def test_set_random_seeds_raise_value_error(): with pytest.raises(ValueError, match="cudnn_benchmark expected to be bool or None, got 'abc'"): set_random_seeds(100, True, "abc")
def train(subject_id): print('\n--------------------------------------------------\n') print( 'Training on BCI_IV_2a dataset | Cross-subject | ID: {:02d}\n'.format( subject_id)) ##### subject_range = [subject_id] subject_range = [x for x in range(1, 10)] dataset = MOABBDataset(dataset_name="BNCI2014001", subject_ids=subject_range) ###################################################################### # Preprocessing low_cut_hz = 4. # low cut frequency for filtering high_cut_hz = 38. # high cut frequency for filtering # Parameters for exponential moving standardization factor_new = 1e-3 init_block_size = 1000 preprocessors = [ Preprocessor('pick_types', eeg=True, eog=False, meg=False, stim=False), # Keep EEG sensors Preprocessor(lambda x: x * 1e6), # Convert from V to uV Preprocessor('filter', l_freq=low_cut_hz, h_freq=high_cut_hz), # Bandpass filter #Preprocessor('set_eeg_reference', ref_channels='average', ch_type='eeg'), Preprocessor('resample', sfreq=125), Preprocessor(covariance_align), ## Preprocessor(exponential_moving_standardize, # Exponential moving standardization ## factor_new=factor_new, init_block_size=init_block_size) ## Preprocessor('pick_channels', ch_names=short_ch_names, ordered=True), ] # Transform the data print('Preprocessing dataset\n') preprocess(dataset, preprocessors) ###################################################################### # Cut Compute Windows # ~~~~~~~~~~~~~~~~~~~ trial_start_offset_seconds = -0.5 trial_stop_offset_seconds = 0.0 # Extract sampling frequency, check that they are same in all datasets sfreq = dataset.datasets[0].raw.info['sfreq'] assert all([ds.raw.info['sfreq'] == sfreq for ds in dataset.datasets]) # Calculate the trial start offset in samples. trial_start_offset_samples = int(trial_start_offset_seconds * sfreq) trial_stop_offset_samples = int(trial_stop_offset_seconds * sfreq) # Create windows using braindecode function for this. It needs parameters to define how # trials should be used. print('Windowing dataset\n') windows_dataset = create_windows_from_events( dataset, # picks=["Fz", "FC3", "FC1", "FCz", "FC2", "FC4", "C5", "C3", "C1", "Cz", "C2", "C4", "C6", "CP3", "CP1", "CPz", "CP2", "CP4", "P1", "Pz", "P2", "POz"], trial_start_offset_samples=trial_start_offset_samples, trial_stop_offset_samples=trial_stop_offset_samples, preload=True, ) print('Computing covariances of each WindowsDataset') windows_dataset.compute_covariances_concat() # print(windows_dataset.datasets[0].windows) ###################################################################### # Merge multiple datasets into a single WindowDataset # metadata_all = [ds.windows.metadata for ds in windows_dataset.datasets] # metadata_full = pd.concat(metadata_all) """ epochs_all = [ds.windows for ds in windows_dataset.datasets] epochs_full = mne.concatenate_epochs(epochs_all) full_dataset = WindowsDataset(windows=epochs_full, description=None, transform=None) windows_dataset = full_dataset """ ###################################################################### # Split dataset into train and valid # keep only session 1: # temp = windows_dataset.split( 'session' ) # windows_dataset = temp['session_T'] # print(windows_dataset.datasets[0].windows) # print(windows_dataset.datasets[0].windows.get_data().shape) # quit() subject_column = windows_dataset.description['subject'].values inds_train = list(np.where(subject_column != subject_id)[0]) inds_valid = list(np.where(subject_column == subject_id)[0]) splitted = windows_dataset.split([inds_train, inds_valid]) train_set = splitted['0'] valid_set = splitted['1'] ####### epochs_all = [ds.windows for ds in train_set.datasets] epochs_full = mne.concatenate_epochs(epochs_all) trialwise_weights_all = [ds.trialwise_weights for ds in train_set.datasets] trialwise_weights_full = np.hstack(trialwise_weights_all) full_dataset = WindowsDataset(windows=epochs_full, description=None, transform=None) full_dataset.trialwise_weights = trialwise_weights_full train_set = full_dataset # print(train_set.windows.metadata) ###################################################################### # Create model cuda = torch.cuda.is_available( ) # check if GPU is available, if True chooses to use it device = 'cuda' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = 20200220 # random seed to make results reproducible # Set random seed to be able to reproduce results set_random_seeds(seed=seed, cuda=cuda) n_classes = 4 # Extract number of chans and time steps from dataset n_chans = train_set[0][0].shape[0] input_window_samples = train_set[0][0].shape[1] """ model = ShallowFBCSPNet( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length='auto') """ """ model = EEGNetv1( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length="auto", pool_mode="mean", second_kernel_size=(2, 32), third_kernel_size=(8, 4), drop_prob=0.25) """ """ model = HybridNet(n_chans, n_classes, input_window_samples=input_window_samples) """ """ model = TCN(n_chans, n_classes, n_blocks=6, n_filters=32, kernel_size=9, drop_prob=0.0, add_log_softmax=True) """ model = EEGNetv4( n_chans, n_classes, input_window_samples=input_window_samples, final_conv_length="auto", pool_mode="mean", F1=8, D=2, F2=16, # usually set to F1*D (?) kernel_length=64, third_kernel_size=(8, 4), drop_prob=0.2) if cuda: model.cuda() ###################################################################### # Training # These values we found good for shallow network: lr = 0.01 # 0.0625 * 0.01 weight_decay = 0.0005 # For deep4 they should be: # lr = 1 * 0.01 # weight_decay = 0.5 * 0.001 batch_size = 64 n_epochs = 100 # clf = EEGClassifier( clf = EEGClassifier_weighted( model, criterion=torch.nn.NLLLoss, optimizer=torch.optim.SGD, #AdamW, train_split=predefined_split( valid_set), # using valid_set for validation optimizer__lr=lr, optimizer__momentum=0.9, optimizer__weight_decay=weight_decay, batch_size=batch_size, callbacks=[ "accuracy", #("lr_scheduler", LRScheduler('CosineAnnealingLR', T_max=n_epochs - 1)), ], device=device, ) # Model training for a specified number of epochs. `y` is None as it is already supplied # in the dataset. clf.fit(train_set, y=None, epochs=n_epochs) results_columns = [ 'train_loss', 'valid_loss', 'train_accuracy', 'valid_accuracy' ] df = pd.DataFrame(clf.history[:, results_columns], columns=results_columns, index=clf.history[:, 'epoch']) val_accs = df['valid_accuracy'].values max_val_acc = 100.0 * np.max(val_accs) return max_val_acc
def exp(subject_id): cuda = torch.cuda.is_available( ) # check if GPU is available, if True chooses to use it device = 'cuda:1' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True seed = 10 # random seed to make results reproducible # Set random seed to be able to reproduce results set_random_seeds(seed=seed, cuda=cuda) test_subj = np.r_[subject_id] print('test subj:' + str(test_subj)) train_subj = np.setdiff1d(np.r_[1:10], test_subj) tr = [] val = [] #10%씩 떼어내서 val만듬 for ids in train_subj: train_size = int(0.9 * len(splitted[ids])) test_size = len(splitted[ids]) - train_size tr_i, val_i = torch.utils.data.random_split(splitted[ids], [train_size, test_size]) tr.append(tr_i) val.append(val_i) train_set = torch.utils.data.ConcatDataset(tr) valid_set = torch.utils.data.ConcatDataset(val) test_set = BaseConcatDataset([splitted[ids] for ids in test_subj]) # model = Deep4Net( # n_chans, # n_classes, # input_window_samples=input_window_samples, # final_conv_length="auto", # ) crop_size = 1125 embedding_net = EEGNet_v2_old(n_classes, n_chans, crop_size) model = FcClfNet(embedding_net) print(model) epochs = 100 batch_size = 64 train_loader = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True) valid_loader = torch.utils.data.DataLoader(valid_set, batch_size=batch_size, shuffle=False) test_loader = torch.utils.data.DataLoader(test_set, batch_size=batch_size, shuffle=False) # Send model to GPU if cuda: model.cuda(device=device) from torch.optim import lr_scheduler import torch.optim as optim import argparse parser = argparse.ArgumentParser( description='cross subject domain adaptation') parser.add_argument('--batch-size', type=int, default=50, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=50, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--epochs', type=int, default=100, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.001, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--save-model', action='store_true', default=True, help='For Saving the current Model') args = parser.parse_args() args.gpuidx = 1 args.seed = 0 args.use_tensorboard = False args.save_model = False optimizer = optim.AdamW(model.parameters(), lr=0.001, weight_decay=0.5 * 0.001) # scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=200) scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=50) # # #test lr # lr = [] # for i in range(200): # scheduler.step() # lr.append(scheduler.get_lr()) # # import matplotlib.pyplot as plt # plt.plot(lr) import pandas as pd results_columns = [ 'val_loss', 'test_loss', 'val_accuracy', 'test_accuracy' ] df = pd.DataFrame(columns=results_columns) for epochidx in range(1, epochs): print(epochidx) train(10, model, device, train_loader, optimizer, scheduler, cuda, device) val_loss, val_score = eval(model, device, valid_loader) test_loss, test_score = eval(model, device, test_loader) results = { 'val_loss': val_loss, 'test_loss': test_loss, 'val_accuracy': val_score, 'test_accuracy': test_score } df = df.append(results, ignore_index=True) print(results) return df
# neural network. We use the time distributed version of the model, where the # feature vectors of a sequence of windows are concatenated and passed to a # linear layer for classification. # import torch from torch import nn from braindecode.util import set_random_seeds from braindecode.models import SleepStagerEldele2021, TimeDistributed cuda = torch.cuda.is_available() # check if GPU is available device = 'cuda' if torch.cuda.is_available() else 'cpu' if cuda: torch.backends.cudnn.benchmark = True # Set random seed to be able to reproduce results set_random_seeds(seed=31, cuda=cuda) n_classes = 5 # Extract number of channels and time steps from dataset n_channels, input_size_samples = train_set[0][0].shape feat_extractor = SleepStagerEldele2021(sfreq, n_classes=n_classes, input_size_s=input_size_samples / sfreq, return_feats=True) model = nn.Sequential( TimeDistributed(feat_extractor), # apply model on each 30-s window nn.Sequential( # apply linear layer on concatenated feature vectors nn.Flatten(start_dim=1), nn.Dropout(0.5), nn.Linear(feat_extractor.len_last_layer * n_windows, n_classes)))
# We can now create the deep learning model. In this tutorial, we use the sleep # staging architecture introduced in [1]_, which is a four-layer convolutional # neural network. # import torch from torch import nn from braindecode.util import set_random_seeds from braindecode.models import SleepStagerChambon2018 cuda = torch.cuda.is_available() # check if GPU is available device = 'cuda' if torch.cuda.is_available() else 'cpu' if cuda: torch.backends.cudnn.benchmark = True # Set random seed to be able to reproduce results set_random_seeds(seed=random_state, cuda=cuda) n_classes = 5 # Extract number of channels and time steps from dataset n_channels, input_size_samples = train_set[0][0].shape class TimeDistributedNet(nn.Module): """Extract features for multiple windows then concatenate & classify them. """ def __init__(self, feat_extractor, len_last_layer, n_windows, n_classes, dropout=0.25): super().__init__() self.feat_extractor = feat_extractor self.clf = nn.Sequential( nn.Dropout(dropout),
model_name = "shallow" # 'shallow' or 'deep' n_epochs = 3 seed = 20200220 input_window_samples = 7500 batch_size = 64 cuda = torch.cuda.is_available() device = 'cuda' if cuda else 'cpu' if cuda: torch.backends.cudnn.benchmark = True n_chans = 62 # set to how many targets you want to regress (age -> 1, [x, y, z] -> 3) n_classes = 5 set_random_seeds(seed=seed, cuda=cuda) # initialize a model, transform to dense and move to gpu if model_name == "shallow": model = ShallowFBCSPNet( in_chans=n_chans, n_classes=n_classes, input_window_samples=input_window_samples, n_filters_time=40, n_filters_spat=40, final_conv_length=35, ) optimizer_lr = 0.000625 optimizer_weight_decay = 0 elif model_name == "deep": model = Deep4Net(
def test_eeg_classifier(): # 5,6,7,10,13,14 are codes for executed and imagined hands/feet subject_id = 1 event_codes = [5, 6, 9, 10, 13, 14] # This will download the files if you don't have them yet, # and then return the paths to the files. physionet_paths = mne.datasets.eegbci.load_data(subject_id, event_codes, update_path=False) # Load each of the files parts = [ mne.io.read_raw_edf(path, preload=True, stim_channel="auto", verbose="WARNING") for path in physionet_paths ] # Concatenate them raw = concatenate_raws(parts) # Find the events in this dataset events, _ = mne.events_from_annotations(raw) # Use only EEG channels eeg_channel_inds = mne.pick_types(raw.info, meg=False, eeg=True, stim=False, eog=False, exclude="bads") # Extract trials, only using EEG channels epoched = mne.Epochs( raw, events, dict(hands=2, feet=3), tmin=1, tmax=4.1, proj=False, picks=eeg_channel_inds, baseline=None, preload=True, ) # Convert data from volt to millivolt # Pytorch expects float32 for input and int64 for labels. X = (epoched.get_data() * 1e6).astype(np.float32) y = (epoched.events[:, 2] - 2).astype(np.int64) # 2,3 -> 0,1 # Set if you want to use GPU # You can also use torch.cuda.is_available() to determine if cuda is available on your machine. cuda = False set_random_seeds(seed=20170629, cuda=cuda) # This will determine how many crops are processed in parallel input_window_samples = 450 n_classes = 2 in_chans = X.shape[1] # final_conv_length determines the size of the receptive field of the ConvNet model = ShallowFBCSPNet( in_chans=in_chans, n_classes=n_classes, input_window_samples=input_window_samples, final_conv_length=12, ) to_dense_prediction_model(model) if cuda: model.cuda() # determine output size test_input = np_to_var( np.ones((2, in_chans, input_window_samples, 1), dtype=np.float32)) if cuda: test_input = test_input.cuda() out = model(test_input) n_preds_per_input = out.cpu().data.numpy().shape[2] train_set = create_from_X_y(X[:48], y[:48], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) valid_set = create_from_X_y(X[48:60], y[48:60], drop_last_window=False, window_size_samples=input_window_samples, window_stride_samples=n_preds_per_input) cropped_cb_train = CroppedTrialEpochScoring( "accuracy", name="train_trial_accuracy", lower_is_better=False, on_train=True, ) cropped_cb_valid = CroppedTrialEpochScoring( "accuracy", on_train=False, name="valid_trial_accuracy", lower_is_better=False, ) clf = EEGClassifier( model, criterion=CroppedLoss, criterion__loss_function=nll_loss, optimizer=optim.Adam, train_split=predefined_split(valid_set), batch_size=32, callbacks=[ ("train_trial_accuracy", cropped_cb_train), ("valid_trial_accuracy", cropped_cb_valid), ], ) clf.fit(train_set, y=None, epochs=4) expected = [ { "batches": [ { "train_loss": 1.9391239881515503, "train_batch_size": 32 }, { "train_loss": 2.895704507827759, "train_batch_size": 32 }, { "train_loss": 1.0713893175125122, "train_batch_size": 32 }, { "valid_loss": 1.1811838150024414, "valid_batch_size": 24 }, ], "epoch": 1, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.9687392711639404, "train_loss_best": True, "valid_loss": 1.1811838150024414, "valid_loss_best": True, "train_trial_accuracy": 0.4791666666666667, "train_trial_accuracy_best": True, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": True, }, { "batches": [ { "train_loss": 1.5488793849945068, "train_batch_size": 32 }, { "train_loss": 1.1174801588058472, "train_batch_size": 32 }, { "train_loss": 1.1525697708129883, "train_batch_size": 32 }, { "valid_loss": 2.202029228210449, "valid_batch_size": 24 }, ], "epoch": 2, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.2729764382044475, "train_loss_best": True, "valid_loss": 2.202029228210449, "valid_loss_best": False, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": True, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, { "batches": [ { "train_loss": 1.0049529075622559, "train_batch_size": 32 }, { "train_loss": 1.0266971588134766, "train_batch_size": 32 }, { "train_loss": 1.0799436569213867, "train_batch_size": 32 }, { "valid_loss": 1.0638500452041626, "valid_batch_size": 24 }, ], "epoch": 3, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 1.0371979077657063, "train_loss_best": True, "valid_loss": 1.0638500452041626, "valid_loss_best": True, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": False, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, { "batches": [ { "train_loss": 1.0052555799484253, "train_batch_size": 32 }, { "train_loss": 0.8479514718055725, "train_batch_size": 32 }, { "train_loss": 0.9589881300926208, "train_batch_size": 32 }, { "valid_loss": 0.8794112801551819, "valid_batch_size": 24 }, ], "epoch": 4, "train_batch_count": 3, "valid_batch_count": 1, "train_loss": 0.9373983939488729, "train_loss_best": True, "valid_loss": 0.8794112801551819, "valid_loss_best": True, "train_trial_accuracy": 0.5, "train_trial_accuracy_best": False, "valid_trial_accuracy": 0.5, "valid_trial_accuracy_best": False, }, ] history_without_dur = [{k: v for k, v in h.items() if k != "dur"} for h in clf.history] assert_deep_allclose(expected, history_without_dur, atol=1e-3, rtol=1e-3)