コード例 #1
0
def processing_data(data_folder, subject_id, low_cut_hz, high_cut_hz, factor_new, init_block_size, ival, valid_set_fraction):
    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(
        train_filepath, labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(
        test_filepath, labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    # Preprocessing
    train_cnt = train_cnt.drop_channels(['STI 014', 'EOG-left',
                                         'EOG-central', 'EOG-right'])
    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz,
                               train_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), train_cnt)
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T, factor_new=factor_new,
                                                  init_block_size=init_block_size,
                                                  eps=1e-4).T,
        train_cnt)

    test_cnt = test_cnt.drop_channels(['STI 014', 'EOG-left',
                                       'EOG-central', 'EOG-right'])
    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(a, low_cut_hz, high_cut_hz,
                               test_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T, factor_new=factor_new,
                                                  init_block_size=init_block_size,
                                                  eps=1e-4).T,
        test_cnt)

    marker_def = OrderedDict([('Left Hand', [1]), ('Right Hand', [2],),
                              ('Foot', [3]), ('Tongue', [4])])

    train_set = create_signal_target_from_raw_mne(
        train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(
        test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(
        train_set, first_set_fraction=1 - valid_set_fraction)

    return train_set, valid_set, test_set
コード例 #2
0
def preprocessing(data_folder, subject_id, low_cut_hz):
    global train_set, test_set, valid_set, n_classes, n_chans
    global n_iters, input_time_length
# def run_exp(data_folder, subject_id, low_cut_hz, model, cuda):
    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(
        train_filepath, labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(
        test_filepath, labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()
    train_cnt = train_cnt.drop_channels(['STI 014', 'EOG-left',
                                         'EOG-central', 'EOG-right'])
    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(a, low_cut_hz, 38, train_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), train_cnt)
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T, factor_new=1e-3,
                                                  init_block_size=1000,
                                                  eps=1e-4).T,
        train_cnt)

    test_cnt = test_cnt.drop_channels(['STI 014', 'EOG-left',
                                       'EOG-central', 'EOG-right'])
    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(a, low_cut_hz, 38, test_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T, factor_new=1e-3,
                                                  init_block_size=1000,
                                                  eps=1e-4).T,
        test_cnt)

    marker_def = OrderedDict([('Left Hand', [1]), ('Right Hand', [2],),
                              ('Foot', [3]), ('Tongue', [4])])
    ival = [-500, 4000]
    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(train_set,
                                               first_set_fraction=0.8)
    set_random_seeds(seed=20190706, cuda=cuda)
    n_classes = 4
    n_chans = int(train_set.X.shape[1])
    input_time_length=1000
コード例 #3
0
    def load_data(filenames, sensor_names, name_to_start_codes,
                  name_to_stop_codes, trial_ival, break_ival,
                  min_break_length_ms, max_break_length_ms, input_time_length,
                  filename_to_extra_args):
        all_sets = []
        original_args = locals()
        for filename in filenames:
            kwargs = deepcopy(original_args)
            if filename in filename_to_extra_args:
                kwargs.update(filename_to_extra_args[filename])
            log.info("Loading {:s}...".format(filename))
            cnt = BBCIDataset(filename, load_sensor_names=sensor_names).load()
            cnt = cnt.drop_channels(['STI 014'])
            log.info("Resampling...")
            cnt = resample_cnt(cnt, 100)
            log.info("Standardizing...")
            cnt = mne_apply(
                lambda a: exponential_running_standardize(
                    a.T, init_block_size=50).T, cnt)

            log.info("Transform to set...")
            full_set = (create_signal_target_with_breaks_from_mne(
                cnt,
                kwargs['name_to_start_codes'],
                kwargs['trial_ival'],
                kwargs['name_to_stop_codes'],
                kwargs['min_break_length_ms'],
                kwargs['max_break_length_ms'],
                kwargs['break_ival'],
                prepad_trials_to_n_samples=kwargs['input_time_length'],
            ))
            all_sets.append(full_set)
        return all_sets
コード例 #4
0
ファイル: dataloader.py プロジェクト: sameedhayat/InfoGAN
def get_data():
    import os
    os.sys.path.append('/home/schirrmr/braindecode/code/braindecode/')
    from braindecode.datautil.trial_segment import create_signal_target_from_raw_mne
    from braindecode.datasets.bbci import BBCIDataset
    from braindecode.mne_ext.signalproc import mne_apply, resample_cnt
    from braindecode.datautil.signalproc import exponential_running_standardize
    subject_id = 4  # 1-14
    loader = BBCIDataset(
        '/data/schirrmr/schirrmr/HGD-public/reduced/train/{:d}.mat'.format(
            subject_id),
        load_sensor_names=['C3'])
    cnt = loader.load()
    cnt = cnt.drop_channels(['STI 014'])
    from collections import OrderedDict
    marker_def = OrderedDict([('Right Hand', [1]), (
        'Left Hand',
        [2],
    ), ('Rest', [3]), ('Feet', [4])])
    # Here you can choose a larger sampling rate later
    # Right now chosen very small to allow fast initial experiments
    cnt = resample_cnt(cnt, new_fs=500)
    cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, cnt)
    ival = [0, 2000]  # ms to cut trial
    dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)
    return dataset.X, dataset.y
コード例 #5
0
 def load_data(self, filename=None, params=None, create_frame_seg=50):
     self.dataframe = np.zeros((9, 288, 22, 1000))
     self.datalabel = np.zeros((9, 288))
     for i in range(1, 10, 1):
         AT_slice = h5py.File('dataset/A0' + str(i) + 'T_slice.mat', 'r')
         X = np.copy(AT_slice['image'])
         X = X[:, :22, :]  # select first 22 channels
         y = np.copy(AT_slice['type'])
         y = y[0, 0:X.shape[0]:1]
         y = np.asarray(y, dtype=np.int32)
         # replace NaN as 0
         X[np.isnan(X)] = 0
         self.datadict['A0' + str(i) + 'T'] = (X, y)
         self.dataframe[i - 1, :, :, :] = X
         self.datalabel[i - 1, :] = y
     # preprocessing using braincode
     for i in range(9):
         for j in range(228):
             self.dataframe[i, j] = np.transpose(
                 signalproc.highpass_cnt(np.transpose(self.dataframe[i, j]),
                                         4,
                                         1000,
                                         filt_order=3,
                                         axis=0))
             self.dataframe[i, j] = np.transpose(
                 signalproc.exponential_running_standardize(
                     np.transpose(self.dataframe[i, j]),
                     factor_new=0.001,
                     init_block_size=None,
                     eps=0.0001))
     print("Data filtered")
     if create_frame_seg:
         self.create_frame(create_frame_seg)
     print("Data fully loaded!")
コード例 #6
0
def crops_from_trial(X, y, crop_len, stride=0, time_last=True, dummy_idx=0, normalize=True):
    crop_len = int(crop_len)
    x_list, y_list = list(), list()
    if stride > 0:
        num_valid_crops = int((X.shape[0] - crop_len) / stride) + 1
    else:
        num_valid_crops = int(X.shape[0] // crop_len)

    for crop in range(num_valid_crops):
        if stride > 0:
            crop_idx = int(crop * stride)
        else:
            crop_idx = int(crop * crop_len)

        x_crop = X[crop_idx:crop_idx + crop_len, ]
        y_crop = y[crop_idx:crop_idx + crop_len, ]
        if normalize:
            y_crop = MinMaxScaler(feature_range=(-1, 1)).fit_transform(y_crop.reshape(-1, 1)).squeeze()
            x_crop = exponential_running_standardize(x_crop, init_block_size=250, factor_new=0.001, eps=1e-4)

        x_list.append(
            np.expand_dims(x_crop.T if time_last else x_crop, axis=dummy_idx).astype(np.float32)
        )
        y_list.append(y_crop.astype(np.float32))
    return x_list, y_list
コード例 #7
0
def load_bbci_data(filename, low_cut_hz):
	load_sensor_names = None
	loader = BBCIDataset(filename, load_sensor_names=load_sensor_names)


	log.info("Loading data...")
	cnt = loader.load()

	# Cleaning: First find all trials that have absolute microvolt values
	# larger than +- 800 inside them and remember them for removal later
	log.info("Cutting trials...")

	marker_def = OrderedDict([('Right Hand', [1]), ('Left Hand', [2],),
							  ('Rest', [3]), ('Feet', [4])])
	clean_ival = [0, 4000]

	set_for_cleaning = create_signal_target_from_raw_mne(cnt, marker_def,
												  clean_ival)

	clean_trial_mask = np.max(np.abs(set_for_cleaning.X), axis=(1, 2)) < 800

	log.info("Clean trials: {:3d}  of {:3d} ({:5.1f}%)".format(
		np.sum(clean_trial_mask),
		len(set_for_cleaning.X),
		np.mean(clean_trial_mask) * 100))

	# now pick only sensors with C in their name
	# as they cover motor cortex
	C_sensors = ['FC5', 'FC1', 'FC2', 'FC6', 'C3', 'C4', 'CP5',
				 'CP1', 'CP2', 'CP6', 'FC3', 'FCz', 'FC4', 'C5', 'C1', 'C2',
				 'C6',
				 'CP3', 'CPz', 'CP4', 'FFC5h', 'FFC3h', 'FFC4h', 'FFC6h',
				 'FCC5h',
				 'FCC3h', 'FCC4h', 'FCC6h', 'CCP5h', 'CCP3h', 'CCP4h', 'CCP6h',
				 'CPP5h',
				 'CPP3h', 'CPP4h', 'CPP6h', 'FFC1h', 'FFC2h', 'FCC1h', 'FCC2h',
				 'CCP1h',
				 'CCP2h', 'CPP1h', 'CPP2h']

	cnt = cnt.pick_channels(C_sensors)

	# Further preprocessings
	log.info("Resampling...")
	cnt = resample_cnt(cnt, 250.0)

	print("REREFERENCING")

	log.info("Highpassing...")
	cnt = mne_apply(lambda a: highpass_cnt(a, low_cut_hz, cnt.info['sfreq'], filt_order=3, axis=1),cnt)
	log.info("Standardizing...")
	cnt = mne_apply(lambda a: exponential_running_standardize(a.T, factor_new=1e-3,init_block_size=1000,eps=1e-4).T,cnt)

	# Trial interval, start at -500 already, since improved decoding for networks
	ival = [-500, 4000]

	dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)

	dataset.X = dataset.X[clean_trial_mask]
	dataset.y = dataset.y[clean_trial_mask]
	return dataset.X, dataset.y
コード例 #8
0
def test_exponential_running_standardize(mock_data):
    mock_input, expected_data, _ = mock_data
    standardized_data = exponential_running_standardize(mock_input)
    assert mock_input.shape == standardized_data.shape == expected_data.shape
    np.testing.assert_allclose(standardized_data,
                               expected_data,
                               rtol=1e-4,
                               atol=1e-4)
コード例 #9
0
    def _create_examples(self):
        name_to_code = OrderedDict([('Right', 1), ('Left', 2), ('Rest', 3),
                                    ('Feet', 4)])

        data_list_list = []
        for file_name in self.file_names:
            cnt = BBCIDataset(file_name,
                              load_sensor_names=self.load_sensor_names).load()
            cnt = cnt.drop_channels(['STI 014'])
            cnt = resample_cnt(cnt, self.sampling_freq)
            if self.normalization_type == 'exponential':
                cnt = mne_apply(
                    lambda a: exponential_running_standardize(
                        a.T, init_block_size=1000, factor_new=0.001, eps=1e-4).
                    T, cnt)

            data = create_signal_target_from_raw_mne(cnt, name_to_code,
                                                     self.segment_ival_ms)
            data_list = [(d, l) for d, l in zip(data.X, data.y)]
            data_list = self.cv_split(data_list)

            # Normalize the data
            if self.normalization_type == 'standard':
                running_statistics = RunningStatistics(
                    dim=data_list[0][0].shape[0], time_dimension_first=False)
                for data, label in data_list:
                    running_statistics.append(data)

                mean = running_statistics.mean_vector()
                sdev = np.clip(np.sqrt(running_statistics.var_vector()), 1e-5,
                               None)

                logger.info('Normalize with \n mean: %s, \n sdev: %s' %
                            (mean, sdev))
                for i in range(len(data_list)):
                    data_list[i] = ((data_list[i][0] - mean) / sdev,
                                    data_list[i][1])

            data_list_list.append(data_list)

        # Create examples for 4 classes
        for i, data_list in enumerate(data_list_list):
            for label in range(4):

                class_data_list = [
                    data for data in data_list if data[1] == label
                ]

                self.examples.append([
                    BBCIDataReaderMulti.ExampleInfo(
                        example_id=str((i, label, j)),
                        random_mode=self.random_mode,
                        offset_size=self.offset_size,
                        label=label,
                        data=data,
                        context=i)
                    for (j, (data, label)) in enumerate(class_data_list)
                ])
コード例 #10
0
def load_bbci_data(filename, low_cut_hz, debug=False):
    load_sensor_names = None
    if debug:
        load_sensor_names = ['C3', 'C4', 'C2']
    loader = BBCIDataset(filename, load_sensor_names=load_sensor_names)

    log.info("Loading data...")
    cnt = loader.load()

    log.info("Cutting trials...")

    marker_def = OrderedDict([('Right Hand', [1]), (
        'Left Hand',
        [2],
    ), ('Rest', [3]), ('Feet', [4])])
    clean_ival = [0, 4000]

    set_for_cleaning = create_signal_target_from_raw_mne(
        cnt, marker_def, clean_ival)

    clean_trial_mask = np.max(np.abs(set_for_cleaning.X), axis=(1, 2)) < 800

    log.info("Clean trials: {:3d}  of {:3d} ({:5.1f}%)".format(
        np.sum(clean_trial_mask), len(set_for_cleaning.X),
        np.mean(clean_trial_mask) * 100))

    # lets convert to millivolt for numerical stability of next operations
    C_sensors = [
        'FC5', 'FC1', 'FC2', 'FC6', 'C3', 'C4', 'CP5', 'CP1', 'CP2', 'CP6',
        'FC3', 'FCz', 'FC4', 'C5', 'C1', 'C2', 'C6', 'CP3', 'CPz', 'CP4',
        'FFC5h', 'FFC3h', 'FFC4h', 'FFC6h', 'FCC5h', 'FCC3h', 'FCC4h', 'FCC6h',
        'CCP5h', 'CCP3h', 'CCP4h', 'CCP6h', 'CPP5h', 'CPP3h', 'CPP4h', 'CPP6h',
        'FFC1h', 'FFC2h', 'FCC1h', 'FCC2h', 'CCP1h', 'CCP2h', 'CPP1h', 'CPP2h'
    ]
    if debug:
        C_sensors = load_sensor_names
    cnt = cnt.pick_channels(C_sensors)
    cnt = mne_apply(lambda a: a * 1e6, cnt)
    log.info("Resampling...")
    cnt = resample_cnt(cnt, 250.0)
    log.info("Highpassing...")
    cnt = mne_apply(
        lambda a: highpass_cnt(
            a, low_cut_hz, cnt.info['sfreq'], filt_order=3, axis=1), cnt)
    log.info("Standardizing...")
    cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, cnt)

    ival = [-500, 4000]

    dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)
    return dataset
コード例 #11
0
def preprocess_cnt(cnt, final_hz, half_before):
    log.info("Resampling...")
    cnt = resample_cnt(cnt, 250.0)
    log.info("Standardizing...")
    cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4
        ).T,
        cnt,
    )
    if half_before:
        cnt = resample_cnt(cnt, final_hz / 2.0)
    cnt = resample_cnt(cnt, final_hz)
    return cnt
コード例 #12
0
def test_exponential_running_init_block_size(mock_data):
    mock_input, _, _ = mock_data
    init_block_size = 3
    standardized_data = exponential_running_standardize(
        mock_input, init_block_size=init_block_size)
    np.testing.assert_allclose(standardized_data[:, :init_block_size].sum(),
                               [0],
                               rtol=1e-4,
                               atol=1e-4)

    demeaned_data = exponential_running_demean(mock_input,
                                               init_block_size=init_block_size)
    np.testing.assert_allclose(demeaned_data[:, :init_block_size].sum(), [0],
                               rtol=1e-4,
                               atol=1e-4)
コード例 #13
0
def import_EEGData_test(start=0, end=9, dir='../data_HGD/test/'):
    X, y = [], []
    for i in range(start, end):
        dataFile = str(dir + str(i + 1) + '.mat')
        print("File:", dataFile, " loading...")
        cnt = BBCIDataset(filename=dataFile, load_sensor_names=None).load()
        marker_def = OrderedDict([('Right Hand', [1]), (
            'Left Hand',
            [2],
        ), ('Rest', [3]), ('Feet', [4])])
        clean_ival = [0, 4000]

        set_for_cleaning = create_signal_target_from_raw_mne(
            cnt, marker_def, clean_ival)
        clean_trial_mask = np.max(np.abs(set_for_cleaning.X),
                                  axis=(1, 2)) < 800

        C_sensors = [
            'FC5', 'FC1', 'FC2', 'FC6', 'C3', 'C4', 'CP5', 'CP1', 'CP2', 'CP6',
            'FC3', 'FCz', 'FC4', 'C5', 'C1', 'C2', 'C6', 'CP3', 'CPz', 'CP4',
            'FFC5h', 'FFC3h', 'FFC4h', 'FFC6h', 'FCC5h', 'FCC3h', 'FCC4h',
            'FCC6h', 'CCP5h', 'CCP3h', 'CCP4h', 'CCP6h', 'CPP5h', 'CPP3h',
            'CPP4h', 'CPP6h', 'FFC1h', 'FFC2h', 'FCC1h', 'FCC2h', 'CCP1h',
            'CCP2h', 'CPP1h', 'CPP2h'
        ]

        cnt = cnt.pick_channels(C_sensors)
        cnt = resample_cnt(cnt, 250.0)
        cnt = mne_apply(
            lambda a: exponential_running_standardize(
                a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, cnt)
        ival = [-500, 4000]

        dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)
        dataset.X = dataset.X[clean_trial_mask]
        dataset.X = dataset.X[:, :, np.newaxis, :]
        dataset.y = dataset.y[clean_trial_mask]
        dataset.y = dataset.y[:, np.newaxis]

        X.extend(dataset.X)
        y.extend(dataset.y)

    X = data_in_one(np.array(X))
    y = np.array(y)
    print("X:", X.shape)
    print("y:", y.shape)
    dataset = EEGDataset(X, y)
    return dataset
コード例 #14
0
 def add_training_blocks_from_old_data(self, old_samples, old_markers,
                                       factor_new, eps):
     # first standardize data
     old_samples = exponential_running_standardize(old_samples,
                                                   factor_new=factor_new,
                                                   init_block_size=1000,
                                                   eps=eps)
     trial_starts, trial_stops = self.get_trial_start_stop_indices(
         old_markers)
     log.info("Adding {:d} trials".format(len(trial_starts)))
     for trial_start, trial_stop in zip(trial_starts, trial_stops):
         self.add_trial(trial_start, trial_stop, old_samples, old_markers)
     # now lets add breaks
     log.info("Adding {:d} breaks".format(len(trial_starts) - 1))
     for break_start, break_stop in zip(trial_stops[:-1], trial_starts[1:]):
         self.add_break(break_start, break_stop, old_samples, old_markers)
コード例 #15
0
    def default_preprocessing_functions(self):
        preprocessing_functions = []

        if self.secs_to_cut_at_start_end > 0:
            preprocessing_functions.append(lambda data, fs: (data[:, int(self.secs_to_cut_at_start_end * fs):-int(
                self.secs_to_cut_at_start_end * fs)], fs))

        if self.duration_mins > 0:
            preprocessing_functions.append(lambda data, fs: (data[:, :int(self.duration_mins * 60 * fs)], fs))

        preprocessing_functions.append(lambda data, fs:
                                       (resampy.resample(data, sr_orig=fs, sr_new=self.sampling_freq, axis=1,
                                                         filter=self.subsample_filter), self.sampling_freq))
        if self.max_abs_value > 0:
            preprocessing_functions.append(lambda data, fs: (
                np.clip(data, -self.max_abs_value, self.max_abs_value), fs))

        if self.exponential_normalization:
            preprocessing_functions.append(lambda data, fs: (
                                           exponential_running_standardize(data.T, init_block_size=1000,
                                                                           factor_new=0.001, eps=1e-4).T, fs))

        return preprocessing_functions
コード例 #16
0
def process_bbci_data(filename, labels_filename, low_cut_hz):
    ival = [-500, 4000]
    high_cut_hz = 38
    factor_new = 1e-3
    init_block_size = 1000

    loader = BCICompetition4Set2A(filename, labels_filename=labels_filename)
    cnt = loader.load()

    # Preprocessing
    cnt = cnt.drop_channels(
        ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right'])
    assert len(cnt.ch_names) == 22

    # lets convert to millvolt for numerical stability of next operations
    cnt = mne_apply(lambda a: a * 1e6, cnt)
    cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), cnt)
    cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, cnt)

    marker_def = OrderedDict([('Left Hand', [1]), (
        'Right Hand',
        [2],
    ), ('Foot', [3]), ('Tongue', [4])])

    dataset = create_signal_target_from_raw_mne(cnt, marker_def, ival)
    return dataset
コード例 #17
0
def standardize_cnt(cnt, standardize_mode=0):
    # computing frequencies
    sampling_freq = cnt.info['sfreq']
    init_freq = 0.1
    stop_freq = sampling_freq / 2 - 0.1
    filt_order = 3
    axis = 0
    filtfilt = False

    # filtering DC and frequencies higher than the nyquist one
    cnt = mne_apply(
        lambda x: bandpass_cnt(data=x,
                               low_cut_hz=init_freq,
                               high_cut_hz=stop_freq,
                               fs=sampling_freq,
                               filt_order=filt_order,
                               axis=axis,
                               filtfilt=filtfilt), cnt)

    # removing mean and normalizing in 3 different ways
    if standardize_mode == 0:
        # x - mean
        cnt = mne_apply(lambda x: x - mean(x, axis=0, keepdims=True), cnt)
    elif standardize_mode == 1:
        # (x - mean) / std
        cnt = mne_apply(
            lambda x: (x - mean(x, axis=0, keepdims=True)) / std(
                x, axis=0, keepdims=True), cnt)
    elif standardize_mode == 2:
        # parsing to milli volt for numerical stability of next operations
        cnt = mne_apply(lambda a: a * 1e6, cnt)

        # applying exponential_running_standardize (Schirrmeister)
        cnt = mne_apply(
            lambda x: exponential_running_standardize(
                x.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, cnt)
    return cnt
コード例 #18
0
def run_exp(max_recording_mins, n_recordings, sec_to_cut,
            duration_recording_mins, max_abs_val, max_min_threshold,
            max_min_expected, shrink_val, max_min_remove, batch_set_zero_val,
            batch_set_zero_test, sampling_freq, low_cut_hz, high_cut_hz,
            exp_demean, exp_standardize, moving_demean, moving_standardize,
            channel_demean, channel_standardize, divisor, n_folds, i_test_fold,
            input_time_length, final_conv_length, pool_stride, n_blocks_to_add,
            sigmoid, model_constraint, batch_size, max_epochs,
            only_return_exp):
    cuda = True

    preproc_functions = []
    preproc_functions.append(lambda data, fs: (
        data[:, int(sec_to_cut * fs):-int(sec_to_cut * fs)], fs))
    preproc_functions.append(lambda data, fs: (data[:, :int(
        duration_recording_mins * 60 * fs)], fs))
    if max_abs_val is not None:
        preproc_functions.append(
            lambda data, fs: (np.clip(data, -max_abs_val, max_abs_val), fs))
    if max_min_threshold is not None:
        preproc_functions.append(lambda data, fs: (clean_jumps(
            data, 200, max_min_threshold, max_min_expected, cuda), fs))
    if max_min_remove is not None:
        window_len = 200
        preproc_functions.append(lambda data, fs: (set_jumps_to_zero(
            data,
            window_len=window_len,
            threshold=max_min_remove,
            cuda=cuda,
            clip_min_max_to_zero=True), fs))

    if shrink_val is not None:
        preproc_functions.append(lambda data, fs: (shrink_spikes(
            data,
            shrink_val,
            1,
            9,
        ), fs))

    preproc_functions.append(lambda data, fs: (resampy.resample(
        data, fs, sampling_freq, axis=1, filter='kaiser_fast'), sampling_freq))
    preproc_functions.append(lambda data, fs: (bandpass_cnt(
        data, low_cut_hz, high_cut_hz, fs, filt_order=4, axis=1), fs))

    if exp_demean:
        preproc_functions.append(lambda data, fs: (exponential_running_demean(
            data.T, factor_new=0.001, init_block_size=100).T, fs))
    if exp_standardize:
        preproc_functions.append(
            lambda data, fs: (exponential_running_standardize(
                data.T, factor_new=0.001, init_block_size=100).T, fs))
    if moving_demean:
        preproc_functions.append(lambda data, fs: (padded_moving_demean(
            data, axis=1, n_window=201), fs))
    if moving_standardize:
        preproc_functions.append(lambda data, fs: (padded_moving_standardize(
            data, axis=1, n_window=201), fs))
    if channel_demean:
        preproc_functions.append(lambda data, fs: (demean(data, axis=1), fs))
    if channel_standardize:
        preproc_functions.append(lambda data, fs:
                                 (standardize(data, axis=1), fs))
    if divisor is not None:
        preproc_functions.append(lambda data, fs: (data / divisor, fs))

    dataset = DiagnosisSet(n_recordings=n_recordings,
                           max_recording_mins=max_recording_mins,
                           preproc_functions=preproc_functions)
    if not only_return_exp:
        X, y = dataset.load()

    splitter = Splitter(
        n_folds,
        i_test_fold,
    )
    if not only_return_exp:
        train_set, valid_set, test_set = splitter.split(X, y)
        del X, y  # shouldn't be necessary, but just to make sure
    else:
        train_set = None
        valid_set = None
        test_set = None

    set_random_seeds(seed=20170629, cuda=cuda)
    if sigmoid:
        n_classes = 1
    else:
        n_classes = 2
    in_chans = 21

    net = Deep4Net(
        in_chans=in_chans,
        n_classes=n_classes,
        input_time_length=input_time_length,
        final_conv_length=final_conv_length,
        pool_time_length=pool_stride,
        pool_time_stride=pool_stride,
        n_filters_2=50,
        n_filters_3=80,
        n_filters_4=120,
    )
    model = net_with_more_layers(net, n_blocks_to_add, nn.MaxPool2d)
    if sigmoid:
        model = to_linear_plus_minus_net(model)
    optimizer = optim.Adam(model.parameters())
    to_dense_prediction_model(model)
    log.info("Model:\n{:s}".format(str(model)))
    if cuda:
        model.cuda()
    # determine output size
    test_input = np_to_var(
        np.ones((2, in_chans, input_time_length, 1), dtype=np.float32))
    if cuda:
        test_input = test_input.cuda()
    out = model(test_input)
    n_preds_per_input = out.cpu().data.numpy().shape[2]
    log.info("{:d} predictions per input/trial".format(n_preds_per_input))
    iterator = CropsFromTrialsIterator(batch_size=batch_size,
                                       input_time_length=input_time_length,
                                       n_preds_per_input=n_preds_per_input)
    if sigmoid:
        loss_function = lambda preds, targets: binary_cross_entropy_with_logits(
            th.mean(preds, dim=2)[:, 1, 0], targets.type_as(preds))
    else:
        loss_function = lambda preds, targets: F.nll_loss(
            th.mean(preds, dim=2)[:, :, 0], targets)

    if model_constraint is not None:
        model_constraint = MaxNormDefaultConstraint()
    monitors = [
        LossMonitor(),
        MisclassMonitor(col_suffix='sample_misclass'),
        CroppedTrialMisclassMonitor(input_time_length),
        RuntimeMonitor(),
    ]
    stop_criterion = MaxEpochs(max_epochs)
    batch_modifier = None
    if batch_set_zero_val is not None:
        batch_modifier = RemoveMinMaxDiff(batch_set_zero_val,
                                          clip_max_abs=True,
                                          set_zero=True)
    if (batch_set_zero_val is not None) and (batch_set_zero_test == True):
        iterator = ModifiedIterator(
            iterator,
            batch_modifier,
        )
        batch_modifier = None
    exp = Experiment(model,
                     train_set,
                     valid_set,
                     test_set,
                     iterator,
                     loss_function,
                     optimizer,
                     model_constraint,
                     monitors,
                     stop_criterion,
                     remember_best_column='valid_misclass',
                     run_after_early_stop=True,
                     batch_modifier=batch_modifier,
                     cuda=cuda)
    if not only_return_exp:
        exp.run()
    else:
        exp.dataset = dataset
        exp.splitter = splitter

    return exp
コード例 #19
0
ファイル: data_preprocessing.py プロジェクト: erap129/EEGNAS
def get_bci_iv_2a_train_val_test(data_folder, subject_id, low_cut_hz):
    ival = [
        -500, 4000
    ]  # this is the window around the event from which we will take data to feed to the classifier
    high_cut_hz = 38  # cut off parts of signal higher than 38 hz
    factor_new = 1e-3  # ??? has to do with exponential running standardize
    init_block_size = 1000  # ???

    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(train_filepath,
                                        labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(test_filepath,
                                       labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    train_cnt = train_cnt.drop_channels(
        ['EOG-left', 'EOG-central', 'EOG-right'])
    if len(train_cnt.ch_names) > 22:
        train_cnt = train_cnt.drop_channels(['STI 014'])
    assert len(train_cnt.ch_names) == 22

    # convert measurements to millivolt
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(  # signal processing procedure that I don't understand
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               train_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), train_cnt)
    train_cnt = mne_apply(  # signal processing procedure that I don't understand
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, train_cnt)

    test_cnt = test_cnt.drop_channels(['EOG-left', 'EOG-central', 'EOG-right'])
    if len(test_cnt.ch_names) > 22:
        test_cnt = test_cnt.drop_channels(['STI 014'])
    assert len(test_cnt.ch_names) == 22

    # convert measurements to millivolt
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               test_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, test_cnt)

    marker_def = OrderedDict([('Left Hand', [1]), (
        'Right Hand',
        [2],
    ), ('Foot', [3]), ('Tongue', [4])])

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)
    train_set, valid_set = split_into_two_sets(
        train_set,
        first_set_fraction=1 - global_vars.get('valid_set_fraction'))

    return train_set, valid_set, test_set
コード例 #20
0
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda):
    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(train_filepath,
                                        labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(test_filepath,
                                       labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    # Preprocessing

    train_cnt = train_cnt.drop_channels(
        ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right'])
    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(
            a, low_cut_hz, 38, train_cnt.info['sfreq'], filt_order=3, axis=1),
        train_cnt)
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, train_cnt)

    test_cnt = test_cnt.drop_channels(
        ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right'])
    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(
            a, low_cut_hz, 38, test_cnt.info['sfreq'], filt_order=3, axis=1),
        test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T, factor_new=1e-3, init_block_size=1000, eps=1e-4).T, test_cnt)

    marker_def = OrderedDict([('Left Hand', [1]), (
        'Right Hand',
        [2],
    ), ('Foot', [3]), ('Tongue', [4])])
    ival = [-500, 4000]

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(train_set,
                                               first_set_fraction=0.8)

    set_random_seeds(seed=20190706, cuda=cuda)

    n_classes = 4
    n_chans = int(train_set.X.shape[1])
    input_time_length = train_set.X.shape[2]
    if model == 'shallow':
        model = ShallowFBCSPNet(n_chans,
                                n_classes,
                                input_time_length=input_time_length,
                                final_conv_length='auto').create_network()
    elif model == 'deep':
        model = Deep4Net(n_chans,
                         n_classes,
                         input_time_length=input_time_length,
                         final_conv_length='auto').create_network()
    if cuda:
        model.cuda()
    log.info("Model: \n{:s}".format(str(model)))

    optimizer = optim.Adam(model.parameters())

    iterator = BalancedBatchSizeIterator(batch_size=60)

    stop_criterion = Or([MaxEpochs(1600), NoDecrease('valid_misclass', 160)])

    monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()]

    model_constraint = MaxNormDefaultConstraint()

    exp = Experiment(model,
                     train_set,
                     valid_set,
                     test_set,
                     iterator=iterator,
                     loss_function=F.nll_loss,
                     optimizer=optimizer,
                     model_constraint=model_constraint,
                     monitors=monitors,
                     stop_criterion=stop_criterion,
                     remember_best_column='valid_misclass',
                     run_after_early_stop=True,
                     cuda=cuda)
    exp.run()
    return exp
コード例 #21
0
ファイル: bcic_iv_2a.py プロジェクト: zhangys2019/braindecode
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda):
    ival = [-500, 4000]
    max_epochs = 1600
    max_increase_epochs = 160
    batch_size = 60
    high_cut_hz = 38
    factor_new = 1e-3
    init_block_size = 1000
    valid_set_fraction = 0.2

    train_filename = "A{:02d}T.gdf".format(subject_id)
    test_filename = "A{:02d}E.gdf".format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace(".gdf", ".mat")
    test_label_filepath = test_filepath.replace(".gdf", ".mat")

    train_loader = BCICompetition4Set2A(
        train_filepath, labels_filename=train_label_filepath
    )
    test_loader = BCICompetition4Set2A(
        test_filepath, labels_filename=test_label_filepath
    )
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    # Preprocessing

    train_cnt = train_cnt.drop_channels(
        ["EOG-left", "EOG-central", "EOG-right"]
    )
    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(
            a,
            low_cut_hz,
            high_cut_hz,
            train_cnt.info["sfreq"],
            filt_order=3,
            axis=1,
        ),
        train_cnt,
    )
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T,
            factor_new=factor_new,
            init_block_size=init_block_size,
            eps=1e-4,
        ).T,
        train_cnt,
    )

    test_cnt = test_cnt.drop_channels(["EOG-left", "EOG-central", "EOG-right"])
    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(
            a,
            low_cut_hz,
            high_cut_hz,
            test_cnt.info["sfreq"],
            filt_order=3,
            axis=1,
        ),
        test_cnt,
    )
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T,
            factor_new=factor_new,
            init_block_size=init_block_size,
            eps=1e-4,
        ).T,
        test_cnt,
    )

    marker_def = OrderedDict(
        [
            ("Left Hand", [1]),
            ("Right Hand", [2]),
            ("Foot", [3]),
            ("Tongue", [4]),
        ]
    )

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(
        train_set, first_set_fraction=1 - valid_set_fraction
    )

    set_random_seeds(seed=20190706, cuda=cuda)

    n_classes = 4
    n_chans = int(train_set.X.shape[1])
    input_time_length = train_set.X.shape[2]
    if model == "shallow":
        model = ShallowFBCSPNet(
            n_chans,
            n_classes,
            input_time_length=input_time_length,
            final_conv_length="auto",
        ).create_network()
    elif model == "deep":
        model = Deep4Net(
            n_chans,
            n_classes,
            input_time_length=input_time_length,
            final_conv_length="auto",
        ).create_network()
    if cuda:
        model.cuda()
    log.info("Model: \n{:s}".format(str(model)))

    optimizer = optim.Adam(model.parameters())

    iterator = BalancedBatchSizeIterator(batch_size=batch_size)

    stop_criterion = Or(
        [
            MaxEpochs(max_epochs),
            NoDecrease("valid_misclass", max_increase_epochs),
        ]
    )

    monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()]

    model_constraint = MaxNormDefaultConstraint()

    exp = Experiment(
        model,
        train_set,
        valid_set,
        test_set,
        iterator=iterator,
        loss_function=F.nll_loss,
        optimizer=optimizer,
        model_constraint=model_constraint,
        monitors=monitors,
        stop_criterion=stop_criterion,
        remember_best_column="valid_misclass",
        run_after_early_stop=True,
        cuda=cuda,
    )
    exp.run()
    return exp
コード例 #22
0
def dlvr_braindecode(path,
                     files,
                     timeframe_start,
                     target_fps,
                     preprocessing=True):
    """ Uses event markers to extract motor tasks from multiple DLVR .xdf files.
    Args:
        path: If the files share a single path, you can specify it here.
        
        files: A list of .xdf files to extract data from.
        
        timeframe_start: The time in seconds before the event, in which the EEG Data is extracted.
        
        target_fps: Downsample the EEG-data to this value.     

        preprocessing: Filters and demean/unitvariance activated (true) or deactivated (false)
    
    Returns:
        X: A list of trials
        y: An array specifying the action
    """

    # Epochs list containing differently sized arrays [#eeg_electrodes, times]
    X = []
    #event ids corresponding to the trials where 'left' = 0 and 'right' = 1
    y = np.array([])
    for file in files:
        #load a file
        print('Reading ', file)
        current_raw = xdf_loader(path + file)

        # For MEGVR experiments switch EMG into C3/4
        current_raw._data[14, :] = current_raw.get_data(picks=['EMG_LH'])  #C3
        current_raw._data[16, :] = current_raw.get_data(picks=['EMG_RH'])  #C4

        #discard EOG/EMG
        current_raw.pick_types(meg=False, eeg=True)
        #pick only relevant events
        events = current_raw.events[
            (current_raw.events[:, 2] == current_raw.event_id['Monster left'])
            | (current_raw.events[:,
                                  2] == current_raw.event_id['Monster right'])]
        #timestamps where a monster deactivates
        stops = current_raw.events[:, 0][(current_raw.events[:,
                                                             2] == current_raw.
                                          event_id['Monster destroyed'])]
        #timestamps where trials begin
        starts = events[:, 0]

        #extract event_ids and shift them to [0, 1]
        key = events[:, 2]
        key = (key == key.max()).astype(np.int64)

        #standardize, convert to size(time, channels)
        #current_raw._data = exponential_running_standardize(current_raw._data.T, factor_new=0.001, init_block_size=None, eps=0.0001).T

        #Find the trials and their corresponding end points

        bads = np.array([])
        for count, event in enumerate(starts):
            #in case the last trial has no end (experiment ended before the trial ends), discard it
            if len(stops[stops > event]) == 0:
                key = key[:-sum(starts >= event)]
                break

            if stops[stops > event][0] - event < 5000:
                bads = np.append(bads, count)
                continue

            #Get the trial from 1 second before the task starts to the next 'Monster deactived' flag
            current_epoch = current_raw._data[:, event -
                                              round(timeframe_start *
                                                    5000):stops[
                                                        stops > event][0]]

            if preprocessing == True:
                #filter signal
                B_1, A_1 = butter(5, 1, btype='high', output='ba', fs=5000)

                # Butter filter (lowpass) for 30 Hz
                B_40, A_40 = butter(6, 120, btype='low', output='ba', fs=5000)

                # Notch filter with 50 HZ
                F0 = 50.0
                Q = 30.0  # Quality factor
                # Design notch filter
                B_50, A_50 = iirnotch(F0, Q, 5000)

                current_epoch = filtfilt(B_50, A_50, current_epoch)
                current_epoch = filtfilt(B_40, A_40, current_epoch)
                #current_epoch = filtfilt(B_1, A_1, current_epoch)

            #downsample to 250 Hz
            current_epoch = resampy.resample(current_epoch, 5000, 250, axis=1)
            current_epoch = current_epoch.astype(np.float32)

            if preprocessing == True:
                #standardize, convert to size(time, channels)
                current_epoch = exponential_running_standardize(
                    current_epoch.T,
                    factor_new=0.001,
                    init_block_size=None,
                    eps=0.0001).T

            X.append(current_epoch)

        if len(bads) > 0:
            key = np.delete(key, bads)
        y = np.append(y, key)
    y = y.astype(np.int64)
    return (X, y)
コード例 #23
0
def static_epochs(path, files, regexp, timeframe_start, timeframe_end,
                  target_fps):
    """ Extracts and accumulates given timeframes around events from different .xdf files.
    Args:
        path: If the files share a single path, you can specify it here.
        
        files: A list of .xdf files to extract data from.
        
        regexp: A regular expression that defines the format of the extracted events.
        
        timeframe_start: The time in seconds before the event, in which the EEG Data is extracted.
        
        timeframe_end: The time in seconds after the event, in which the EEG Data is extracted.
        
        target_fps: Downsample the EEG-data to this value.         
    
    Returns:
        epoch: An mne Epochs object
    """

    master_id = {}
    epoch = []
    master_legend = []
    for file in files:
        print('Reading ', file)
        current_raw = xdf_loader(path + file)
        current_events = current_raw.events
        current_id = current_raw.event_id

        current_raw._data = exponential_running_standardize(
            current_raw._data,
            factor_new=0.001,
            init_block_size=None,
            eps=0.0001)
        #current_raw.filter(1,40)

        # Compute which actions are available in the current file
        here = np.array([
            bool(re.search(regexp, element))
            for element in list(current_id.keys())
        ])
        legend = np.array(list(current_id.keys()))[here]

        # Update Master legend and ID if the current file includes new actions
        for event in legend[[item not in master_legend for item in legend]]:
            master_id[event] = len(master_id)
            master_legend = np.append(master_legend, event)

        picked_events = np.empty([0, 3], dtype=int)
        picked_id = {}

        for this in legend:
            # Get all appropriate events
            picked_events = np.append(
                picked_events,
                current_events[current_events[:, 2] == current_id[this]],
                axis=0)
            # Update the ID according to master
            picked_events[:, 2][picked_events[:, 2] ==
                                current_id[this]] = master_id[this]
            # Build up a temp ID dict for the current Epochs
            picked_id[this] = master_id[this]

        # Building empty Epochs will throw errors
        if not picked_id:
            continue
        current_epoch = mne.Epochs(current_raw,
                                   picked_events,
                                   picked_id,
                                   tmin=-timeframe_start,
                                   tmax=timeframe_end)
        current_epoch.load_data()
        current_epoch.resample(target_fps)

        # Append the current epochs if there are epochs to append to
        if not epoch:
            epoch = current_epoch.copy()
        else:
            epoch = mne.EpochsArray(np.append(epoch[:].get_data(),
                                              current_epoch[:].get_data(),
                                              axis=0),
                                    info=epoch.info,
                                    events=np.append(epoch.events,
                                                     current_epoch.events,
                                                     axis=0),
                                    event_id=master_id)

    return epoch
コード例 #24
0
ファイル: SourceCode.py プロジェクト: High-East/Deep-BCI
    ### (3) Preprocessing #####################################################
    ###########################################################################
    from braindecode.datautil.signalproc import lowpass_cnt, highpass_cnt, exponential_running_standardize
    for ii in range(
            0, 60):  # change according to the number of trials (default = 60)
        # 1. Data reconstruction
        temp_data = FeatVect[ii, :, :]
        temp_data = temp_data.transpose()
        # 2. Lowpass filtering
        lowpassed_data = lowpass_cnt(temp_data, 13, 200, filt_order=3)
        # 3. Highpass filtering
        bandpassed_data = highpass_cnt(lowpassed_data, 8, 200, filt_order=3)
        # 4. Exponential running standardization
        ExpRunStand_data = exponential_running_standardize(
            bandpassed_data,
            factor_new=0.001,
            init_block_size=None,
            eps=0.0001)
        # 5. Renewal preprocessed data
        ExpRunStand_data = ExpRunStand_data.transpose()
        FeatVect[ii, :, :] = ExpRunStand_data
        del temp_data, lowpassed_data, bandpassed_data, ExpRunStand_data

    ###########################################################################
    ### (3) Convert data to braindecode format ################################
    ###########################################################################
    # pytorch expects float 32 for input and int64 for labels.
    X = (FeatVect).astype(np.float32)
    y = (y_labels).astype(np.int64)
    y = np.delete(y, [60], None)
    del FeatVect, y_labels
コード例 #25
0
def run_exp(data_folder, subject_id, low_cut_hz, model, cuda):
    ival = [-500, 4000]
    input_time_length = 1000
    max_epochs = 800
    max_increase_epochs = 80
    batch_size = 60
    high_cut_hz = 38
    factor_new = 1e-3
    init_block_size = 1000
    valid_set_fraction = 0.2

    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(train_filepath,
                                        labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(test_filepath,
                                       labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    # Preprocessing

    train_cnt = train_cnt.drop_channels(
        ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right'])
    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               train_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), train_cnt)
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, train_cnt)

    test_cnt = test_cnt.drop_channels(
        ['STI 014', 'EOG-left', 'EOG-central', 'EOG-right'])
    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               test_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, test_cnt)

    marker_def = OrderedDict([('Left Hand', [1]), (
        'Right Hand',
        [2],
    ), ('Foot', [3]), ('Tongue', [4])])

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(train_set,
                                               first_set_fraction=1 -
                                               valid_set_fraction)

    set_random_seeds(seed=20190706, cuda=cuda)

    n_classes = 4
    n_chans = int(train_set.X.shape[1])
    if model == 'shallow':
        model = ShallowFBCSPNet(n_chans,
                                n_classes,
                                input_time_length=input_time_length,
                                final_conv_length=30).create_network()
    elif model == 'deep':
        model = Deep4Net(n_chans,
                         n_classes,
                         input_time_length=input_time_length,
                         final_conv_length=2).create_network()

    to_dense_prediction_model(model)
    if cuda:
        model.cuda()

    log.info("Model: \n{:s}".format(str(model)))
    dummy_input = np_to_var(train_set.X[:1, :, :, None])
    if cuda:
        dummy_input = dummy_input.cuda()
    out = model(dummy_input)

    n_preds_per_input = out.cpu().data.numpy().shape[2]

    optimizer = optim.Adam(model.parameters())

    iterator = CropsFromTrialsIterator(batch_size=batch_size,
                                       input_time_length=input_time_length,
                                       n_preds_per_input=n_preds_per_input)

    stop_criterion = Or([
        MaxEpochs(max_epochs),
        NoDecrease('valid_misclass', max_increase_epochs)
    ])

    monitors = [
        LossMonitor(),
        MisclassMonitor(col_suffix='sample_misclass'),
        CroppedTrialMisclassMonitor(input_time_length=input_time_length),
        RuntimeMonitor()
    ]

    model_constraint = MaxNormDefaultConstraint()

    loss_function = lambda preds, targets: F.nll_loss(
        th.mean(preds, dim=2, keepdim=False), targets)

    exp = Experiment(model,
                     train_set,
                     valid_set,
                     test_set,
                     iterator=iterator,
                     loss_function=loss_function,
                     optimizer=optimizer,
                     model_constraint=model_constraint,
                     monitors=monitors,
                     stop_criterion=stop_criterion,
                     remember_best_column='valid_misclass',
                     run_after_early_stop=True,
                     cuda=cuda)
    exp.run()
    return exp
コード例 #26
0
def concat_prepare_cnn(input_signal):
    # In this particular data set
    # it was required by the author of it,
    # that for preventing the algorithm
    # to pick on data of the eye movement,
    # a high band filter of Hz had to
    # be implimented.
    low_cut_hz = 1

    # The authors prove both configuration >38  an 38< frequenzy
    # in the current experiment, we see that band pass filter will take
    # Theta to part of Gamma frequenzy band
    # whihch what Filter Bank Commun spatial patters would do.
    # This value is a hiperpartemer that should be ajusted
    # per data set... In my opinion.
    high_cut_hz = 40

    # factor for exponential smothing
    # are this numbers usually setup used
    # on neuro sciencie?
    factor_new = 1e-3

    # initianlization values for the the mean and variance,
    # see prior discussion
    init_block_size = 1000

    # model = "shallow"  #'shallow' or 'deep'
    # GPU support
    # cuda = True

    # It was stated in the paper [1] that
    # "trial   window   for   later   experiments   with   convolutional
    # networks,  that  is,  from  0.5  to  4  s."
    # 0- 20s?
    # so "ival" variable simple states what milisecond interval to analize
    # per trial.
    ival = [0, 20000]

    # An epoch increase every time the whole training data point
    # had been input to the network. An epoch is not a batch
    # example, if we have 100 training data points
    # and we use batch_size 10, it will take 10 iterations of
    # batch_size to reach 1 epoch.
    # max_epochs = 1600

    # max_increase_epochs = 160

    # 60 data point per forward-backwards propagation
    # batch_size = 60

    # pertecentage of data to be used as test-set
    valid_set_fraction = 0.2

    gdf_events = mne.find_events(input_signal)

    input_signal = input_signal.drop_channels(["stim"])

    raw_training_signal = input_signal.get_data()

    print("data shape:", raw_training_signal.shape)

    for i_chan in range(raw_training_signal.shape[0]):
        # first set to nan, than replace nans by nanmean.
        this_chan = raw_training_signal[i_chan]
        raw_training_signal[i_chan] = np.where(this_chan == np.min(this_chan),
                                               np.nan, this_chan)
        mask = np.isnan(raw_training_signal[i_chan])
        chan_mean = np.nanmean(raw_training_signal[i_chan])
        raw_training_signal[i_chan, mask] = chan_mean

    # Reconstruct
    input_signal = mne.io.RawArray(raw_training_signal,
                                   input_signal.info,
                                   verbose="WARNING")
    # append the extracted events
    # raw_gdf_training_signal
    # raw_gdf_training_signal
    input_signal.info["events"] = gdf_events

    train_cnt = input_signal
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(
            a,
            low_cut_hz,
            high_cut_hz,
            train_cnt.info["sfreq"],
            filt_order=3,
            axis=1,
        ),
        train_cnt,
    )

    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(
            a.T,
            factor_new=factor_new,
            init_block_size=init_block_size,
            eps=1e-4,
        ).T,
        train_cnt,
    )

    marker_def = OrderedDict([("ec", [30])])

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    return train_set
コード例 #27
0
def process_epochs(epochs, filters, channels, target_fs):
    r''' function for epoch preprocesing: filtering, channels extraction and resampling

    Arguments:
        epochs (mne.Epochs object, required): mne.Epochs object that contains epoched data
        filters (bool, required): if True - apply filters to data:
            highpass (butter) filter with fc=1Hz
            notch filter with fc=50Hz
            lowpass (butter) filter with fc=30Hz
            NOTE: currently filtering is  implemented with filfilt function
        channels (str, required): channels to extract from data
            'eeg': only eeg channels,
            'eeg_eog': eog channels averaged with respect to eeg channels,
            'MI': only motor-imagery relevant channels (C-)
        target_fs (int,required): resampling frequency

    Output:
        epochs (numpy array, (number of trials, number of channels, signal length)): processed epochs
    '''

    if channels == 'eeg_eog':
        epochs.pick_types(eeg=True, eog=True)
        print('Only eog averaged with eeg')
        ch_names = np.array(epochs.ch_names)
        epochs = epochs.get_data()

        epochs = eog_preprocess(epochs, ch_names)

    elif channels == 'eeg':
        epochs.pick_types(eeg=True)
        print('Only eeg')
        epochs = epochs.get_data()

    elif channels == 'MI':

        channels_motor = [
            'FC5', 'FC1', 'FC2', 'FC6', 'C3', 'C4', 'CP5', 'CP1', 'CP2', 'CP6',
            'FC3', 'FCz', 'FC4', 'C5', 'C1', 'C2', 'C6', 'CP3', 'CPz', 'CP4'
        ]
        chan_nr = np.array([])
        for channel in channels_motor:
            chan_nr = np.append(chan_nr,
                                epochs.ch_names.index(channel)).astype(int)

        epochs = epochs.get_data()

        epochs = epochs[:, chan_nr, :]
        print('Only MI paradigm relevant channels')

    if filters:
        ### filters ###

        # Butter filter (highpass) for 1 Hz
        b_1, a_1 = signal.butter(6, 1 / target_fs, btype='high', output='ba')

        # Butter filter (lowpass) for 30 Hz
        b_30, a_30 = signal.butter(6, 30 / target_fs, btype='low', output='ba')

        # Notch filter with 50 HZ
        f0 = 50.0
        Q = 30.0  # Quality factor
        # Design notch filter
        b_50, a_50 = signal.iirnotch(f0, Q, target_fs)

        #####

        epochs = signal.filtfilt(b_50, a_50, epochs)
        epochs = signal.filtfilt(b_1, a_1, epochs)
        epochs = signal.filtfilt(b_30, a_30, epochs)
        print('Filtered: highpass with 1Hz and lowpass with 30Hz')

    # exp_mean_standardize
    for ep in range(epochs.shape[0]):
        processed = exponential_running_standardize(epochs[ep, :, :].swapaxes(
            0, 1),
                                                    factor_new=0.001,
                                                    init_block_size=None,
                                                    eps=0.0001)
        epochs[ep, :, :] = np.swapaxes(processed, 0, 1)

    return epochs
コード例 #28
0
            # Remove stimulus channel
            cnt = cnt.drop_channels(['STI 014'])

            # for now, remove also robot,ecg and respiration channels
            #cnt = cnt.drop_channels(sensor_names_robot_aux)

            # resample if wrong sf
            resampleToHz = samplingRate
            cnt = resample_cnt(cnt, resampleToHz)

            # mne apply will apply the function to the data (a 2d-numpy-array)
            # have to transpose data back and forth, since
            # exponential_running_standardize expects time x chans order
            # while mne object has chans x time order
            cnt = mne_apply(
                lambda a: exponential_running_standardize(
                    a.T, init_block_size=1000, factor_new=0.001, eps=1e-4).T,
                cnt)

            name_to_start_codes = OrderedDict([('ScoreExp', 1)])
            name_to_stop_codes = OrderedDict([('ScoreExp', 2)])

            train_set = create_signal_target_from_raw_mne(
                cnt, name_to_start_codes, [0, 0], name_to_stop_codes)

            train_set.y = Score[:, :-1]

            # Outer added axis is the trial axis (size one always...)
            #from braindecode.datautil.signal_target import SignalAndTarget
            #datasets = SignalAndTarget(train_set.X[0].astype(np.float32), train_set.y.astype(np.float32))

            # split data and test set
コード例 #29
0
def run_exp(data_folder, session_id, subject_id, low_cut_hz, model, cuda):
    ival = [-500, 4000]
    max_epochs = 1600
    max_increase_epochs = 160
    batch_size = 10
    high_cut_hz = 38
    factor_new = 1e-3
    init_block_size = 1000
    valid_set_fraction = .2
    ''' # BCIcompetition
    train_filename = 'A{:02d}T.gdf'.format(subject_id)
    test_filename = 'A{:02d}E.gdf'.format(subject_id)
    train_filepath = os.path.join(data_folder, train_filename)
    test_filepath = os.path.join(data_folder, test_filename)
    train_label_filepath = train_filepath.replace('.gdf', '.mat')
    test_label_filepath = test_filepath.replace('.gdf', '.mat')

    train_loader = BCICompetition4Set2A(
        train_filepath, labels_filename=train_label_filepath)
    test_loader = BCICompetition4Set2A(
        test_filepath, labels_filename=test_label_filepath)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()
    '''

    # GIGAscience
    filename = 'sess{:02d}_subj{:02d}_EEG_MI.mat'.format(
        session_id, subject_id)
    filepath = os.path.join(data_folder, filename)
    train_variable = 'EEG_MI_train'
    test_variable = 'EEG_MI_test'

    train_loader = GIGAscience(filepath, train_variable)
    test_loader = GIGAscience(filepath, test_variable)
    train_cnt = train_loader.load()
    test_cnt = test_loader.load()

    # Preprocessing
    ''' channel
    ['Fp1', 'Fp2', 'F7', 'F3', 'Fz', 'F4', 'F8', 'FC5', 'FC1', 'FC2', 'FC6', 'T7', 'C3', 'Cz', 'C4', 'T8', 'TP9', 'CP5',
     'CP1', 'CP2', 'CP6', 'TP10', 'P7', 'P3', 'Pz', 'P4', 'P8', 'PO9', 'O1', 'Oz', 'O2', 'PO10', 'FC3', 'FC4', 'C5',
     'C1', 'C2', 'C6', 'CP3', 'CPz', 'CP4', 'P1', 'P2', 'POz', 'FT9', 'FTT9h', 'TTP7h', 'TP7', 'TPP9h', 'FT10',
     'FTT10h', 'TPP8h', 'TP8', 'TPP10h', 'F9', 'F10', 'AF7', 'AF3', 'AF4', 'AF8', 'PO3', 'PO4']
    '''

    train_cnt = train_cnt.pick_channels([
        'FC5', 'FC3', 'FC1', 'Fz', 'FC2', 'FC4', 'FC6', 'C5', 'C3', 'C1', 'Cz',
        'C2', 'C4', 'C6', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'Pz'
    ])
    train_cnt, train_cnt.info['events'] = train_cnt.copy().resample(
        250, npad='auto', events=train_cnt.info['events'])

    assert len(train_cnt.ch_names) == 22
    # lets convert to millvolt for numerical stability of next operations
    train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
    train_cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               train_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), train_cnt)
    train_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, train_cnt)

    test_cnt = test_cnt.pick_channels([
        'FC5', 'FC3', 'FC1', 'Fz', 'FC2', 'FC4', 'FC6', 'C5', 'C3', 'C1', 'Cz',
        'C2', 'C4', 'C6', 'CP5', 'CP3', 'CP1', 'CPz', 'CP2', 'CP4', 'CP6', 'Pz'
    ])
    test_cnt, test_cnt.info['events'] = test_cnt.copy().resample(
        250, npad='auto', events=test_cnt.info['events'])

    assert len(test_cnt.ch_names) == 22
    test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
    test_cnt = mne_apply(
        lambda a: bandpass_cnt(a,
                               low_cut_hz,
                               high_cut_hz,
                               test_cnt.info['sfreq'],
                               filt_order=3,
                               axis=1), test_cnt)
    test_cnt = mne_apply(
        lambda a: exponential_running_standardize(a.T,
                                                  factor_new=factor_new,
                                                  init_block_size=
                                                  init_block_size,
                                                  eps=1e-4).T, test_cnt)

    marker_def = OrderedDict([('Right Hand', [1]), ('Left Hand', [2])])

    train_set = create_signal_target_from_raw_mne(train_cnt, marker_def, ival)
    test_set = create_signal_target_from_raw_mne(test_cnt, marker_def, ival)

    train_set, valid_set = split_into_two_sets(train_set,
                                               first_set_fraction=1 -
                                               valid_set_fraction)

    set_random_seeds(seed=20190706, cuda=cuda)

    n_classes = 2
    n_chans = int(train_set.X.shape[1])
    input_time_length = train_set.X.shape[2]
    if model == 'shallow':
        model = ShallowFBCSPNet(n_chans,
                                n_classes,
                                input_time_length=input_time_length,
                                final_conv_length='auto').create_network()
    elif model == 'deep':
        model = Deep4Net(n_chans,
                         n_classes,
                         input_time_length=input_time_length,
                         final_conv_length='auto').create_network()
    if cuda:
        model.cuda()
    log.info("Model: \n{:s}".format(str(model)))

    optimizer = optim.Adam(model.parameters())

    iterator = BalancedBatchSizeIterator(batch_size=batch_size)

    stop_criterion = Or([
        MaxEpochs(max_epochs),
        NoDecrease('valid_misclass', max_increase_epochs)
    ])

    monitors = [LossMonitor(), MisclassMonitor(), RuntimeMonitor()]

    model_constraint = MaxNormDefaultConstraint()

    exp = Experiment(model,
                     train_set,
                     valid_set,
                     test_set,
                     iterator=iterator,
                     loss_function=F.nll_loss,
                     optimizer=optimizer,
                     model_constraint=model_constraint,
                     monitors=monitors,
                     stop_criterion=stop_criterion,
                     remember_best_column='valid_misclass',
                     run_after_early_stop=True,
                     cuda=cuda)
    exp.run()
    return exp
コード例 #30
0
# lets convert to millvolt for numerical stability of next operations
train_cnt = mne_apply(lambda a: a * 1e6, train_cnt)
train_cnt = mne_apply(
    lambda a: bandpass_cnt(
        a,
        low_cut_hz,
        high_cut_hz,
        train_cnt.info["sfreq"],
        filt_order=3,
        axis=1,
    ),
    train_cnt,
)
train_cnt = mne_apply(
    lambda a: exponential_running_standardize(
        a.T, factor_new=factor_new, init_block_size=init_block_size, eps=1e-4
    ).T,
    train_cnt,
)

test_cnt = test_cnt.drop_channels(["EOG-left", "EOG-central", "EOG-right"])
assert len(test_cnt.ch_names) == 22
test_cnt = mne_apply(lambda a: a * 1e6, test_cnt)
test_cnt = mne_apply(
    lambda a: bandpass_cnt(
        a, low_cut_hz, high_cut_hz, test_cnt.info["sfreq"], filt_order=3, axis=1
    ),
    test_cnt,
)
test_cnt = mne_apply(
    lambda a: exponential_running_standardize(