def lsl_channel_list(inlet): """ Reads XML description of LSL header and returns channel list Input: pylsl.StreamInlet object Returns: ch_list: [ name1, name2, ... ] """ if not type(inlet) is pylsl.StreamInlet: logger.error('lsl_channel_list(): wrong input type %s' % type(inlet)) raise TypeError root = ET.fromstring(inlet.info().as_xml()) desc = root.find('desc') ch_list = [] for ch in list(desc.find('channels')): ch_name = ch.find('label').text ch_list.append(ch_name) ''' This code may throw access violation error due to bug in pylsl.XMLElement # for some reason type(inlet) returns 'instance' type in Python 2. ch = inlet.info().desc().child('channels').first_child() ch_list = [] for k in range(inlet.info().channel_count()): ch_name = ch.child_value('label') ch_list.append(ch_name) ch = ch.next_sibling() ''' return ch_list
def log_decoding_helper(state, event_queue, amp_name=None, amp_serial=None, autostop=False): """ Helper function to run StreamReceiver object in background """ logger.info('Event acquisition subprocess started.') # wait for the start signal while state.value == 0: time.sleep(0.01) # acquire event values and returns event times and event values sr = StreamReceiver(buffer_size=0, amp_name=amp_name, amp_serial=amp_serial) tm = qc.Timer(autoreset=True) started = False while state.value == 1: chunk, ts_list = sr.acquire() if autostop: if started is True: if len(ts_list) == 0: state.value = 0 break elif len(ts_list) > 0: started = True tm.sleep_atleast(0.001) logger.info('Event acquisition subprocess finishing up ...') buffers, times = sr.get_buffer() events = buffers[:, 0] # first channel is the trigger channel event_index = np.where(events != 0)[0] event_times = times[event_index].reshape(-1).tolist() event_values = events[event_index].tolist() if len(event_times) != len(event_values): logger.error('event_times length (%d) is different from event_values length (%d)' % (len(event_times), len(event_values))) event_queue.put((event_times, event_values))
def any2fif(filename, interactive=False, outdir=None, channel_file=None): """ Generic file format converter """ p = qc.parse_path(filename) if outdir is not None: qc.make_dirs(outdir) if p.ext == 'pcl': eve_file = '%s/%s.txt' % (p.dir, p.name.replace('raw', 'eve')) if os.path.exists(eve_file): logger.info('Adding events from %s' % eve_file) else: eve_file = None pcl2fif(filename, interactive=interactive, outdir=outdir, external_event=eve_file) elif p.ext == 'eeg': eeg2fif(filename, interactive=interactive, outdir=outdir) elif p.ext in ['edf', 'bdf']: bdf2fif(filename, interactive=interactive, outdir=outdir) elif p.ext == 'gdf': gdf2fif(filename, interactive=interactive, outdir=outdir, channel_file=channel_file) elif p.ext == 'xdf': xdf2fif(filename, interactive=interactive, outdir=outdir) else: # unknown format logger.error( 'Ignored unrecognized file extension %s. It should be [.pcl | .eeg | .gdf | .bdf]' % p.ext)
def get_decoder_info(classifier): """ Get only the classifier information without connecting to a server Params ------ classifier: model file Returns ------- info dictionary object """ model = qc.load_obj(classifier) if model is None: logger.error('>> Error loading %s' % model) raise ValueError cls = model['cls'] psde = model['psde'] labels = list(cls.classes_) w_seconds = model['w_seconds'] w_frames = model['w_frames'] wstep = model['wstep'] sfreq = model['sfreq'] psd_temp = psde.transform(np.zeros((1, len(model['picks']), w_frames))) psd_shape = psd_temp.shape psd_size = psd_temp.size info = dict(labels=labels, cls=cls, psde=psde, w_seconds=w_seconds, w_frames=w_frames,\ wstep=wstep, sfreq=sfreq, psd_shape=psd_shape, psd_size=psd_size) return info
def get_timelags(epochs, wlen, wstep, downsample=1, picks=None): """ (DEPRECATED FUNCTION) Get concatenated timelag features TODO: Unit test. Input ===== epochs: input signals wlen: window length (# time points) in downsampled data wstep: window step in downsampled data downsample: downsample signal to be 1/downsample length picks: ignored for now Output ====== X: [epochs] x [windows] x [channels*freqs] y: [epochs] x [labels] """ ''' wlen = int(wlen) wstep = int(wstep) downsample = int(downsample) X_data = None y_data = None labels = epochs.events[:, -1] # every epoch must have event id epochs_data = epochs.get_data() n_channels = epochs_data.shape[1] # trim to the nearest divisible length epoch_ds_len = int(epochs_data.shape[2] / downsample) epoch_len = downsample * epoch_ds_len range_epochs = np.arange(epochs_data.shape[0]) range_channels = np.arange(epochs_data.shape[1]) range_windows = np.arange(epoch_ds_len - wlen, 0, -wstep) X_data = np.zeros((len(range_epochs), len(range_windows), wlen * n_channels)) # for each epoch for ep in range_epochs: epoch = epochs_data[ep, :, :epoch_len] ds = qc.average_every_n(epoch.reshape(-1), downsample) # flatten to 1-D, then downsample epoch_ds = ds.reshape(n_channels, -1) # recover structure to channel x samples # for each window over all channels for i in range(len(range_windows)): w = range_windows[i] X = epoch_ds[:, w:w + wlen].reshape(1, -1) # our feature vector X_data[ep, i, :] = X # fill labels y = np.empty((1, len(range_windows))) # 1 x windows y.fill(labels[ep]) if y_data is None: y_data = y else: y_data = np.concatenate((y_data, y), axis=0) return X_data, y_data ''' logger.error('This function is deprecated.') raise NotImplementedError
def rereference(raw, ref_new, ref_old=None): """ Reference to new channels. raw object is modified in-place for efficiency. raw: mne.io.RawArray ref_new: None | list of str (RawArray) | list of int (numpy array) Channel(s) to re-reference, e.g. M1, M2. Average of these channel values are substracted from all channel values. ref_old: None | str Channel to recover, assuming this channel was originally used as a reference. """ # Re-reference and recover the original reference channel values if possible if type(raw) == np.ndarray: if raw_ch_old is not None: logger.error( 'Recovering original reference channel is not yet supported for numpy arrays.' ) raise NotImplementedError if type(raw_ch_new[0]) is not int: logger.error('Channels must be integer values for numpy arrays') raise ValueError raw -= np.mean(raw[ref_new], axis=0) else: if ref_old is not None: # Add a blank (zero-valued) channel mne.io.add_reference_channels(raw, ref_old, copy=False) # Re-reference mne.io.set_eeg_reference(raw, ref_new, copy=False) return True
def move(self, dir, dx, overlay=False, barcolor=None, caption='', caption_color='W'): if barcolor is None: if dx == self.xl2: c = 'G' else: c = 'R' else: c = barcolor self.glass.fullbar_color(c) color = self.color[c] if dir == 'L': if self.pc_feedback: self.img = self.left_images[dx] if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) elif dir == 'R': if self.pc_feedback: self.img = self.right_images[dx] if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) else: logger.error('Unknown direction %s' % dir) self.put_text(caption, caption_color) self.update()
def fit_predict_thres(cls, X_train, Y_train, X_test, Y_test, cnum, label_list, ignore_thres=None, decision_thres=None): """ Any likelihood lower than a threshold is not counted as classification score Confusion matrix, accuracy and F1 score (macro average) are computed. Params ====== ignore_thres: if not None or larger than 0, likelihood values lower than ignore_thres will be ignored while computing confusion matrix. """ timer = qc.Timer() cls.fit(X_train, Y_train) assert ignore_thres is None or ignore_thres >= 0 if ignore_thres is None or ignore_thres == 0: Y_pred = cls.predict(X_test) score = skmetrics.accuracy_score(Y_test, Y_pred) cm = skmetrics.confusion_matrix(Y_test, Y_pred, label_list) f1 = skmetrics.f1_score(Y_test, Y_pred, average='macro') else: if decision_thres is not None: logger.error( 'decision threshold and ignore_thres cannot be set at the same time.' ) raise ValueError Y_pred = cls.predict_proba(X_test) Y_pred_labels = np.argmax(Y_pred, axis=1) Y_pred_maxes = np.array([x[i] for i, x in zip(Y_pred_labels, Y_pred)]) Y_index_overthres = np.where(Y_pred_maxes >= ignore_thres)[0] Y_index_underthres = np.where(Y_pred_maxes < ignore_thres)[0] Y_pred_overthres = np.array( [cls.classes_[x] for x in Y_pred_labels[Y_index_overthres]]) Y_pred_underthres = np.array( [cls.classes_[x] for x in Y_pred_labels[Y_index_underthres]]) Y_pred_underthres_count = np.array( [np.count_nonzero(Y_pred_underthres == c) for c in label_list]) Y_test_overthres = Y_test[Y_index_overthres] score = skmetrics.accuracy_score(Y_test_overthres, Y_pred_overthres) cm = skmetrics.confusion_matrix(Y_test_overthres, Y_pred_overthres, label_list) cm = np.concatenate((cm, Y_pred_underthres_count[:, np.newaxis]), axis=1) f1 = skmetrics.f1_score(Y_test_overthres, Y_pred_overthres, average='macro') logger.info('Cross-validation %d (%.3f) - %.1f sec' % (cnum, score, timer.sec())) return score, cm, f1
def set_pin(self, pin): if self.lpttype == 'SOFTWARE': logger.error('set_pin() not supported for software trigger.') return False elif self.lpttype == 'FAKE': logger.info('FAKE trigger pin %s' % pin) return True else: self.set_data(2 ** (pin - 1))
def balance_samples(X, Y, balance_type, verbose=False): if balance_type == 'OVER': """ Oversample from classes that lack samples """ label_set = np.unique(Y) max_set = [] X_balanced = np.array(X) Y_balanced = np.array(Y) # find a class with maximum number of samples for c in label_set: yl = np.where(Y == c)[0] if len(max_set) == 0 or len(yl) > max_set[1]: max_set = [c, len(yl)] for c in label_set: if c == max_set[0]: continue yl = np.where(Y == c)[0] extra_samples = max_set[1] - len(yl) extra_idx = np.random.choice(yl, extra_samples) X_balanced = np.append(X_balanced, X[extra_idx], axis=0) Y_balanced = np.append(Y_balanced, Y[extra_idx], axis=0) elif balance_type == 'UNDER': """ Undersample from classes that are excessive """ label_set = np.unique(Y) min_set = [] # find a class with minimum number of samples for c in label_set: yl = np.where(Y == c)[0] if len(min_set) == 0 or len(yl) < min_set[1]: min_set = [c, len(yl)] yl = np.where(Y == min_set[0])[0] X_balanced = np.array(X[yl]) Y_balanced = np.array(Y[yl]) for c in label_set: if c == min_set[0]: continue yl = np.where(Y == c)[0] reduced_idx = np.random.choice(yl, min_set[1]) X_balanced = np.append(X_balanced, X[reduced_idx], axis=0) Y_balanced = np.append(Y_balanced, Y[reduced_idx], axis=0) elif balance_type is None or balance_type is False: return X, Y else: logger.error('Unknown balancing type %s' % balance_type) raise ValueError logger.info_green('\nNumber of samples after %ssampling' % balance_type.lower()) for c in label_set: logger.info( '%s: %d -> %d' % (c, len(np.where(Y == c)[0]), len(np.where(Y_balanced == c)[0]))) return X_balanced, Y_balanced
def move(self, dir, dx, overlay=False, barcolor=None, caption='', caption_color='W'): if not overlay: self.draw_cue() if barcolor is None: if dx == self.xl2: c = 'G' else: c = 'R' else: c = barcolor self.glass.fullbar_color(c) color = self.color[c] if dir == 'L': if self.pc_feedback: cv2.rectangle(self.img, (self.xl1 - dx, self.yl1), (self.xl1, self.yr1), color, -1) if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) elif dir == 'U': if self.pc_feedback: cv2.rectangle(self.img, (self.xl1, self.yl1 - dx), (self.xr1, self.yl1), color, -1) if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) elif dir == 'R': if self.pc_feedback: cv2.rectangle(self.img, (self.xr1, self.yl1), (self.xr1 + dx, self.yr1), color, -1) if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) elif dir == 'D': if self.pc_feedback: cv2.rectangle(self.img, (self.xl1, self.yr1), (self.xr1, self.yr1 + dx), color, -1) if self.glass_feedback: self.glass.move_bar(dir, dx, overlay) elif dir == 'B': if self.pc_feedback: cv2.rectangle(self.img, (self.xl1 - dx, self.yl1), (self.xl1, self.yr1), color, -1) cv2.rectangle(self.img, (self.xr1, self.yl1), (self.xr1 + dx, self.yr1), color, -1) if self.glass_feedback: self.glass.move_bar('S', dx, overlay) else: logger.error('Unknown direction %s' % dir) self.put_text(caption, caption_color)
def check_connect(self): """ Check connection and automatically connect if not connected """ while not self.connected: logger.error( 'LSL server not connected yet. Trying to connect automatically.' ) self.connect() time.sleep(1)
def get_index_max(seq): """ Get the index of the maximum item in a list or dict """ if type(seq) == list: return max(range(len(seq)), key=seq.__getitem__) elif type(seq) == dict: return max(seq, key=seq.__getitem__) else: logger.error('Unsupported input %s' % type(seq)) return None
def make_dirs(dirname, delete=False): """ Recusively create directories. if delete=true, directory will be deleted first if exists. """ if os.path.exists(dirname) and delete == True: try: shutil.rmtree(dirname) except OSError: logger.error( 'Directory was not completely removed. (Perhaps a Dropbox folder?). Continuing.' ) if not os.path.exists(dirname): os.makedirs(dirname)
def load_raw(rawfile, spfilter=None, spchannels=None, events_ext=None, multiplier=1, verbose='ERROR'): """ Loads data from a fif-format file. You can convert non-fif files (.eeg, .bdf, .gdf, .pcl) to fif format. Parameters: rawfile: (absolute) data file path spfilter: 'car' | 'laplacian' | None spchannels: None | list (for CAR) | dict (for LAPLACIAN) 'car': channel indices used for CAR filtering. If None, use all channels except the trigger channel (index 0). 'laplacian': {channel:[neighbor1, neighbor2, ...], ...} *** Note *** Since PyCNBI puts trigger channel as index 0, data channel starts from index 1. events_ext: Add externally recorded events. [ [sample_index1, 0, event_value1],... ] multiplier: Multiply all values except triggers (to convert unit). Returns: raw: mne.io.RawArray object. First channel (index 0) is always trigger channel. events: mne-compatible events numpy array object (N x [frame, 0, type]) spfilter= {None | 'car' | 'laplacian'} """ if not os.path.exists(rawfile): logger.error('File %s not found' % rawfile) raise IOError if not os.path.isfile(rawfile): logger.error('%s is not a file' % rawfile) raise IOError extension = qc.parse_path(rawfile).ext assert extension in ['fif', 'fiff'], 'only fif format is supported' raw = mne.io.Raw(rawfile, preload=True, verbose=verbose) if spfilter is not None or multiplier is not 1: preprocess(raw, spatial=spfilter, spatial_ch=spchannels, multiplier=multiplier) if events_ext is not None: events = mne.read_events(events_ext) else: tch = find_event_channel(raw) if tch is not None: events = mne.find_events(raw, stim_channel=raw.ch_names[tch], shortest_event=1, uint_cast=True, consecutive='increasing') # MNE's annoying hidden cockroach: first_samp events[:, 0] -= raw.first_samp else: events = np.array([], dtype=np.int64) return raw, events
def start(self): """ Start the daemon """ if self.is_running() > 0: msg = 'Cannot start. Daemon already running. (PID' + ', '.join(['%d' % proc.pid for proc in self.procs]) + ')' logger.error(msg) return for proc in self.procs: proc.start() if self.wait_init: for running in self.running: while running.value == 0: time.sleep(0.001) logger.info(self.startmsg)
def convert2mat(filename, matfile): """ Convert to mat using MATLAB BioSig sload(). """ basename = '.'.join(filename.split('.')[:-1]) # extension= filename.split('.')[-1] matfile = basename + '.mat' if not os.path.exists(matfile): logger.info('Converting input to mat file') run = "[sig,header]=sload('%s'); save('%s.mat','sig','header');" % ( filename, basename) qc.matlab(run) if not os.path.exists(matfile): logger.error('mat file convertion error.') sys.exit()
def set_data(self, value): if self.lpttype == 'SOFTWARE': logger.error('set_data() not supported for software trigger.') return False elif self.lpttype == 'FAKE': logger.info('FAKE trigger value %s' % value) return True else: if self.lpttype == 'USB2LPT': self.lpt.setdata(value) elif self.lpttype == 'DESKTOP': self.lpt.setdata(self.portaddr, value) elif self.lpttype == 'ARDUINO': self.ser.write(bytes([value])) else: raise RuntimeError('Wrong trigger device')
def reset(self): """ Reset classifier to the initial state """ # share numpy array self.psd between processes. # to compute the shared memory size, we need to create a temporary decoder object. if self.fake == True: psd_size = None psd_shape = None psd_ctypes = None self.psd = None else: info = get_decoder_info(self.classifier) psd_size = info['psd_size'] psd_shape = info['psd_shape'][1:] # we get only the last window psd_ctypes = sharedctypes.RawArray('d', np.zeros(psd_size)) self.psd = np.frombuffer(psd_ctypes, dtype=np.float64, count=psd_size) self.probs = mp.Array('d', [1.0 / len(self.labels)] * len(self.labels)) self.probs_smooth = mp.Array('d', [1.0 / len(self.labels)] * len(self.labels)) self.pread = mp.Value('i', 1) self.t_problast = mp.Value('d', 0) self.return_psd = mp.Value('i', 0) self.procs = [] mp.freeze_support() if self.parallel: logger.error('Parallel decoding is under a rigorous test. Please do not use it for now.') raise NotImplementedError num_strides = self.parallel['num_strides'] period = self.parallel['period'] self.running = [mp.Value('i', 0)] * num_strides if num_strides > 1: stride = period / num_strides else: stride = 0 t_start = time.time() for i in range(num_strides): self.procs.append(mp.Process(target=self.daemon, args=\ [self.classifier, self.probs, self.probs_smooth, self.pread, self.t_problast,\ self.running[i], self.return_psd, psd_ctypes, self.psdlock,\ dict(t_start=(t_start+i*stride), period=period)])) else: self.running = [mp.Value('i', 0)] self.procs = [mp.Process(target=self.daemon, args=\ [self.classifier, self.probs, self.probs_smooth, self.pread, self.t_problast,\ self.running[0], self.return_psd, psd_ctypes, self.psdlock, None])]
def init(self, duration): if self.lpttype == 'SOFTWARE': logger.info('Ignoring delay parameter for software trigger.') return True elif self.lpttype == 'FAKE': return True else: self.delay = duration / 1000.0 if self.lpttype in ['DESKTOP', 'USB2LPT']: if self.lpt.init() == -1: logger.error('Connecting to LPT port failed. Check the driver status.') self.lpt = None return False self.action = False self.offtimer = threading.Timer(self.delay, self.signal_off) return True
def check_config(cfg): critical_vars = { 'COMMON': [ 'TRIGGER_DEVICE', 'TRIGGER_FILE', 'SCREEN_SIZE', 'DIRECTIONS', 'DIR_RANDOM', 'TRIALS_EACH' ], 'TIMINGS': [ 'INIT', 'GAP', 'CUE', 'READY', 'READY_RANDOMIZE', 'DIR', 'DIR_RANDOMIZE' ] } optional_vars = { 'FEEDBACK_TYPE': 'BAR', 'FEEDBACK_IMAGE_PATH': None, 'SCREEN_POS': (0, 0), 'DIR_RANDOM': True, 'GLASS_USE': False, 'TRIAL_PAUSE': False, 'REFRESH_RATE': 30 } for key in critical_vars['COMMON']: if not hasattr(cfg, key): raise RuntimeError('%s is a required parameter' % key) if not hasattr(cfg, 'TIMINGS'): logger.error('"TIMINGS" not defined in config.') raise RuntimeError for v in critical_vars['TIMINGS']: if v not in cfg.TIMINGS: logger.error('%s not defined in config.' % v) raise RuntimeError for key in optional_vars: if not hasattr(cfg, key): setattr(cfg, key, optional_vars[key]) logger.warning('Setting undefined %s=%s' % (key, optional[key])) if getattr(cfg, 'TRIGGER_DEVICE') == None: logger.warning( 'The trigger device is set to None! No events will be saved.') raise RuntimeError( 'The trigger device is set to None! No events will be saved.')
def bdf2fif(filename, interactive=False, outdir=None): """ EDF or BioSemi BDF format """ # convert to mat using MATLAB (MNE's edf reader has an offset bug) fdir, fname, fext = qc.parse_path_list(filename) if outdir is None: outdir = fdir elif outdir[-1] != '/': outdir += '/' fiffile = outdir + fname + '.fif' raw = mne.io.read_raw_edf(filename, preload=True) # process event channel if raw.info['chs'][-1]['ch_name'] != 'STI 014': logger.error( "The last channel (%s) doesn't seem to be an event channel. Entering debugging mode." % raw.info['chs'][-1]['ch_name']) pdb.set_trace() raw.info['chs'][-1]['ch_name'] = 'TRIGGER' events = mne.find_events(raw, stim_channel='TRIGGER', shortest_event=1, uint_cast=True, consecutive=True) events[:, 2] -= events[:, 1] # set offset to 0 events[:, 1] = 0 # move the event channel to index 0 (for consistency) raw._data = np.concatenate( (raw._data[-1, :].reshape(1, -1), raw._data[:-1, :])) raw._data[0] *= 0 # init the event channel raw.info['chs'] = [raw.info['chs'][-1]] + raw.info['chs'][:-1] # add events raw.add_events(events, 'TRIGGER') # save and close raw.save(fiffile, verbose=False, overwrite=True, fmt='double') logger.info('Saved to %s' % fiffile) saveChannels2txt(outdir, ch_names)
def check_config(cfg): """ Ensure that the config file contains the parameters """ critical_vars = { 'COMMON': ['DATA_PATH'], } optional_vars = {} for key in critical_vars['COMMON']: if not hasattr(cfg, key): logger.error('%s is a required parameter' % key) raise RuntimeError for key in optional_vars: if not hasattr(cfg, key): setattr(cfg, key, optional_vars[key]) logger.warning('Setting undefined parameter %s=%s' % (key, getattr(cfg, key))) return cfg
def print_c(msg, color=None, end='\n'): """ Colored print using colorama. Fullset: https://pypi.python.org/pypi/colorama Fore: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. Back: BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE, RESET. Style: DIM, NORMAL, BRIGHT, RESET_ALL TODO: Make it using *args and **kwargs to support fully featured print(). """ if color is None: print(str(msg), end=end) return color = str(color) if len(color) != 1: raise ValueError( 'Color parameter must be a single color code, not %s' % type(color)) if color.upper() == 'B': c = colorama.Fore.BLUE elif color.upper() == 'R': c = colorama.Fore.RED elif color.upper() == 'G': c = colorama.Fore.GREEN elif color.upper() == 'Y': c = colorama.Fore.YELLOW elif color.upper() == 'W': c = colorama.Fore.WHITE elif color.upper() == 'C': c = colorama.Fore.CYAN else: logger.error('print_c(): Unknown color code %s' % color) raise ValueError print(colorama.Style.BRIGHT + c + str(msg) + colorama.Style.RESET_ALL, end=end)
def run(record_dir, amp_name, amp_serial, eeg_only=False): logger.info('\nOutput directory: %s' % (record_dir)) # spawn the recorder as a child process logger.info('\n>> Press Enter to start recording.') key = input() state = mp.Value('i', 1) proc = mp.Process(target=record, args=[state, amp_name, amp_serial, record_dir, eeg_only]) proc.start() # clean up time.sleep(1) # required on some Python distribution input() state.value = 0 logger.info('(main) Waiting for recorder process to finish.') proc.join(10) if proc.is_alive(): logger.error( 'Recorder process not finishing. Are you running from Spyder?') logger.error('Dropping into a shell') qc.shell() sys.stdout.flush() logger.info('Recording finished.')
def __init__(self, window_size=1, buffer_size=1, amp_serial=None, eeg_only=False, amp_name=None): """ Params: window_size (in seconds): keep the latest window_size seconds of the buffer. buffer_size (in seconds): 1-day is the maximum size. Large buffer may lead to a delay if not pulled frequently. amp_name: connect to a server named 'amp_name'. None: no constraint. amp_serial: connect to a server with serial number 'amp_serial'. None: no constraint. eeg_only: ignore non-EEG servers """ _MAX_BUFFER_SIZE = 86400 # max buffer size allowed by StreamReceiver (24 hours) _MAX_PYLSL_STREAM_BUFSIZE = 360 # max buffer size for pylsl.StreamInlet if window_size <= 0: logger.error('Wrong window_size %d.' % window_size) raise ValueError() self.winsec = window_size if buffer_size == 0: buffer_size = _MAX_BUFFER_SIZE elif buffer_size < 0 or buffer_size > _MAX_BUFFER_SIZE: logger.error('Improper buffer size %.1f. Setting to %d.' % (buffer_size, _MAX_BUFFER_SIZE)) buffer_size = _MAX_BUFFER_SIZE elif buffer_size < self.winsec: logger.error( 'Buffer size %.1f is smaller than window size. Setting to %.1f.' % (buffer_size, self.winsec)) buffer_size = self.winsec self.bufsec = buffer_size self.bufsize = 0 # to be calculated using sampling rate self.stream_bufsec = int( math.ceil(min(_MAX_PYLSL_STREAM_BUFSIZE, self.bufsec))) self.stream_bufsize = 0 # to be calculated using sampling rate self.amp_serial = amp_serial self.eeg_only = eeg_only self.amp_name = amp_name self.tr_channel = None # trigger indx used by StreamReceiver class self.eeg_channels = [] # signal indx used by StreamReceiver class self._lsl_tr_channel = None # raw trigger indx in pylsl.pull_chunk() self._lsl_eeg_channels = [] # raw signal indx in pylsl.pull_chunk() self.ready = False # False until the buffer is filled for the first time self.connected = False self.buffers = [] self.timestamps = [] self.watchdog = qc.Timer() self.multiplier = 1 # 10**6 for uV unit (automatically updated for openvibe servers) self.connect()
def load_multi(src, spfilter=None, spchannels=None, multiplier=1): """ Load multiple data files and concatenate them into a single series - Assumes all files have the same sampling rate and channel order. - Event locations are updated accordingly with new offset. @params: src: directory or list of files. spfilter: apply spatial filter while loading. spchannels: list of channel names to apply spatial filter. multiplier: to change units for better numerical stability. See load_raw() for more low-level details. """ if type(src) == str: if not os.path.isdir(src): logger.error('%s is not a directory or does not exist.' % src) raise IOError flist = [] for f in qc.get_file_list(src): if qc.parse_path_list(f)[2] == 'fif': flist.append(f) elif type(src) in [list, tuple]: flist = src else: logger.error('Unknown input type %s' % type(src)) raise TypeError if len(flist) == 0: logger.error('load_multi(): No fif files found in %s.' % src) raise RuntimeError elif len(flist) == 1: return load_raw(flist[0], spfilter=spfilter, spchannels=spchannels, multiplier=multiplier) # load raw files rawlist = [] for f in flist: logger.info('Loading %s' % f) raw, _ = load_raw(f, spfilter=spfilter, spchannels=spchannels, multiplier=multiplier) rawlist.append(raw) # concatenate signals signals = None for raw in rawlist: if signals is None: signals = raw._data else: signals = np.concatenate((signals, raw._data), axis=1) # append samples # create a concatenated raw object and update channel names raw = rawlist[0] trigch = find_event_channel(raw) ch_types = ['eeg'] * len(raw.ch_names) if trigch is not None: ch_types[trigch] = 'stim' info = mne.create_info(raw.ch_names, raw.info['sfreq'], ch_types) raw_merged = mne.io.RawArray(signals, info) # re-calculate event positions events = mne.find_events(raw_merged, stim_channel='TRIGGER', shortest_event=1, uint_cast=True, consecutive='increasing', output='onset', initial_event=True) return raw_merged, events
def preprocess(raw, sfreq=None, spatial=None, spatial_ch=None, spectral=None, spectral_ch=None, notch=None, notch_ch=None, multiplier=1, ch_names=None, rereference=None, decim=None, n_jobs=1): """ Apply spatial, spectral, notch filters and convert unit. raw is modified in-place. Input ------ raw: mne.io.Raw | mne.io.RawArray | mne.Epochs | numpy.array (n_channels x n_samples) numpy.array type assumes the data has only pure EEG channnels without event channels sfreq: required only if raw is numpy array. spatial: None | 'car' | 'laplacian' Spatial filter type. spatial_ch: None | list (for CAR) | dict (for LAPLACIAN) Reference channels for spatial filtering. May contain channel names. 'car': channel indices used for CAR filtering. If None, use all channels except the trigger channel (index 0). 'laplacian': {channel:[neighbor1, neighbor2, ...], ...} *** Note *** Since PyCNBI puts trigger channel as index 0, data channel starts from index 1. spectral: None | [l_freq, h_freq] Spectral filter. if l_freq is None: lowpass filter is applied. if h_freq is None: highpass filter is applied. if l_freq < h_freq: bandpass filter is applied. if l_freq > h_freq: band-stop filter is applied. spectral_ch: None | list Channel picks for spectra filtering. May contain channel names. notch: None | float | list of frequency in floats Notch filter. notch_ch: None | list Channel picks for notch filtering. May contain channel names. multiplier: float If not 1, multiply data values excluding trigger values. ch_names: None | list If raw is numpy array and channel picks are list of strings, ch_names will be used as a look-up table to convert channel picks to channel numbers. rereference: Not supported yet. decim: None | int Apply low-pass filter and decimate (downsample). sfreq must be given. Ignored if 1. Output ------ Same input data structure. Note: To save computation time, input data may be modified in-place. TODO: Add an option to disable in-place modification. """ # Check datatype if type(raw) == np.ndarray: # Numpy array: assume we don't have event channel data = raw assert sfreq is not None and sfreq > 0, 'Wrong sfreq value.' assert 2 <= len( data.shape ) <= 3, 'Unknown data shape. The dimension must be 2 or 3.' if len(data.shape) == 3: n_channels = data.shape[1] elif len(data.shape) == 2: n_channels = data.shape[0] eeg_channels = list(range(n_channels)) if decim is not None and decim != 1: if sfreq is None: logger.error('Decimation cannot be applied if sfreq is None.') raise ValueError else: # MNE Raw object: exclude event channel ch_names = raw.ch_names data = raw._data sfreq = raw.info['sfreq'] assert 2 <= len( data.shape ) <= 3, 'Unknown data shape. The dimension must be 2 or 3.' if len(data.shape) == 3: # assert type(raw) is mne.epochs.Epochs n_channels = data.shape[1] elif len(data.shape) == 2: n_channels = data.shape[0] eeg_channels = list(range(n_channels)) tch = find_event_channel(raw) if tch is None: logger.warning('No trigger channel found. Using all channels.') else: tch_name = ch_names[tch] eeg_channels.pop(tch) # Re-reference channels if rereference is not None: logger.error('re-referencing not implemented yet. Sorry.') raise NotImplementedError # Do unit conversion if multiplier != 1: data[eeg_channels] *= multiplier # Apply spatial filter if spatial is None: pass elif spatial == 'car': if spatial_ch is None: spatial_ch = eeg_channels if type(spatial_ch[0]) == str: assert ch_names is not None, 'preprocess(): ch_names must not be None' spatial_ch_i = [ch_names.index(c) for c in spatial_ch] else: spatial_ch_i = spatial_ch if len(spatial_ch_i) > 1: if len(data.shape) == 2: data[spatial_ch_i] -= np.mean(data[spatial_ch_i], axis=0) elif len(data.shape) == 3: means = np.mean(data[:, spatial_ch_i, :], axis=1) data[:, spatial_ch_i, :] -= means[:, np.newaxis, :] else: logger.error('Unknown data shape %s' % str(data.shape)) raise ValueError elif spatial == 'laplacian': if type(spatial_ch) is not dict: logger.error( 'preprocess(): For Lapcacian, spatial_ch must be of form {CHANNEL:[NEIGHBORS], ...}' ) raise TypeError if type(spatial_ch.keys()[0]) == str: spatial_ch_i = {} for c in spatial_ch: ref_ch = ch_names.index(c) spatial_ch_i[ref_ch] = [ ch_names.index(n) for n in spatial_ch[c] ] else: spatial_ch_i = spatial_ch if len(spatial_ch_i) > 1: rawcopy = data.copy() for src in spatial_ch: nei = spatial_ch[src] if len(data.shape) == 2: data[src] = rawcopy[src] - np.mean(rawcopy[nei], axis=0) elif len(data.shape) == 3: data[:, src, :] = rawcopy[:, src, :] - np.mean( rawcopy[:, nei, :], axis=1) else: logger.error('preprocess(): Unknown data shape %s' % str(data.shape)) raise ValueError else: logger.error('preprocess(): Unknown spatial filter %s' % spatial) raise ValueError # Downsample if decim is not None and decim != 1: if type(raw) == np.ndarray: data = mne.filter.resample(data, down=decim, npad='auto', window='boxcar', n_jobs=1) else: # resample() of Raw* and Epochs object internally calls mne.filter.resample() raw = raw.resample(raw.info['sfreq'] / decim, npad='auto', window='boxcar', n_jobs=1) data = raw._data sfreq /= decim # Apply spectral filter if spectral is not None: if spectral_ch is None: spectral_ch = eeg_channels if type(spectral_ch[0]) == str: assert ch_names is not None, 'preprocess(): ch_names must not be None' spectral_ch_i = [ch_names.index(c) for c in spectral_ch] else: spectral_ch_i = spectral_ch # fir_design='firwin' is especially important for ICA analysis. See: # http://martinos.org/mne/dev/generated/mne.preprocessing.ICA.html?highlight=score_sources#mne.preprocessing.ICA.score_sources mne.filter.filter_data(data, sfreq, spectral[0], spectral[1], picks=spectral_ch_i, filter_length='auto', l_trans_bandwidth='auto', h_trans_bandwidth='auto', n_jobs=n_jobs, method='fir', iir_params=None, copy=False, phase='zero', fir_window='hamming', fir_design='firwin', verbose='ERROR') # Apply notch filter if notch is not None: if notch_ch is None: notch_ch = eeg_channels if type(notch_ch[0]) == str: assert ch_names is not None, 'preprocess(): ch_names must not be None' notch_ch_i = [ch_names.index(c) for c in notch_ch] else: notch_ch_i = notch_ch mne.filter.notch_filter(data, Fs=sfreq, freqs=notch, notch_widths=5, picks=notch_ch_i, method='fft', n_jobs=n_jobs, copy=False) if type(raw) == np.ndarray: raw = data return raw
def train_decoder(cfg, featdata, feat_file=None): """ Train the final decoder using all data """ # Init a classifier selected_classifier = cfg.CLASSIFIER['selected'] if selected_classifier == 'GB': cls = GradientBoostingClassifier( loss='deviance', learning_rate=cfg.CLASSIFIER[selected_classifier]['learning_rate'], n_estimators=cfg.CLASSIFIER[selected_classifier]['trees'], subsample=1.0, max_depth=cfg.CLASSIFIER[selected_classifier]['depth'], random_state=cfg.CLASSIFIER[selected_classifier]['seed'], max_features='sqrt', verbose=0, warm_start=False, presort='auto') elif selected_classifier == 'XGB': cls = XGBClassifier( loss='deviance', learning_rate=cfg.CLASSIFIER[selected_classifier]['learning_rate'], n_estimators=cfg.CLASSIFIER[selected_classifier]['trees'], subsample=1.0, max_depth=cfg.CLASSIFIER[selected_classifier]['depth'], random_state=cfg.GB['seed'], max_features='sqrt', verbose=0, warm_start=False, presort='auto') elif selected_classifier == 'RF': cls = RandomForestClassifier( n_estimators=cfg.CLASSIFIER[selected_classifier]['trees'], max_features='auto', max_depth=cfg.CLASSIFIER[selected_classifier]['depth'], n_jobs=cfg.N_JOBS, random_state=cfg.CLASSIFIER[selected_classifier]['seed'], oob_score=False, class_weight='balanced_subsample') elif selected_classifier == 'LDA': cls = LDA() elif selected_classifier == 'rLDA': cls = rLDA(cfg.CLASSIFIER[selected_classifier][r_coeff]) else: logger.error('Unknown classifier %s' % selected_classifier) raise ValueError # Setup features X_data = featdata['X_data'] Y_data = featdata['Y_data'] wlen = featdata['wlen'] if cfg.FEATURES['PSD']['wlen'] is None: cfg.FEATURES['PSD']['wlen'] = wlen w_frames = featdata['w_frames'] ch_names = featdata['ch_names'] X_data_merged = np.concatenate(X_data) Y_data_merged = np.concatenate(Y_data) if cfg.CV['BALANCE_SAMPLES']: X_data_merged, Y_data_merged = balance_samples( X_data_merged, Y_data_merged, cfg.CV['BALANCE_SAMPLES'], verbose=True) # Start training the decoder logger.info_green('Training the decoder') timer = qc.Timer() cls.n_jobs = cfg.N_JOBS cls.fit(X_data_merged, Y_data_merged) logger.info('Trained %d samples x %d dimension in %.1f sec' %\ (X_data_merged.shape[0], X_data_merged.shape[1], timer.sec())) cls.n_jobs = 1 # always set n_jobs=1 for testing # Export the decoder classes = {c: cfg.tdef.by_value[c] for c in np.unique(Y_data)} if cfg.FEATURES['selected'] == 'PSD': data = dict(cls=cls, ch_names=ch_names, psde=featdata['psde'], sfreq=featdata['sfreq'], picks=featdata['picks'], classes=classes, epochs=cfg.EPOCH, w_frames=w_frames, w_seconds=cfg.FEATURES['PSD']['wlen'], wstep=cfg.FEATURES['PSD']['wstep'], spatial=cfg.SP_FILTER, spatial_ch=featdata['picks'], spectral=cfg.TP_FILTER[cfg.TP_FILTER['selected']], spectral_ch=featdata['picks'], notch=cfg.NOTCH_FILTER[cfg.NOTCH_FILTER['selected']], notch_ch=featdata['picks'], multiplier=cfg.MULTIPLIER, ref_ch=cfg.REREFERENCE[cfg.REREFERENCE['selected']], decim=cfg.FEATURES['PSD']['decim']) clsfile = '%s/classifier/classifier-%s.pkl' % (cfg.DATA_PATH, platform.architecture()[0]) qc.make_dirs('%s/classifier' % cfg.DATA_PATH) qc.save_obj(clsfile, data) logger.info('Decoder saved to %s' % clsfile) # Reverse-lookup frequency from FFT fq = 0 if type(cfg.FEATURES['PSD']['wlen']) == list: fq_res = 1.0 / cfg.FEATURES['PSD']['wlen'][0] else: fq_res = 1.0 / cfg.FEATURES['PSD']['wlen'] fqlist = [] while fq <= cfg.FEATURES['PSD']['fmax']: if fq >= cfg.FEATURES['PSD']['fmin']: fqlist.append(fq) fq += fq_res # Show top distinctive features if cfg.FEATURES['selected'] == 'PSD': logger.info_green('Good features ordered by importance') if selected_classifier in ['RF', 'GB', 'XGB']: keys, values = qc.sort_by_value(list(cls.feature_importances_), rev=True) elif selected_classifier in ['LDA', 'rLDA']: keys, values = qc.sort_by_value(cls.w, rev=True) keys = np.array(keys) values = np.array(values) if cfg.EXPORT_GOOD_FEATURES: if feat_file is None: gfout = open('%s/classifier/good_features.txt' % cfg.DATA_PATH, 'w') else: gfout = open(feat_file, 'w') if type(wlen) is not list: ch_names = [ch_names[c] for c in featdata['picks']] else: ch_names = [] for w in range(len(wlen)): for c in featdata['picks']: ch_names.append('w%d-%s' % (w, ch_names[c])) chlist, hzlist = features.feature2chz(keys, fqlist, ch_names=ch_names) valnorm = values[:cfg.FEAT_TOPN].copy() valsum = np.sum(valnorm) if valsum == 0: valsum = 1 valnorm = valnorm / valsum * 100.0 # show top-N features for i, (ch, hz) in enumerate(zip(chlist, hzlist)): if i >= cfg.FEAT_TOPN: break txt = '%-3s %5.1f Hz normalized importance %-6s raw importance %-6s feature %-5d' %\ (ch, hz, '%.2f%%' % valnorm[i], '%.2f%%' % (values[i] * 100.0), keys[i]) logger.info(txt) if cfg.EXPORT_GOOD_FEATURES: gfout.write('Importance(%) Channel Frequency Index\n') for i, (ch, hz) in enumerate(zip(chlist, hzlist)): gfout.write('%.3f\t%s\t%s\t%d\n' % (values[i] * 100.0, ch, hz, keys[i])) gfout.close()
def check_config(cfg): critical_vars = { 'COMMON': [ 'TRIGGER_FILE', 'TRIGGER_DEF', 'EPOCH', 'DATA_PATH', 'PICKED_CHANNELS', 'SP_FILTER', 'SP_CHANNELS', 'TP_FILTER', 'NOTCH_FILTER', 'FEATURES', 'CLASSIFIER', 'CV_PERFORM' ], 'RF': ['trees', 'depth', 'seed'], 'GB': ['trees', 'learning_rate', 'depth', 'seed'], 'LDA': [], 'rLDA': ['r_coeff'], 'StratifiedShuffleSplit': ['test_ratio', 'folds', 'seed', 'export_result'], 'LeaveOneOut': ['export_result'] } # optional variables with default values optional_vars = { 'MULTIPLIER': 1, 'EXPORT_GOOD_FEATURES': False, 'FEAT_TOPN': 10, 'EXPORT_CLS': False, 'REREFERENCE': None, 'N_JOBS': None, 'EXCLUDED_CHANNELS': None, 'LOAD_EVENTS': None, 'CV': { 'IGNORE_THRES': None, 'DECISION_THRES': None, 'BALANCE_SAMPLES': False }, } for v in critical_vars['COMMON']: if not hasattr(cfg, v): logger.error('%s not defined in config.' % v) raise RuntimeError for key in optional_vars: if not hasattr(cfg, key): setattr(cfg, key, optional_vars[key]) logger.warning('Setting undefined parameter %s=%s' % (key, getattr(cfg, key))) if 'decim' not in cfg.FEATURES['PSD']: cfg.FEATURES['PSD']['decim'] = 1 # classifier parameters check selected_classifier = cfg.CLASSIFIER[cfg.CLASSIFIER['selected']] if selected_classifier == 'RF': if 'RF' not in cfg.CLASSIFIER: logger.error('"RF" not defined in config.') raise RuntimeError for v in critical_vars['RF']: if v not in cfg.CLASSIFIER['RF']: logger.error('%s not defined in config.' % v) raise RuntimeError elif selected_classifier == 'GB' or selected_classifier == 'XGB': if 'GB' not in cfg.CLASSIFIER: logger.error('"GB" not defined in config.') raise RuntimeError for v in critical_vars['GB']: if v not in cfg.CLASSIFIER[selected_classifier]: logger.error('%s not defined in config.' % v) raise RuntimeError elif selected_classifier == 'rLDA': if 'rLDA' not in cfg.CLASSIFIER: logger.error('"rLDA" not defined in config.') raise RuntimeError for v in critical_vars['rLDA']: if v not in cfg.CLASSIFIER['rLDA']: logger.error('%s not defined in config.' % v) raise RuntimeError cv_selected = cfg.CV_PERFORM['selected'] if cfg.CV_PERFORM[cv_selected] is not None: if cv_selected == 'StratifiedShuffleSplit': if 'StratifiedShuffleSplit' not in cfg.CV_PERFORM: logger.error('"StratifiedShuffleSplit" not defined in config.') raise RuntimeError for v in critical_vars['StratifiedShuffleSplit']: if v not in cfg.CV_PERFORM[cv_selected]: logger.error('%s not defined in config.' % v) raise RuntimeError elif cv_selected == 'LeaveOneOut': if 'LeaveOneOut' not in cfg.CV_PERFORM: logger.error('"LeaveOneOut" not defined in config.') raise RuntimeError for v in critical_vars['LeaveOneOut']: if v not in cfg.CV_PERFORM[cv_selected]: logger.error('%s not defined in config.' % v) raise RuntimeError if cfg.N_JOBS is None: cfg.N_JOBS = mp.cpu_count() return cfg