def bci2000chain(datfile, chain='SpectralSignalProcessing', parms=(), dims='auto', start=None, duration=None, verbose=False, keep=False, binpath=None, **kwargs): """ This is a provisional Python equivalent for tools/matlab/bci2000chain.m Excuse the relative paths - this example works if you're currently working in the root directory of the BCI2000 distro: bci2000chain(datfile='data/samplefiles/eeg3_2.dat', chain='TransmissionFilter|SpatialFilter|ARFilter', binpath='tools/cmdline', parms=['tools/matlab/ExampleParameters.prm'], SpatialFilterType=3) TODO: documentation For clues, see http://www.bci2000.org/wiki/index.php/User_Reference:Matlab_Tools and matlab documentation in bci2000chain.m Most arguments are like the flags in the Matlab equivalent. mutatis to a large extent mutandis. Note that for now there is no global way of managing the system path. Either add the tools/cmdline directory to your $PATH variable before starting Python, or supply its location every time while calling, in the <binpath> argument. TODO: test on Windoze """### if isinstance(chain, basestring): if chain.lower() == 'SpectralSignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|SpectralEstimator|LinearClassifier|LPFilter|ExpressionFilter|Normalizer' elif chain.lower() == 'ARSignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|ARFilter|LinearClassifier|LPFilter|ExpressionFilter|Normalizer' elif chain.lower() == 'P3SignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|P3TemporalFilter|LinearClassifier' chain = chain.split('|') chain = [c.strip() for c in chain if len(c.strip())] if len(chain) == 0: print('WARNING: chain is empty') if start != None and len(str(start).strip()): start = ' --start=' + str(start).replace(' ', '') else: start = '' if duration != None and len(str(duration).strip()): duration = ' --duration=' + str(duration).replace(' ', '') else: duration = '' if dims == None or str(dims).lower() == 'auto': dims = 0 if dims not in (0, 2, 3): raise ValueError("dims must be 2, 3 or 'auto'") out = SigTools.sstruct() err = '' cmd = '' binaries = [] tmpdir = tempfile.mkdtemp(prefix='bci2000chain_') tmpdatfile = os.path.join(tmpdir, 'in.dat') prmfile_in = os.path.join(tmpdir, 'in.prm') prmfile_out = os.path.join(tmpdir, 'out.prm') matfile = os.path.join(tmpdir, 'out.mat') bcifile = os.path.join(tmpdir, 'out.bci') shfile = os.path.join(tmpdir, 'go.bat') logfile = os.path.join(tmpdir, 'log.txt') if not isinstance(datfile, basestring): raise ValueError('datfile must be a filename') # TODO: if datfile contains the appropriate info, use some create_bcidat equivalent and do datfile = tmpdatfile if not os.path.isfile(datfile): raise IOError('file not found: %s' % datfile) mappings = { '$DATFILE': datfile, '$PRMFILE_IN': prmfile_in, '$PRMFILE_OUT': prmfile_out, '$MATFILE': matfile, '$BCIFILE': bcifile, '$SHFILE': shfile, '$LOGFILE': logfile, } def exe(name): if binpath: return os.path.join(binpath, name) else: return name if parms == None: parms = [] if isinstance(parms, tuple): parms = list(parms) if not isinstance(parms, list): parms = [parms] else: parms = list(parms) if len(kwargs): parms.append(kwargs) if parms == None or len(parms) == 0: cmd += exe('bci_dat2stream') + start + duration + ' < "$DATFILE"' binaries.append('bci_dat2stream') else: if verbose: print '# writing custom parameter file %s' % prmfile_in parms = make_bciprm(datfile, parms, '>', prmfile_in) if dat2stream_has_p_flag: cmd += exe('bci_dat2stream') + ' -p$PRMFILE_IN' + start + duration + ' < "$DATFILE"' # new-style bci_dat2stream with -p option binaries.append('bci_dat2stream') else: if len(start) or len(duration): raise ValueError('old versionsof bci_dat2stream have no --start or --duration option') cmd += '(' # old-style bci_dat2stream with no -p option cmd += exe('bci_prm2stream') + ' < $PRMFILE_IN' cmd += '&& ' + exe('bci_dat2stream') + ' --transmit-sd < $DATFILE' cmd += ')' binaries.append('bci_dat2stream') binaries.append('bci_prm2stream') for c in chain: cmd += ' | ' + exe(c) binaries.append(c) if stream2mat_saves_parms: cmd += ' | ' + exe('bci_stream2mat') + ' > $MATFILE' # new-style bci_stream2mat with Parms output binaries.append('bci_stream2mat') else: cmd += ' > $BCIFILE' cmd += ' && ' + exe('bci_stream2mat') + ' < $BCIFILE > $MATFILE' cmd += ' && ' + exe('bci_stream2prm') + ' < $BCIFILE > $PRMFILE_OUT' # old-style bci_stream2mat without Parms output binaries.append('bci_stream2mat') binaries.append('bci_stream2prm') for k,v in mappings.items(): cmd = cmd.replace(k, v) win = sys.platform.lower().startswith('win') if win: shebang = '' else: shebang = '#!/bin/sh\n\n' open(shfile, 'wt').write(shebang+cmd+'\n') if not win: os.chmod(shfile, 484) # rwxr--r-- def tidytext(x): return x.strip().replace('\r\n', '\n').replace('\r', '\n') def getstatusoutput(cmd): pipe = os.popen(cmd + ' 2>&1', 'r') # TODO: does this work on Windows? could always make use of logfile here if not output = pipe.read() status = pipe.close() if status == None: status = 0 return status,tidytext(output) def getoutput(cmd): return getstatusoutput(cmd)[1] if verbose: print '# querying version information' binaries = SigTools.sstruct([(bin, getoutput(exe(bin) + ' --version').replace('\n', ' ') ) for bin in binaries]) if verbose: print cmd t0 = time.time() failed,output = getstatusoutput(shfile) chaintime = time.time() - t0 failsig = 'Configuration Error: ' if failsig in output: failed = 1 printable_output = output printable_lines = output.split('\n') maxlines = 10 if len(printable_lines) > maxlines: printable_output = '\n'.join(printable_lines[:maxlines] + ['[%d more lines omitted]' % (len(printable_lines) - maxlines)]) if failed: if verbose: err = 'system call failed:\n' + printable_output # cmd has already been printed, so don't clutter things further else: err = 'system call failed:\n%s\n%s' % (cmd, printable_output) if err == '': if verbose: print '# loading %s' % matfile try: mat = SigTools.loadmat(matfile) except: err = "The chain must have failed: could not load %s\nShell output was as follows:\n%s" % (matfile, printable_output) else: if 'Data' not in mat: err = "The chain must have failed: no 'Data' variable found in %s\nShell output was as follows:\n%s" % (matfile, printable_output) if 'Index' not in mat: err = "The chain must have failed: no 'Index' variable found in %s\nShell output was as follows:\n%s" % (matfile, printable_output) if err == '': out.FileName = datfile if stream2mat_saves_parms: if verbose: print '# decoding parameters loaded from the mat-file' parms = make_bciprm(mat.Parms[0]) else: if verbose: print '# reading output parameter file' + prmfile_out parms = ParmList(prmfile_out) # if you get an error that prmfile_out does not exist, recompile your bci_dat2stream and bci_stream2mat binaries from up-to-date sources, and ensure that dat2stream_has_p_flag and stream2mat_saves_parms, at the top of this file, are both set to 1 out.DateStr = read_bcidate(parms, 'ISO') out.DateNum = read_bcidate(parms) out.FilterChain = chain out.ToolVersions = binaries out.ShellInput = cmd out.ShellOutput = output out.ChainTime = chaintime out.ChainSpeedFactor = None out.Megabytes = None out.Parms = parms out.Parms.sort() mat.Index = mat.Index[0,0] sigind = mat.Index.Signal - 1 # indices vary across channels fastest, then elements nChannels,nElements = sigind.shape nBlocks = mat.Data.shape[1] out.Blocks = nBlocks out.BlocksPerSecond = float(parms.SamplingRate.ScaledValue) / float(parms.SampleBlockSize.ScaledValue) out.SecondsPerBlock = float(parms.SampleBlockSize.ScaledValue) / float(parms.SamplingRate.ScaledValue) out.ChainSpeedFactor = float(out.Blocks * out.SecondsPerBlock) / float(out.ChainTime) def unnumpify(x): while isinstance(x, numpy.ndarray) and x.size == 1: x = x[0] if isinstance(x, (numpy.ndarray,tuple,list)): x = [unnumpify(xi) for xi in x] return x out.Channels = nChannels out.ChannelLabels = unnumpify(mat.get('ChannelLabels', [])) try: out.ChannelSet = SigTools.ChannelSet(out.ChannelLabels) except: out.ChannelSet = None out.Elements = nElements out.ElementLabels = unnumpify(mat.get('ElementLabels', [])) out.ElementValues = numpy.ravel(mat.get('ElementValues', [])) out.ElementUnit = unnumpify(mat.get('ElementUnit', None)) out.ElementRate = out.BlocksPerSecond * out.Elements out.Time = out.SecondsPerBlock * numpy.arange(0, nBlocks) out.FullTime = out.Time out.FullElementValues = out.ElementValues # Why does sigind have to be transposed before vectorizing to achieve the same result as sigind(:) WITHOUT a transpose in Matlab? # This will probably forever be one of the many deep mysteries of numpy dimensionality handling out.Signal = mat.Data[sigind.T.ravel(), :] # nChannels*nElements - by - nBlocks out.Signal = out.Signal + 0.0 # make a contiguous copy def seconds(s): # -1 means "no information", 0 means "not units of time", >0 means the scaling factor if getattr(s, 'ElementUnit', None) in ('',None): return -1 s = s.ElementUnit if s.endswith('seconds'): s = s[:-6] elif s.endswith('second'): s = s[:-5] elif s.endswith('sec'): s = s[:-2] if s.endswith('s'): return { 'ps': 1e-12, 'ns': 1e-9, 'us': 1e-6, 'mus': 1e-6, 'ms': 1e-3, 's' : 1e+0, 'ks': 1e+3, 'Ms': 1e+6, 'Gs': 1e+9, 'Ts': 1e+12,}.get(s, 0) else: return 0 if dims == 0: # dimensionality has not been specified explicitly: so guess, based on ElementUnit and/or filter name # 3-dimensional output makes more sense than continuous 2-D whenever "elements" can't just be concatenated into an unbroken time-stream if len(chain): lastfilter = chain[-1].lower() else: lastfilter = '' if lastfilter == 'p3temporalfilter': dims = 3 else: factor = seconds(out) if factor > 0: # units of time. TODO: could detect whether the out.ElementValues*factor are (close enough to) contiguous from block to block; then p3temporalfilter wouldn't have to be a special case above dims = 2 elif factor == 0: # not units of time: use 3D by default dims = 3 elif lastfilter in ['p3temporalfilter', 'arfilter', 'fftfilter', 'coherencefilter', 'coherencefftfilter']: # no ElementUnit info? guess based on filter name dims = 3 else: dims = 2 if dims == 3: out.Signal = numpy.reshape(out.Signal, (nChannels, nElements, nBlocks), order='F') # nChannels - by - nElements - by - nBlocks out.Signal = numpy.transpose(out.Signal, (2,0,1)) # nBlocks - by - nChannels - by - nElements out.Signal = out.Signal + 0.0 # make a contiguous copy elif dims == 2: out.FullTime = numpy.repeat(out.Time, nElements) factor = seconds(out) if len(out.ElementValues): out.FullElementValues = numpy.tile(out.ElementValues, nBlocks) if factor > 0: out.FullTime = out.FullTime + out.FullElementValues * factor out.Signal = numpy.reshape(out.Signal, (nChannels, nElements*nBlocks), order='F') # nChannels - by - nSamples out.Signal = numpy.transpose(out.Signal, (1,0)) # nSamples - by - nChannels out.Signal = out.Signal + 0.0 # make a contiguous copy else: raise RuntimeError('internal error') out.States = SigTools.sstruct() states = [(k,int(getattr(mat.Index, k))-1) for k in mat.Index._fieldnames if k != 'Signal'] for k,v in states: setattr(out.States, k, mat.Data[v,:]) # TODO: how do the command-line tools handle event states? this seems to be set up to deliver one value per block whatever kind of state we're dealing with out.Megabytes = megs(out) else: out = err keep = True print err print if os.path.isdir(tmpdir): files = sorted([os.path.join(tmpdir, file) for file in os.listdir(tmpdir) if file not in ['.','..']]) if keep: print 'The following commands should be executed to clean up the temporary files:' elif verbose: print '# removing temp files and directory ' + tmpdir for file in files: if keep: print " os.remove('%s')" % file else: os.remove(file)
def TimingWindow(filename='.', ind=-1, save=None): """ Recreate BCI2000's timing window offline, from a saved .dat file specified by <filename>. It is also possible to supply a directory name as <filename>, and an index <ind> (default value -1 meaning "the last run") to choose a file automatically from that directory. Based on BCI2000's src/shared/modules/signalsource/DataIOFilter.cpp where the timing window content is computed in DataIOFilter::Process(), this is what appears to happen: Begin SampleBlock #t: Enter SignalSource module's first Process() method (DataIOFilter::Process) Save previous SampleBlock to file Wait to acquire new SampleBlock from hardware +--------- Measure SourceTime in SignalSource module | | | Make a local copy of all states (NB: all except SourceTime were set during #t-1) ---+ B| R| S| Pipe the signal through the rest of BCI2000 | | | +- Measure StimulusTime in Application module, on leaving last Process() method | | | | | | | | | Begin SampleBlock #t+1: | | +----- Enter SignalSource module's first Process() method (DataIOFilter::Process) | | Save data from #t, SourceTime state from #t, and other states from #t-1, to file <--+ | Wait to acquire new SampleBlock from hardware +--------- Measure SourceTime in SignalSource module Make a local copy of all states (NB: all except SourceTime were set during #t) Leave DataIOFilter::Process() and pipe the signal through the rest of BCI2000 Measure StimulusTime in Application module, on leaving last Process() method B stands for Block duration. R stands for Roundtrip time (visible in VisualizeTiming, not reconstructable from the .dat file) S is the filter cascade time (marked "Stimulus" in the VisualizeTiming window). Note that, on any given SampleBlock as saved in the file, SourceTime will be *greater* than any other timestamp states (including StimulusTime), because it is the only state that is updated in time to be saved with the data packet it belongs to. All the others lag by one packet. This is corrected for at the point commented with ??? in the Python code. """ if hasattr(filename, 'filename'): filename = filename.filename b = bcistream(filename=filename, ind=ind) out = SigTools.sstruct() out.filename = b.filename #print "decoding..." sig, states = b.decode('all') #print "done" b.close() dT, T, rT = {}, {}, {} statenames = ['SourceTime', 'StimulusTime' ] + ['PythonTiming%02d' % (x + 1) for x in range(2)] statenames = [s for s in statenames if s in states] for key in statenames: dT[key], T[key] = SigTools.unwrapdiff(states[key].flatten(), base=65536, dtype=numpy.float64) sel, = numpy.where(dT['SourceTime']) for key in statenames: dT[key] = dT[key][sel[1:]] if key == 'SourceTime': tsel = sel[:-1] # ??? why the shift else: tsel = sel[1:] # ??? relative to here? T[key] = T[key][tsel + 1] t0 = T['SourceTime'][0] for key in statenames: T[key] -= t0 t = T['SourceTime'] / 1000 expected = b.samples2msec(b.params['SampleBlockSize']) datestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(b.datestamp)) paramstr = ', '.join([ '%s=%s' % (x, b.params[x]) for x in [ 'SampleBlockSize', 'SamplingRate', 'VisualizeTiming', 'VisualizeSource' ] ]) chainstr = '-'.join([ x for x, y in b.params['SignalSourceFilterChain'] + b.params['SignalProcessingFilterChain'] + b.params['ApplicationFilterChain'] ]) titlestr = '\n'.join([b.filename, datestamp, paramstr, chainstr]) SigTools.plot(t[[0, -1]], [expected] * 2, drawnow=False) SigTools.plot(t, dT['SourceTime'], hold=True, drawnow=False) for key in statenames: if key == 'SourceTime': continue rT[key] = T[key] - T['SourceTime'] SigTools.plot(t, rT[key], hold=True, drawnow=False) import pylab pylab.title(titlestr) pylab.grid(True) pylab.xlabel('seconds') pylab.ylabel('milliseconds') ymin, ymax = pylab.ylim() pylab.ylim(ymax=max(ymax, expected * 2)) pylab.xlim(xmax=t[-1]) pylab.draw() out.params = SigTools.sstruct(b.params) out.summarystr = titlestr out.t = t out.SourceTime = T['SourceTime'] out.StimulusTime = T['StimulusTime'] out.BlockDuration = dT['SourceTime'] out.BlockDuration2 = dT['StimulusTime'] out.ProcessingTime = out.StimulusTime - out.SourceTime out.ExpectedBlockDuration = expected out.rT = rT out.dT = dT out.T = T if save: pylab.gcf().savefig(save, orientation='landscape') return out
def StimulusTiming(filename='.', ind=None, channels=0, trigger='StimulusCode > 0', msec=200, rectify=False, threshold=0.5, use_eo=True, save=None, **kwargs): """ In <filename> and <ind>, give it - a directory and ind=None: for all .dat files in the directory, in session/run order - a directory and ind=an index or list of indices: for selected .dat files in the directory - a dat-file name and ind=anything: for that particular file - a list of filenames and ind=anything: for certain explicitly-specified files <channels> may be a 0-based index, list of indices, list of channel names, or space- or comma- delimited string of channel names <rectify> subtracts the median and takes the abs before doing anything else <threshold> is on the normalized scale of min=0, max=1 within the resulting image <use_eo> uses the EventOffset state to correct timings """### if hasattr(filename, 'filename'): filename = filename.filename if ind == None: ind = -1 if os.path.isdir(filename): filename = ListDatFiles(filename) if not isinstance(filename, (tuple, list)): filename = [filename] if not isinstance(ind, (tuple, list)): ind = [ind] n = max(len(filename), len(ind)) if len(filename) == 1: filename = list(filename) * n if len(ind) == 1: ind = list(ind) * n if isinstance(channels, basestring): channels = channels.replace(',', ' ').split() if not isinstance(channels, (tuple, list)): channels = [channels] out = [ SigTools.sstruct( files=[], events=[], t=None, channel=ch, img=[], edges=[], threshold=None, EventOffsets=[], UseEventOffsets=False, ) for ch in channels ] if len(filename) == 0: raise ValueError("no data files specified") for f, i in zip(filename, ind): b = bcistream(filename=f, ind=i) nsamp = b.msec2samples(msec) sig, st = b.decode('all') statenames = zip(*sorted([(-len(x), x) for x in st]))[1] criterion = trigger for x in statenames: criterion = criterion.replace(x, "st['%s']" % x) criterion = numpy.asarray(eval(criterion)).flatten() startind = RisingEdge(criterion).nonzero()[0] + 1 print "%d events found in %s" % (len(startind), b.filename) for s in out: s.files.append(b.filename) s.events.append(len(startind)) ch = s.channel if isinstance(ch, basestring): chn = [x.lower() for x in b.params['ChannelNames']] if ch.lower() in chn: ch = chn.index(ch.lower()) else: raise ValueError("could not find channel %s in %s" % (ch, b.filename)) if len(b.params['ChannelNames']) == len(sig): s.channel = b.params['ChannelNames'][ch] xx = numpy.asarray(sig)[ch] if rectify: xx = numpy.abs(xx - numpy.median(xx)) xx -= xx.min() if xx.max(): xx /= xx.max() s.threshold = threshold for ind in startind: if 'EventOffset' in st: eo = st['EventOffset'].flat[ind] if use_eo: ind += eo - 2**(b.statedefs['EventOffset']['length'] - 1) s.UseEventOffsets = True else: eo = 0 s.EventOffsets.append(eo) x = xx[ind:ind + nsamp].tolist() x += [0.0] * (nsamp - len(x)) s.img.append(x) for s in out: s.img = numpy.asarray(s.img) s.edges = [ min(list(x.nonzero()[0]) + [numpy.nan]) for x in (s.img > s.threshold) ] s.edges = b.samples2msec(numpy.asarray(s.edges)) s.t = b.samples2msec(numpy.arange(nsamp)) import pylab pylab.clf() for i, s in enumerate(out): pylab.subplot(1, len(out), i + 1) y = y = range(1, len(s.img) + 1) SigTools.imagesc(s.img, x=s.t, y=y, aspect='auto', **kwargs) xl, yl = pylab.xlim(), pylab.ylim() pylab.plot(s.edges, y, 'w*', markersize=10) pylab.xlim(xl) pylab.ylim(yl) pylab.grid('on') #pylab.ylim([len(s.img)+0.5,0.5]) # this corrupts the image!! pylab.draw() if save: pylab.gcf().savefig(save, orientation='portrait') return out
def bci2000chain(datfile, chain='SpectralSignalProcessing', parms=(), dims='auto', start=None, duration=None, verbose=False, keep=False, binpath=None, **kwargs): """ This is a provisional Python equivalent for tools/matlab/bci2000chain.m Excuse the relative paths - this example works if you're currently working in the root directory of the BCI2000 distro: bci2000chain(datfile='data/samplefiles/eeg3_2.dat', chain='TransmissionFilter|SpatialFilter|ARFilter', binpath='tools/cmdline', parms=['tools/matlab/ExampleParameters.prm'], SpatialFilterType=3) TODO: documentation For clues, see http://www.bci2000.org/wiki/index.php/User_Reference:Matlab_Tools and matlab documentation in bci2000chain.m Most arguments are like the flags in the Matlab equivalent. mutatis to a large extent mutandis. Note that for now there is no global way of managing the system path. Either add the tools/cmdline directory to your $PATH variable before starting Python, or supply its location every time while calling, in the <binpath> argument. TODO: test on Windoze """### if isinstance(chain, basestring): if chain.lower() == 'SpectralSignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|SpectralEstimator|LinearClassifier|LPFilter|ExpressionFilter|Normalizer' elif chain.lower() == 'ARSignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|ARFilter|LinearClassifier|LPFilter|ExpressionFilter|Normalizer' elif chain.lower() == 'P3SignalProcessing'.lower(): chain = 'TransmissionFilter|SpatialFilter|P3TemporalFilter|LinearClassifier' chain = chain.split('|') chain = [c.strip() for c in chain if len(c.strip())] if len(chain) == 0: print('WARNING: chain is empty') if start != None and len(str(start).strip()): start = ' --start=' + str(start).replace(' ', '') else: start = '' if duration != None and len(str(duration).strip()): duration = ' --duration=' + str(duration).replace(' ', '') else: duration = '' if dims == None or str(dims).lower() == 'auto': dims = 0 if dims not in (0, 2, 3): raise ValueError("dims must be 2, 3 or 'auto'") out = SigTools.sstruct() err = '' cmd = '' binaries = [] tmpdir = tempfile.mkdtemp(prefix='bci2000chain_') tmpdatfile = os.path.join(tmpdir, 'in.dat') prmfile_in = os.path.join(tmpdir, 'in.prm') prmfile_out = os.path.join(tmpdir, 'out.prm') matfile = os.path.join(tmpdir, 'out.mat') bcifile = os.path.join(tmpdir, 'out.bci') shfile = os.path.join(tmpdir, 'go.bat') logfile = os.path.join(tmpdir, 'log.txt') if not isinstance(datfile, basestring): raise ValueError( 'datfile must be a filename' ) # TODO: if datfile contains the appropriate info, use some create_bcidat equivalent and do datfile = tmpdatfile if not os.path.isfile(datfile): raise IOError('file not found: %s' % datfile) mappings = { '$DATFILE': datfile, '$PRMFILE_IN': prmfile_in, '$PRMFILE_OUT': prmfile_out, '$MATFILE': matfile, '$BCIFILE': bcifile, '$SHFILE': shfile, '$LOGFILE': logfile, } def exe(name): if binpath: return os.path.join(binpath, name) else: return name if parms == None: parms = [] if isinstance(parms, tuple): parms = list(parms) if not isinstance(parms, list): parms = [parms] else: parms = list(parms) if len(kwargs): parms.append(kwargs) if parms == None or len(parms) == 0: cmd += exe('bci_dat2stream') + start + duration + ' < "$DATFILE"' binaries.append('bci_dat2stream') else: if verbose: print '# writing custom parameter file %s' % prmfile_in parms = make_bciprm(datfile, parms, '>', prmfile_in) if dat2stream_has_p_flag: cmd += exe( 'bci_dat2stream' ) + ' -p$PRMFILE_IN' + start + duration + ' < "$DATFILE"' # new-style bci_dat2stream with -p option binaries.append('bci_dat2stream') else: if len(start) or len(duration): raise ValueError( 'old versionsof bci_dat2stream have no --start or --duration option' ) cmd += '(' # old-style bci_dat2stream with no -p option cmd += exe('bci_prm2stream') + ' < $PRMFILE_IN' cmd += '&& ' + exe('bci_dat2stream') + ' --transmit-sd < $DATFILE' cmd += ')' binaries.append('bci_dat2stream') binaries.append('bci_prm2stream') for c in chain: cmd += ' | ' + exe(c) binaries.append(c) if stream2mat_saves_parms: cmd += ' | ' + exe( 'bci_stream2mat' ) + ' > $MATFILE' # new-style bci_stream2mat with Parms output binaries.append('bci_stream2mat') else: cmd += ' > $BCIFILE' cmd += ' && ' + exe('bci_stream2mat') + ' < $BCIFILE > $MATFILE' cmd += ' && ' + exe( 'bci_stream2prm' ) + ' < $BCIFILE > $PRMFILE_OUT' # old-style bci_stream2mat without Parms output binaries.append('bci_stream2mat') binaries.append('bci_stream2prm') for k, v in mappings.items(): cmd = cmd.replace(k, v) win = sys.platform.lower().startswith('win') if win: shebang = '' else: shebang = '#!/bin/sh\n\n' open(shfile, 'wt').write(shebang + cmd + '\n') if not win: os.chmod(shfile, 484) # rwxr--r-- def tidytext(x): return x.strip().replace('\r\n', '\n').replace('\r', '\n') def getstatusoutput(cmd): pipe = os.popen( cmd + ' 2>&1', 'r' ) # TODO: does this work on Windows? could always make use of logfile here if not output = pipe.read() status = pipe.close() if status == None: status = 0 return status, tidytext(output) def getoutput(cmd): return getstatusoutput(cmd)[1] if verbose: print '# querying version information' binaries = SigTools.sstruct([ (bin, getoutput(exe(bin) + ' --version').replace('\n', ' ')) for bin in binaries ]) if verbose: print cmd t0 = time.time() failed, output = getstatusoutput(shfile) chaintime = time.time() - t0 failsig = 'Configuration Error: ' if failsig in output: failed = 1 printable_output = output printable_lines = output.split('\n') maxlines = 10 if len(printable_lines) > maxlines: printable_output = '\n'.join( printable_lines[:maxlines] + ['[%d more lines omitted]' % (len(printable_lines) - maxlines)]) if failed: if verbose: err = 'system call failed:\n' + printable_output # cmd has already been printed, so don't clutter things further else: err = 'system call failed:\n%s\n%s' % (cmd, printable_output) if err == '': if verbose: print '# loading %s' % matfile try: mat = SigTools.loadmat(matfile) except: err = "The chain must have failed: could not load %s\nShell output was as follows:\n%s" % ( matfile, printable_output) else: if 'Data' not in mat: err = "The chain must have failed: no 'Data' variable found in %s\nShell output was as follows:\n%s" % ( matfile, printable_output) if 'Index' not in mat: err = "The chain must have failed: no 'Index' variable found in %s\nShell output was as follows:\n%s" % ( matfile, printable_output) if err == '': out.FileName = datfile if stream2mat_saves_parms: if verbose: print '# decoding parameters loaded from the mat-file' parms = make_bciprm(mat.Parms[0]) else: if verbose: print '# reading output parameter file' + prmfile_out parms = ParmList( prmfile_out ) # if you get an error that prmfile_out does not exist, recompile your bci_dat2stream and bci_stream2mat binaries from up-to-date sources, and ensure that dat2stream_has_p_flag and stream2mat_saves_parms, at the top of this file, are both set to 1 out.DateStr = read_bcidate(parms, 'ISO') out.DateNum = read_bcidate(parms) out.FilterChain = chain out.ToolVersions = binaries out.ShellInput = cmd out.ShellOutput = output out.ChainTime = chaintime out.ChainSpeedFactor = None out.Megabytes = None out.Parms = parms out.Parms.sort() mat.Index = mat.Index[0, 0] sigind = mat.Index.Signal - 1 # indices vary across channels fastest, then elements nChannels, nElements = sigind.shape nBlocks = mat.Data.shape[1] out.Blocks = nBlocks out.BlocksPerSecond = float(parms.SamplingRate.ScaledValue) / float( parms.SampleBlockSize.ScaledValue) out.SecondsPerBlock = float(parms.SampleBlockSize.ScaledValue) / float( parms.SamplingRate.ScaledValue) out.ChainSpeedFactor = float(out.Blocks * out.SecondsPerBlock) / float( out.ChainTime) def unnumpify(x): while isinstance(x, numpy.ndarray) and x.size == 1: x = x[0] if isinstance(x, (numpy.ndarray, tuple, list)): x = [unnumpify(xi) for xi in x] return x out.Channels = nChannels out.ChannelLabels = unnumpify(mat.get('ChannelLabels', [])) try: out.ChannelSet = SigTools.ChannelSet(out.ChannelLabels) except: out.ChannelSet = None out.Elements = nElements out.ElementLabels = unnumpify(mat.get('ElementLabels', [])) out.ElementValues = numpy.ravel(mat.get('ElementValues', [])) out.ElementUnit = unnumpify(mat.get('ElementUnit', None)) out.ElementRate = out.BlocksPerSecond * out.Elements out.Time = out.SecondsPerBlock * numpy.arange(0, nBlocks) out.FullTime = out.Time out.FullElementValues = out.ElementValues # Why does sigind have to be transposed before vectorizing to achieve the same result as sigind(:) WITHOUT a transpose in Matlab? # This will probably forever be one of the many deep mysteries of numpy dimensionality handling out.Signal = mat.Data[ sigind.T.ravel(), :] # nChannels*nElements - by - nBlocks out.Signal = out.Signal + 0.0 # make a contiguous copy def seconds( s ): # -1 means "no information", 0 means "not units of time", >0 means the scaling factor if getattr(s, 'ElementUnit', None) in ('', None): return -1 s = s.ElementUnit if s.endswith('seconds'): s = s[:-6] elif s.endswith('second'): s = s[:-5] elif s.endswith('sec'): s = s[:-2] if s.endswith('s'): return { 'ps': 1e-12, 'ns': 1e-9, 'us': 1e-6, 'mus': 1e-6, 'ms': 1e-3, 's': 1e+0, 'ks': 1e+3, 'Ms': 1e+6, 'Gs': 1e+9, 'Ts': 1e+12, }.get(s, 0) else: return 0 if dims == 0: # dimensionality has not been specified explicitly: so guess, based on ElementUnit and/or filter name # 3-dimensional output makes more sense than continuous 2-D whenever "elements" can't just be concatenated into an unbroken time-stream if len(chain): lastfilter = chain[-1].lower() else: lastfilter = '' if lastfilter == 'p3temporalfilter': dims = 3 else: factor = seconds(out) if factor > 0: # units of time. TODO: could detect whether the out.ElementValues*factor are (close enough to) contiguous from block to block; then p3temporalfilter wouldn't have to be a special case above dims = 2 elif factor == 0: # not units of time: use 3D by default dims = 3 elif lastfilter in [ 'p3temporalfilter', 'arfilter', 'fftfilter', 'coherencefilter', 'coherencefftfilter' ]: # no ElementUnit info? guess based on filter name dims = 3 else: dims = 2 if dims == 3: out.Signal = numpy.reshape( out.Signal, (nChannels, nElements, nBlocks), order='F') # nChannels - by - nElements - by - nBlocks out.Signal = numpy.transpose( out.Signal, (2, 0, 1)) # nBlocks - by - nChannels - by - nElements out.Signal = out.Signal + 0.0 # make a contiguous copy elif dims == 2: out.FullTime = numpy.repeat(out.Time, nElements) factor = seconds(out) if len(out.ElementValues): out.FullElementValues = numpy.tile(out.ElementValues, nBlocks) if factor > 0: out.FullTime = out.FullTime + out.FullElementValues * factor out.Signal = numpy.reshape(out.Signal, (nChannels, nElements * nBlocks), order='F') # nChannels - by - nSamples out.Signal = numpy.transpose(out.Signal, (1, 0)) # nSamples - by - nChannels out.Signal = out.Signal + 0.0 # make a contiguous copy else: raise RuntimeError('internal error') out.States = SigTools.sstruct() states = [(k, int(getattr(mat.Index, k)) - 1) for k in mat.Index._fieldnames if k != 'Signal'] for k, v in states: setattr(out.States, k, mat.Data[v, :]) # TODO: how do the command-line tools handle event states? this seems to be set up to deliver one value per block whatever kind of state we're dealing with out.Megabytes = megs(out) else: out = err keep = True print err print if os.path.isdir(tmpdir): files = sorted([ os.path.join(tmpdir, file) for file in os.listdir(tmpdir) if file not in ['.', '..'] ]) if keep: print 'The following commands should be executed to clean up the temporary files:' elif verbose: print '# removing temp files and directory ' + tmpdir for file in files: if keep: print " os.remove('%s')" % file else: os.remove(file) if keep: print " os.rmdir('%s')" % tmpdir else: os.rmdir(tmpdir)
def TimingWindow(filename='.', ind=-1, save=None): """ Recreate BCI2000's timing window offline, from a saved .dat file specified by <filename>. It is also possible to supply a directory name as <filename>, and an index <ind> (default value -1 meaning "the last run") to choose a file automatically from that directory. Based on BCI2000's src/shared/modules/signalsource/DataIOFilter.cpp where the timing window content is computed in DataIOFilter::Process(), this is what appears to happen: Begin SampleBlock #t: Enter SignalSource module's first Process() method (DataIOFilter::Process) Save previous SampleBlock to file Wait to acquire new SampleBlock from hardware +--------- Measure SourceTime in SignalSource module | | | Make a local copy of all states (NB: all except SourceTime were set during #t-1) ---+ B| R| S| Pipe the signal through the rest of BCI2000 | | | +- Measure StimulusTime in Application module, on leaving last Process() method | | | | | | | | | Begin SampleBlock #t+1: | | +----- Enter SignalSource module's first Process() method (DataIOFilter::Process) | | Save data from #t, SourceTime state from #t, and other states from #t-1, to file <--+ | Wait to acquire new SampleBlock from hardware +--------- Measure SourceTime in SignalSource module Make a local copy of all states (NB: all except SourceTime were set during #t) Leave DataIOFilter::Process() and pipe the signal through the rest of BCI2000 Measure StimulusTime in Application module, on leaving last Process() method B stands for Block duration. R stands for Roundtrip time (visible in VisualizeTiming, not reconstructable from the .dat file) S is the filter cascade time (marked "Stimulus" in the VisualizeTiming window). Note that, on any given SampleBlock as saved in the file, SourceTime will be *greater* than any other timestamp states (including StimulusTime), because it is the only state that is updated in time to be saved with the data packet it belongs to. All the others lag by one packet. This is corrected for at the point commented with ??? in the Python code. """ if hasattr(filename, 'filename'): filename = filename.filename b = bcistream(filename=filename, ind=ind) out = SigTools.sstruct() out.filename = b.filename #print "decoding..." sig,states = b.decode('all') #print "done" b.close() dT,T,rT = {},{},{} statenames = ['SourceTime', 'StimulusTime'] + ['PythonTiming%02d' % (x+1) for x in range(2)] statenames = [s for s in statenames if s in states] for key in statenames: dT[key],T[key] = SigTools.unwrapdiff(states[key].flatten(), base=65536, dtype=numpy.float64) sel, = numpy.where(dT['SourceTime']) for key in statenames: dT[key] = dT[key][sel[1:]] if key == 'SourceTime': tsel = sel[:-1] # ??? why the shift else: tsel = sel[1:] # ??? relative to here? T[key] = T[key][tsel+1] t0 = T['SourceTime'][0] for key in statenames: T[key] -= t0 t = T['SourceTime'] / 1000 expected = b.samples2msec(b.params['SampleBlockSize']) datestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(b.datestamp)) paramstr = ', '.join(['%s=%s' % (x,b.params[x]) for x in ['SampleBlockSize', 'SamplingRate', 'VisualizeTiming', 'VisualizeSource']]) chainstr = '-'.join([x for x,y in b.params['SignalSourceFilterChain']+b.params['SignalProcessingFilterChain']+b.params['ApplicationFilterChain']]) titlestr = '\n'.join([b.filename, datestamp, paramstr, chainstr]) SigTools.plot(t[[0,-1]], [expected]*2, drawnow=False) SigTools.plot(t, dT['SourceTime'], hold=True, drawnow=False) for key in statenames: if key == 'SourceTime': continue rT[key] = T[key] - T['SourceTime'] SigTools.plot(t, rT[key], hold=True, drawnow=False) import pylab pylab.title(titlestr) pylab.grid(True) pylab.xlabel('seconds') pylab.ylabel('milliseconds') ymin,ymax = pylab.ylim(); pylab.ylim(ymax=max(ymax,expected*2)) pylab.xlim(xmax=t[-1]) pylab.draw() out.params = SigTools.sstruct(b.params) out.summarystr = titlestr out.t = t out.SourceTime = T['SourceTime'] out.StimulusTime = T['StimulusTime'] out.BlockDuration = dT['SourceTime'] out.BlockDuration2 = dT['StimulusTime'] out.ProcessingTime = out.StimulusTime - out.SourceTime out.ExpectedBlockDuration = expected out.rT = rT out.dT = dT out.T = T if save: pylab.gcf().savefig(save, orientation='landscape') return out
def StimulusTiming(filename='.', ind=None, channels=0, trigger='StimulusCode > 0', msec=200, rectify=False, threshold=0.5, use_eo=True, save=None, **kwargs): """ In <filename> and <ind>, give it - a directory and ind=None: for all .dat files in the directory, in session/run order - a directory and ind=an index or list of indices: for selected .dat files in the directory - a dat-file name and ind=anything: for that particular file - a list of filenames and ind=anything: for certain explicitly-specified files <channels> may be a 0-based index, list of indices, list of channel names, or space- or comma- delimited string of channel names <rectify> subtracts the median and takes the abs before doing anything else <threshold> is on the normalized scale of min=0, max=1 within the resulting image <use_eo> uses the EventOffset state to correct timings """### if hasattr(filename, 'filename'): filename = filename.filename if ind==None: ind = -1 if os.path.isdir(filename): filename = ListDatFiles(filename) if not isinstance(filename, (tuple,list)): filename = [filename] if not isinstance(ind, (tuple,list)): ind = [ind] n = max(len(filename), len(ind)) if len(filename) == 1: filename = list(filename) * n if len(ind) == 1: ind = list(ind) * n if isinstance(channels, basestring): channels = channels.replace(',', ' ').split() if not isinstance(channels, (tuple,list)): channels = [channels] out = [SigTools.sstruct( files=[], events=[], t=None, channel=ch, img=[], edges=[], threshold=None, EventOffsets=[], UseEventOffsets=False, ) for ch in channels] if len(filename) == 0: raise ValueError("no data files specified") for f,i in zip(filename, ind): b = bcistream(filename=f, ind=i) nsamp = b.msec2samples(msec) sig,st = b.decode('all') statenames = zip(*sorted([(-len(x),x) for x in st]))[1] criterion = trigger for x in statenames: criterion = criterion.replace(x, "st['%s']"%x) criterion = numpy.asarray(eval(criterion)).flatten() startind = RisingEdge(criterion).nonzero()[0] + 1 print "%d events found in %s" % (len(startind), b.filename) for s in out: s.files.append(b.filename) s.events.append(len(startind)) ch = s.channel if isinstance(ch, basestring): chn = [x.lower() for x in b.params['ChannelNames']] if ch.lower() in chn: ch = chn.index(ch.lower()) else: raise ValueError("could not find channel %s in %s" % (ch,b.filename)) if len(b.params['ChannelNames']) == len(sig): s.channel = b.params['ChannelNames'][ch] xx = numpy.asarray(sig)[ch] if rectify: xx = numpy.abs(xx - numpy.median(xx)) xx -= xx.min() if xx.max(): xx /= xx.max() s.threshold = threshold for ind in startind: if 'EventOffset' in st: eo = st['EventOffset'].flat[ind] if use_eo: ind += eo - 2**(b.statedefs['EventOffset']['length']-1) s.UseEventOffsets = True else: eo = 0 s.EventOffsets.append(eo) x = xx[ind:ind+nsamp].tolist() x += [0.0] * (nsamp - len(x)) s.img.append(x) for s in out: s.img = numpy.asarray(s.img) s.edges = [min(list(x.nonzero()[0])+[numpy.nan]) for x in (s.img > s.threshold)] s.edges = b.samples2msec(numpy.asarray(s.edges)) s.t = b.samples2msec(numpy.arange(nsamp)) import pylab pylab.clf() for i,s in enumerate(out): pylab.subplot(1, len(out), i+1) y = y=range(1,len(s.img)+1) SigTools.imagesc(s.img, x=s.t, y=y, aspect='auto', **kwargs) xl,yl = pylab.xlim(),pylab.ylim() pylab.plot(s.edges, y, 'w*', markersize=10) pylab.xlim(xl); pylab.ylim(yl) pylab.grid('on') #pylab.ylim([len(s.img)+0.5,0.5]) # this corrupts the image!! pylab.draw() if save: pylab.gcf().savefig(save, orientation='portrait') return out