def bci2000chain(datfile, chain='SpectralSignalProcessing', parms=(), dims='auto', start=None, duration=None, verbose=False, keep=False, binpath=None, **kwargs):
"""
This is a provisional Python equivalent for tools/matlab/bci2000chain.m
Excuse the relative paths - this example works if you're currently working in the root
directory of the BCI2000 distro:
bci2000chain(datfile='data/samplefiles/eeg3_2.dat',
chain='TransmissionFilter|SpatialFilter|ARFilter',
binpath='tools/cmdline',
parms=['tools/matlab/ExampleParameters.prm'], SpatialFilterType=3)
TODO: documentation
For clues, see http://www.bci2000.org/wiki/index.php/User_Reference:Matlab_Tools
and matlab documentation in bci2000chain.m
Most arguments are like the flags in the Matlab equivalent. mutatis to a large extent mutandis.
Note that for now there is no global way of managing the system path. Either add the
tools/cmdline directory to your $PATH variable before starting Python, or supply its location
every time while calling, in the <binpath> argument.
TODO: test on Windoze
"""###
if isinstance(chain, basestring):
if chain.lower() == 'SpectralSignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|SpectralEstimator|LinearClassifier|LPFilter|ExpressionFilter|Normalizer'
elif chain.lower() == 'ARSignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|ARFilter|LinearClassifier|LPFilter|ExpressionFilter|Normalizer'
elif chain.lower() == 'P3SignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|P3TemporalFilter|LinearClassifier'
chain = chain.split('|')
chain = [c.strip() for c in chain if len(c.strip())]
if len(chain) == 0: print('WARNING: chain is empty')
if start != None and len(str(start).strip()): start = ' --start=' + str(start).replace(' ', '')
else: start = ''
if duration != None and len(str(duration).strip()): duration = ' --duration=' + str(duration).replace(' ', '')
else: duration = ''
if dims == None or str(dims).lower() == 'auto': dims = 0
if dims not in (0, 2, 3): raise ValueError("dims must be 2, 3 or 'auto'")
out = SigTools.sstruct()
err = ''
cmd = ''
binaries = []
tmpdir = tempfile.mkdtemp(prefix='bci2000chain_')
tmpdatfile = os.path.join(tmpdir, 'in.dat')
prmfile_in = os.path.join(tmpdir, 'in.prm')
prmfile_out = os.path.join(tmpdir, 'out.prm')
matfile = os.path.join(tmpdir, 'out.mat')
bcifile = os.path.join(tmpdir, 'out.bci')
shfile = os.path.join(tmpdir, 'go.bat')
logfile = os.path.join(tmpdir, 'log.txt')
if not isinstance(datfile, basestring):
raise ValueError('datfile must be a filename') # TODO: if datfile contains the appropriate info, use some create_bcidat equivalent and do datfile = tmpdatfile
if not os.path.isfile(datfile): raise IOError('file not found: %s' % datfile)
mappings = {
'$DATFILE': datfile,
'$PRMFILE_IN': prmfile_in,
'$PRMFILE_OUT': prmfile_out,
'$MATFILE': matfile,
'$BCIFILE': bcifile,
'$SHFILE': shfile,
'$LOGFILE': logfile,
}
def exe(name):
if binpath: return os.path.join(binpath, name)
else: return name
if parms == None: parms = []
if isinstance(parms, tuple): parms = list(parms)
if not isinstance(parms, list): parms = [parms]
else: parms = list(parms)
if len(kwargs): parms.append(kwargs)
if parms == None or len(parms) == 0:
cmd += exe('bci_dat2stream') + start + duration + ' < "$DATFILE"'
binaries.append('bci_dat2stream')
else:
if verbose: print '# writing custom parameter file %s' % prmfile_in
parms = make_bciprm(datfile, parms, '>', prmfile_in)
if dat2stream_has_p_flag:
cmd += exe('bci_dat2stream') + ' -p$PRMFILE_IN' + start + duration + ' < "$DATFILE"' # new-style bci_dat2stream with -p option
binaries.append('bci_dat2stream')
else:
if len(start) or len(duration):
raise ValueError('old versionsof bci_dat2stream have no --start or --duration option')
cmd += '(' # old-style bci_dat2stream with no -p option
cmd += exe('bci_prm2stream') + ' < $PRMFILE_IN'
cmd += '&& ' + exe('bci_dat2stream') + ' --transmit-sd < $DATFILE'
cmd += ')'
binaries.append('bci_dat2stream')
binaries.append('bci_prm2stream')
for c in chain:
cmd += ' | ' + exe(c)
binaries.append(c)
if stream2mat_saves_parms:
cmd += ' | ' + exe('bci_stream2mat') + ' > $MATFILE' # new-style bci_stream2mat with Parms output
binaries.append('bci_stream2mat')
else:
cmd += ' > $BCIFILE'
cmd += ' && ' + exe('bci_stream2mat') + ' < $BCIFILE > $MATFILE'
cmd += ' && ' + exe('bci_stream2prm') + ' < $BCIFILE > $PRMFILE_OUT' # old-style bci_stream2mat without Parms output
binaries.append('bci_stream2mat')
binaries.append('bci_stream2prm')
for k,v in mappings.items(): cmd = cmd.replace(k, v)
win = sys.platform.lower().startswith('win')
if win: shebang = ''
else: shebang = '#!/bin/sh\n\n'
open(shfile, 'wt').write(shebang+cmd+'\n')
if not win: os.chmod(shfile, 484) # rwxr--r--
def tidytext(x):
return x.strip().replace('\r\n', '\n').replace('\r', '\n')
def getstatusoutput(cmd):
pipe = os.popen(cmd + ' 2>&1', 'r') # TODO: does this work on Windows? could always make use of logfile here if not
output = pipe.read()
status = pipe.close()
if status == None: status = 0
return status,tidytext(output)
def getoutput(cmd):
return getstatusoutput(cmd)[1]
if verbose: print '# querying version information'
binaries = SigTools.sstruct([(bin, getoutput(exe(bin) + ' --version').replace('\n', ' ') ) for bin in binaries])
if verbose: print cmd
t0 = time.time()
failed,output = getstatusoutput(shfile)
chaintime = time.time() - t0
failsig = 'Configuration Error: '
if failsig in output: failed = 1
printable_output = output
printable_lines = output.split('\n')
maxlines = 10
if len(printable_lines) > maxlines:
printable_output = '\n'.join(printable_lines[:maxlines] + ['[%d more lines omitted]' % (len(printable_lines) - maxlines)])
if failed:
if verbose: err = 'system call failed:\n' + printable_output # cmd has already been printed, so don't clutter things further
else: err = 'system call failed:\n%s\n%s' % (cmd, printable_output)
if err == '':
if verbose: print '# loading %s' % matfile
try:
mat = SigTools.loadmat(matfile)
except:
err = "The chain must have failed: could not load %s\nShell output was as follows:\n%s" % (matfile, printable_output)
else:
if 'Data' not in mat: err = "The chain must have failed: no 'Data' variable found in %s\nShell output was as follows:\n%s" % (matfile, printable_output)
if 'Index' not in mat: err = "The chain must have failed: no 'Index' variable found in %s\nShell output was as follows:\n%s" % (matfile, printable_output)
if err == '':
out.FileName = datfile
if stream2mat_saves_parms:
if verbose: print '# decoding parameters loaded from the mat-file'
parms = make_bciprm(mat.Parms[0])
else:
if verbose: print '# reading output parameter file' + prmfile_out
parms = ParmList(prmfile_out) # if you get an error that prmfile_out does not exist, recompile your bci_dat2stream and bci_stream2mat binaries from up-to-date sources, and ensure that dat2stream_has_p_flag and stream2mat_saves_parms, at the top of this file, are both set to 1
out.DateStr = read_bcidate(parms, 'ISO')
out.DateNum = read_bcidate(parms)
out.FilterChain = chain
out.ToolVersions = binaries
out.ShellInput = cmd
out.ShellOutput = output
out.ChainTime = chaintime
out.ChainSpeedFactor = None
out.Megabytes = None
out.Parms = parms
out.Parms.sort()
mat.Index = mat.Index[0,0]
sigind = mat.Index.Signal - 1 # indices vary across channels fastest, then elements
nChannels,nElements = sigind.shape
nBlocks = mat.Data.shape[1]
out.Blocks = nBlocks
out.BlocksPerSecond = float(parms.SamplingRate.ScaledValue) / float(parms.SampleBlockSize.ScaledValue)
out.SecondsPerBlock = float(parms.SampleBlockSize.ScaledValue) / float(parms.SamplingRate.ScaledValue)
out.ChainSpeedFactor = float(out.Blocks * out.SecondsPerBlock) / float(out.ChainTime)
def unnumpify(x):
while isinstance(x, numpy.ndarray) and x.size == 1: x = x[0]
if isinstance(x, (numpy.ndarray,tuple,list)): x = [unnumpify(xi) for xi in x]
return x
out.Channels = nChannels
out.ChannelLabels = unnumpify(mat.get('ChannelLabels', []))
try: out.ChannelSet = SigTools.ChannelSet(out.ChannelLabels)
except: out.ChannelSet = None
out.Elements = nElements
out.ElementLabels = unnumpify(mat.get('ElementLabels', []))
out.ElementValues = numpy.ravel(mat.get('ElementValues', []))
out.ElementUnit = unnumpify(mat.get('ElementUnit', None))
out.ElementRate = out.BlocksPerSecond * out.Elements
out.Time = out.SecondsPerBlock * numpy.arange(0, nBlocks)
out.FullTime = out.Time
out.FullElementValues = out.ElementValues
# Why does sigind have to be transposed before vectorizing to achieve the same result as sigind(:) WITHOUT a transpose in Matlab?
# This will probably forever be one of the many deep mysteries of numpy dimensionality handling
out.Signal = mat.Data[sigind.T.ravel(), :] # nChannels*nElements - by - nBlocks
out.Signal = out.Signal + 0.0 # make a contiguous copy
def seconds(s): # -1 means "no information", 0 means "not units of time", >0 means the scaling factor
if getattr(s, 'ElementUnit', None) in ('',None): return -1
s = s.ElementUnit
if s.endswith('seconds'): s = s[:-6]
elif s.endswith('second'): s = s[:-5]
elif s.endswith('sec'): s = s[:-2]
if s.endswith('s'): return { 'ps': 1e-12, 'ns': 1e-9, 'us': 1e-6, 'mus': 1e-6, 'ms': 1e-3, 's' : 1e+0, 'ks': 1e+3, 'Ms': 1e+6, 'Gs': 1e+9, 'Ts': 1e+12,}.get(s, 0)
else: return 0
if dims == 0: # dimensionality has not been specified explicitly: so guess, based on ElementUnit and/or filter name
# 3-dimensional output makes more sense than continuous 2-D whenever "elements" can't just be concatenated into an unbroken time-stream
if len(chain): lastfilter = chain[-1].lower()
else: lastfilter = ''
if lastfilter == 'p3temporalfilter':
dims = 3
else:
factor = seconds(out)
if factor > 0: # units of time. TODO: could detect whether the out.ElementValues*factor are (close enough to) contiguous from block to block; then p3temporalfilter wouldn't have to be a special case above
dims = 2
elif factor == 0: # not units of time: use 3D by default
dims = 3
elif lastfilter in ['p3temporalfilter', 'arfilter', 'fftfilter', 'coherencefilter', 'coherencefftfilter']: # no ElementUnit info? guess based on filter name
dims = 3
else:
dims = 2
if dims == 3:
out.Signal = numpy.reshape(out.Signal, (nChannels, nElements, nBlocks), order='F') # nChannels - by - nElements - by - nBlocks
out.Signal = numpy.transpose(out.Signal, (2,0,1)) # nBlocks - by - nChannels - by - nElements
out.Signal = out.Signal + 0.0 # make a contiguous copy
elif dims == 2:
out.FullTime = numpy.repeat(out.Time, nElements)
factor = seconds(out)
if len(out.ElementValues):
out.FullElementValues = numpy.tile(out.ElementValues, nBlocks)
if factor > 0: out.FullTime = out.FullTime + out.FullElementValues * factor
out.Signal = numpy.reshape(out.Signal, (nChannels, nElements*nBlocks), order='F') # nChannels - by - nSamples
out.Signal = numpy.transpose(out.Signal, (1,0)) # nSamples - by - nChannels
out.Signal = out.Signal + 0.0 # make a contiguous copy
else:
raise RuntimeError('internal error')
out.States = SigTools.sstruct()
states = [(k,int(getattr(mat.Index, k))-1) for k in mat.Index._fieldnames if k != 'Signal']
for k,v in states: setattr(out.States, k, mat.Data[v,:])
# TODO: how do the command-line tools handle event states? this seems to be set up to deliver one value per block whatever kind of state we're dealing with
out.Megabytes = megs(out)
else:
out = err
keep = True
print err
print
if os.path.isdir(tmpdir):
files = sorted([os.path.join(tmpdir, file) for file in os.listdir(tmpdir) if file not in ['.','..']])
if keep: print 'The following commands should be executed to clean up the temporary files:'
elif verbose: print '# removing temp files and directory ' + tmpdir
for file in files:
if keep: print " os.remove('%s')" % file
else: os.remove(file)
def TimingWindow(filename='.', ind=-1, save=None):
"""
Recreate BCI2000's timing window offline, from a saved .dat file specified by <filename>.
It is also possible to supply a directory name as <filename>, and an index <ind> (default
value -1 meaning "the last run") to choose a file automatically from that directory.
Based on BCI2000's src/shared/modules/signalsource/DataIOFilter.cpp where the timing window
content is computed in DataIOFilter::Process(), this is what appears to happen:
Begin SampleBlock #t:
Enter SignalSource module's first Process() method (DataIOFilter::Process)
Save previous SampleBlock to file
Wait to acquire new SampleBlock from hardware
+--------- Measure SourceTime in SignalSource module
| | | Make a local copy of all states (NB: all except SourceTime were set during #t-1) ---+
B| R| S| Pipe the signal through the rest of BCI2000 |
| | +- Measure StimulusTime in Application module, on leaving last Process() method |
| | |
| | |
| | Begin SampleBlock #t+1: |
| +----- Enter SignalSource module's first Process() method (DataIOFilter::Process) |
| Save data from #t, SourceTime state from #t, and other states from #t-1, to file <--+
| Wait to acquire new SampleBlock from hardware
+--------- Measure SourceTime in SignalSource module
Make a local copy of all states (NB: all except SourceTime were set during #t)
Leave DataIOFilter::Process() and pipe the signal through the rest of BCI2000
Measure StimulusTime in Application module, on leaving last Process() method
B stands for Block duration.
R stands for Roundtrip time (visible in VisualizeTiming, not reconstructable from the .dat file)
S is the filter cascade time (marked "Stimulus" in the VisualizeTiming window).
Note that, on any given SampleBlock as saved in the file, SourceTime will be *greater* than
any other timestamp states (including StimulusTime), because it is the only state that is
updated in time to be saved with the data packet it belongs to. All the others lag by one
packet. This is corrected for at the point commented with ??? in the Python code.
"""
if hasattr(filename, 'filename'): filename = filename.filename
b = bcistream(filename=filename, ind=ind)
out = SigTools.sstruct()
out.filename = b.filename
#print "decoding..."
sig, states = b.decode('all')
#print "done"
b.close()
dT, T, rT = {}, {}, {}
statenames = ['SourceTime', 'StimulusTime'
] + ['PythonTiming%02d' % (x + 1) for x in range(2)]
statenames = [s for s in statenames if s in states]
for key in statenames:
dT[key], T[key] = SigTools.unwrapdiff(states[key].flatten(),
base=65536,
dtype=numpy.float64)
sel, = numpy.where(dT['SourceTime'])
for key in statenames:
dT[key] = dT[key][sel[1:]]
if key == 'SourceTime': tsel = sel[:-1] # ??? why the shift
else: tsel = sel[1:] # ??? relative to here?
T[key] = T[key][tsel + 1]
t0 = T['SourceTime'][0]
for key in statenames:
T[key] -= t0
t = T['SourceTime'] / 1000
expected = b.samples2msec(b.params['SampleBlockSize'])
datestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(b.datestamp))
paramstr = ', '.join([
'%s=%s' % (x, b.params[x]) for x in [
'SampleBlockSize', 'SamplingRate', 'VisualizeTiming',
'VisualizeSource'
]
])
chainstr = '-'.join([
x for x, y in b.params['SignalSourceFilterChain'] +
b.params['SignalProcessingFilterChain'] +
b.params['ApplicationFilterChain']
])
titlestr = '\n'.join([b.filename, datestamp, paramstr, chainstr])
SigTools.plot(t[[0, -1]], [expected] * 2, drawnow=False)
SigTools.plot(t, dT['SourceTime'], hold=True, drawnow=False)
for key in statenames:
if key == 'SourceTime': continue
rT[key] = T[key] - T['SourceTime']
SigTools.plot(t, rT[key], hold=True, drawnow=False)
import pylab
pylab.title(titlestr)
pylab.grid(True)
pylab.xlabel('seconds')
pylab.ylabel('milliseconds')
ymin, ymax = pylab.ylim()
pylab.ylim(ymax=max(ymax, expected * 2))
pylab.xlim(xmax=t[-1])
pylab.draw()
out.params = SigTools.sstruct(b.params)
out.summarystr = titlestr
out.t = t
out.SourceTime = T['SourceTime']
out.StimulusTime = T['StimulusTime']
out.BlockDuration = dT['SourceTime']
out.BlockDuration2 = dT['StimulusTime']
out.ProcessingTime = out.StimulusTime - out.SourceTime
out.ExpectedBlockDuration = expected
out.rT = rT
out.dT = dT
out.T = T
if save:
pylab.gcf().savefig(save, orientation='landscape')
return out
def StimulusTiming(filename='.',
ind=None,
channels=0,
trigger='StimulusCode > 0',
msec=200,
rectify=False,
threshold=0.5,
use_eo=True,
save=None,
**kwargs):
"""
In <filename> and <ind>, give it
- a directory and ind=None: for all .dat files in the directory, in session/run order
- a directory and ind=an index or list of indices: for selected .dat files in the directory
- a dat-file name and ind=anything: for that particular file
- a list of filenames and ind=anything: for certain explicitly-specified files
<channels> may be a 0-based index, list of indices, list of channel names, or space- or comma-
delimited string of channel names
<rectify> subtracts the median and takes the abs before doing anything else
<threshold> is on the normalized scale of min=0, max=1 within the resulting image
<use_eo> uses the EventOffset state to correct timings
"""###
if hasattr(filename, 'filename'): filename = filename.filename
if ind == None:
ind = -1
if os.path.isdir(filename): filename = ListDatFiles(filename)
if not isinstance(filename, (tuple, list)): filename = [filename]
if not isinstance(ind, (tuple, list)): ind = [ind]
n = max(len(filename), len(ind))
if len(filename) == 1: filename = list(filename) * n
if len(ind) == 1: ind = list(ind) * n
if isinstance(channels, basestring):
channels = channels.replace(',', ' ').split()
if not isinstance(channels, (tuple, list)): channels = [channels]
out = [
SigTools.sstruct(
files=[],
events=[],
t=None,
channel=ch,
img=[],
edges=[],
threshold=None,
EventOffsets=[],
UseEventOffsets=False,
) for ch in channels
]
if len(filename) == 0: raise ValueError("no data files specified")
for f, i in zip(filename, ind):
b = bcistream(filename=f, ind=i)
nsamp = b.msec2samples(msec)
sig, st = b.decode('all')
statenames = zip(*sorted([(-len(x), x) for x in st]))[1]
criterion = trigger
for x in statenames:
criterion = criterion.replace(x, "st['%s']" % x)
criterion = numpy.asarray(eval(criterion)).flatten()
startind = RisingEdge(criterion).nonzero()[0] + 1
print "%d events found in %s" % (len(startind), b.filename)
for s in out:
s.files.append(b.filename)
s.events.append(len(startind))
ch = s.channel
if isinstance(ch, basestring):
chn = [x.lower() for x in b.params['ChannelNames']]
if ch.lower() in chn: ch = chn.index(ch.lower())
else:
raise ValueError("could not find channel %s in %s" %
(ch, b.filename))
if len(b.params['ChannelNames']) == len(sig):
s.channel = b.params['ChannelNames'][ch]
xx = numpy.asarray(sig)[ch]
if rectify: xx = numpy.abs(xx - numpy.median(xx))
xx -= xx.min()
if xx.max(): xx /= xx.max()
s.threshold = threshold
for ind in startind:
if 'EventOffset' in st:
eo = st['EventOffset'].flat[ind]
if use_eo:
ind += eo - 2**(b.statedefs['EventOffset']['length'] -
1)
s.UseEventOffsets = True
else:
eo = 0
s.EventOffsets.append(eo)
x = xx[ind:ind + nsamp].tolist()
x += [0.0] * (nsamp - len(x))
s.img.append(x)
for s in out:
s.img = numpy.asarray(s.img)
s.edges = [
min(list(x.nonzero()[0]) + [numpy.nan])
for x in (s.img > s.threshold)
]
s.edges = b.samples2msec(numpy.asarray(s.edges))
s.t = b.samples2msec(numpy.arange(nsamp))
import pylab
pylab.clf()
for i, s in enumerate(out):
pylab.subplot(1, len(out), i + 1)
y = y = range(1, len(s.img) + 1)
SigTools.imagesc(s.img, x=s.t, y=y, aspect='auto', **kwargs)
xl, yl = pylab.xlim(), pylab.ylim()
pylab.plot(s.edges, y, 'w*', markersize=10)
pylab.xlim(xl)
pylab.ylim(yl)
pylab.grid('on')
#pylab.ylim([len(s.img)+0.5,0.5]) # this corrupts the image!!
pylab.draw()
if save:
pylab.gcf().savefig(save, orientation='portrait')
return out
def bci2000chain(datfile,
chain='SpectralSignalProcessing',
parms=(),
dims='auto',
start=None,
duration=None,
verbose=False,
keep=False,
binpath=None,
**kwargs):
"""
This is a provisional Python equivalent for tools/matlab/bci2000chain.m
Excuse the relative paths - this example works if you're currently working in the root
directory of the BCI2000 distro:
bci2000chain(datfile='data/samplefiles/eeg3_2.dat',
chain='TransmissionFilter|SpatialFilter|ARFilter',
binpath='tools/cmdline',
parms=['tools/matlab/ExampleParameters.prm'], SpatialFilterType=3)
TODO: documentation
For clues, see http://www.bci2000.org/wiki/index.php/User_Reference:Matlab_Tools
and matlab documentation in bci2000chain.m
Most arguments are like the flags in the Matlab equivalent. mutatis to a large extent mutandis.
Note that for now there is no global way of managing the system path. Either add the
tools/cmdline directory to your $PATH variable before starting Python, or supply its location
every time while calling, in the <binpath> argument.
TODO: test on Windoze
"""###
if isinstance(chain, basestring):
if chain.lower() == 'SpectralSignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|SpectralEstimator|LinearClassifier|LPFilter|ExpressionFilter|Normalizer'
elif chain.lower() == 'ARSignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|ARFilter|LinearClassifier|LPFilter|ExpressionFilter|Normalizer'
elif chain.lower() == 'P3SignalProcessing'.lower():
chain = 'TransmissionFilter|SpatialFilter|P3TemporalFilter|LinearClassifier'
chain = chain.split('|')
chain = [c.strip() for c in chain if len(c.strip())]
if len(chain) == 0: print('WARNING: chain is empty')
if start != None and len(str(start).strip()):
start = ' --start=' + str(start).replace(' ', '')
else:
start = ''
if duration != None and len(str(duration).strip()):
duration = ' --duration=' + str(duration).replace(' ', '')
else:
duration = ''
if dims == None or str(dims).lower() == 'auto': dims = 0
if dims not in (0, 2, 3): raise ValueError("dims must be 2, 3 or 'auto'")
out = SigTools.sstruct()
err = ''
cmd = ''
binaries = []
tmpdir = tempfile.mkdtemp(prefix='bci2000chain_')
tmpdatfile = os.path.join(tmpdir, 'in.dat')
prmfile_in = os.path.join(tmpdir, 'in.prm')
prmfile_out = os.path.join(tmpdir, 'out.prm')
matfile = os.path.join(tmpdir, 'out.mat')
bcifile = os.path.join(tmpdir, 'out.bci')
shfile = os.path.join(tmpdir, 'go.bat')
logfile = os.path.join(tmpdir, 'log.txt')
if not isinstance(datfile, basestring):
raise ValueError(
'datfile must be a filename'
) # TODO: if datfile contains the appropriate info, use some create_bcidat equivalent and do datfile = tmpdatfile
if not os.path.isfile(datfile):
raise IOError('file not found: %s' % datfile)
mappings = {
'$DATFILE': datfile,
'$PRMFILE_IN': prmfile_in,
'$PRMFILE_OUT': prmfile_out,
'$MATFILE': matfile,
'$BCIFILE': bcifile,
'$SHFILE': shfile,
'$LOGFILE': logfile,
}
def exe(name):
if binpath: return os.path.join(binpath, name)
else: return name
if parms == None: parms = []
if isinstance(parms, tuple): parms = list(parms)
if not isinstance(parms, list): parms = [parms]
else: parms = list(parms)
if len(kwargs): parms.append(kwargs)
if parms == None or len(parms) == 0:
cmd += exe('bci_dat2stream') + start + duration + ' < "$DATFILE"'
binaries.append('bci_dat2stream')
else:
if verbose: print '# writing custom parameter file %s' % prmfile_in
parms = make_bciprm(datfile, parms, '>', prmfile_in)
if dat2stream_has_p_flag:
cmd += exe(
'bci_dat2stream'
) + ' -p$PRMFILE_IN' + start + duration + ' < "$DATFILE"' # new-style bci_dat2stream with -p option
binaries.append('bci_dat2stream')
else:
if len(start) or len(duration):
raise ValueError(
'old versionsof bci_dat2stream have no --start or --duration option'
)
cmd += '(' # old-style bci_dat2stream with no -p option
cmd += exe('bci_prm2stream') + ' < $PRMFILE_IN'
cmd += '&& ' + exe('bci_dat2stream') + ' --transmit-sd < $DATFILE'
cmd += ')'
binaries.append('bci_dat2stream')
binaries.append('bci_prm2stream')
for c in chain:
cmd += ' | ' + exe(c)
binaries.append(c)
if stream2mat_saves_parms:
cmd += ' | ' + exe(
'bci_stream2mat'
) + ' > $MATFILE' # new-style bci_stream2mat with Parms output
binaries.append('bci_stream2mat')
else:
cmd += ' > $BCIFILE'
cmd += ' && ' + exe('bci_stream2mat') + ' < $BCIFILE > $MATFILE'
cmd += ' && ' + exe(
'bci_stream2prm'
) + ' < $BCIFILE > $PRMFILE_OUT' # old-style bci_stream2mat without Parms output
binaries.append('bci_stream2mat')
binaries.append('bci_stream2prm')
for k, v in mappings.items():
cmd = cmd.replace(k, v)
win = sys.platform.lower().startswith('win')
if win: shebang = ''
else: shebang = '#!/bin/sh\n\n'
open(shfile, 'wt').write(shebang + cmd + '\n')
if not win: os.chmod(shfile, 484) # rwxr--r--
def tidytext(x):
return x.strip().replace('\r\n', '\n').replace('\r', '\n')
def getstatusoutput(cmd):
pipe = os.popen(
cmd + ' 2>&1', 'r'
) # TODO: does this work on Windows? could always make use of logfile here if not
output = pipe.read()
status = pipe.close()
if status == None: status = 0
return status, tidytext(output)
def getoutput(cmd):
return getstatusoutput(cmd)[1]
if verbose: print '# querying version information'
binaries = SigTools.sstruct([
(bin, getoutput(exe(bin) + ' --version').replace('\n', ' '))
for bin in binaries
])
if verbose: print cmd
t0 = time.time()
failed, output = getstatusoutput(shfile)
chaintime = time.time() - t0
failsig = 'Configuration Error: '
if failsig in output: failed = 1
printable_output = output
printable_lines = output.split('\n')
maxlines = 10
if len(printable_lines) > maxlines:
printable_output = '\n'.join(
printable_lines[:maxlines] +
['[%d more lines omitted]' % (len(printable_lines) - maxlines)])
if failed:
if verbose:
err = 'system call failed:\n' + printable_output # cmd has already been printed, so don't clutter things further
else:
err = 'system call failed:\n%s\n%s' % (cmd, printable_output)
if err == '':
if verbose: print '# loading %s' % matfile
try:
mat = SigTools.loadmat(matfile)
except:
err = "The chain must have failed: could not load %s\nShell output was as follows:\n%s" % (
matfile, printable_output)
else:
if 'Data' not in mat:
err = "The chain must have failed: no 'Data' variable found in %s\nShell output was as follows:\n%s" % (
matfile, printable_output)
if 'Index' not in mat:
err = "The chain must have failed: no 'Index' variable found in %s\nShell output was as follows:\n%s" % (
matfile, printable_output)
if err == '':
out.FileName = datfile
if stream2mat_saves_parms:
if verbose: print '# decoding parameters loaded from the mat-file'
parms = make_bciprm(mat.Parms[0])
else:
if verbose: print '# reading output parameter file' + prmfile_out
parms = ParmList(
prmfile_out
) # if you get an error that prmfile_out does not exist, recompile your bci_dat2stream and bci_stream2mat binaries from up-to-date sources, and ensure that dat2stream_has_p_flag and stream2mat_saves_parms, at the top of this file, are both set to 1
out.DateStr = read_bcidate(parms, 'ISO')
out.DateNum = read_bcidate(parms)
out.FilterChain = chain
out.ToolVersions = binaries
out.ShellInput = cmd
out.ShellOutput = output
out.ChainTime = chaintime
out.ChainSpeedFactor = None
out.Megabytes = None
out.Parms = parms
out.Parms.sort()
mat.Index = mat.Index[0, 0]
sigind = mat.Index.Signal - 1 # indices vary across channels fastest, then elements
nChannels, nElements = sigind.shape
nBlocks = mat.Data.shape[1]
out.Blocks = nBlocks
out.BlocksPerSecond = float(parms.SamplingRate.ScaledValue) / float(
parms.SampleBlockSize.ScaledValue)
out.SecondsPerBlock = float(parms.SampleBlockSize.ScaledValue) / float(
parms.SamplingRate.ScaledValue)
out.ChainSpeedFactor = float(out.Blocks * out.SecondsPerBlock) / float(
out.ChainTime)
def unnumpify(x):
while isinstance(x, numpy.ndarray) and x.size == 1:
x = x[0]
if isinstance(x, (numpy.ndarray, tuple, list)):
x = [unnumpify(xi) for xi in x]
return x
out.Channels = nChannels
out.ChannelLabels = unnumpify(mat.get('ChannelLabels', []))
try:
out.ChannelSet = SigTools.ChannelSet(out.ChannelLabels)
except:
out.ChannelSet = None
out.Elements = nElements
out.ElementLabels = unnumpify(mat.get('ElementLabels', []))
out.ElementValues = numpy.ravel(mat.get('ElementValues', []))
out.ElementUnit = unnumpify(mat.get('ElementUnit', None))
out.ElementRate = out.BlocksPerSecond * out.Elements
out.Time = out.SecondsPerBlock * numpy.arange(0, nBlocks)
out.FullTime = out.Time
out.FullElementValues = out.ElementValues
# Why does sigind have to be transposed before vectorizing to achieve the same result as sigind(:) WITHOUT a transpose in Matlab?
# This will probably forever be one of the many deep mysteries of numpy dimensionality handling
out.Signal = mat.Data[
sigind.T.ravel(), :] # nChannels*nElements - by - nBlocks
out.Signal = out.Signal + 0.0 # make a contiguous copy
def seconds(
s
): # -1 means "no information", 0 means "not units of time", >0 means the scaling factor
if getattr(s, 'ElementUnit', None) in ('', None): return -1
s = s.ElementUnit
if s.endswith('seconds'): s = s[:-6]
elif s.endswith('second'): s = s[:-5]
elif s.endswith('sec'): s = s[:-2]
if s.endswith('s'):
return {
'ps': 1e-12,
'ns': 1e-9,
'us': 1e-6,
'mus': 1e-6,
'ms': 1e-3,
's': 1e+0,
'ks': 1e+3,
'Ms': 1e+6,
'Gs': 1e+9,
'Ts': 1e+12,
}.get(s, 0)
else:
return 0
if dims == 0: # dimensionality has not been specified explicitly: so guess, based on ElementUnit and/or filter name
# 3-dimensional output makes more sense than continuous 2-D whenever "elements" can't just be concatenated into an unbroken time-stream
if len(chain): lastfilter = chain[-1].lower()
else: lastfilter = ''
if lastfilter == 'p3temporalfilter':
dims = 3
else:
factor = seconds(out)
if factor > 0: # units of time. TODO: could detect whether the out.ElementValues*factor are (close enough to) contiguous from block to block; then p3temporalfilter wouldn't have to be a special case above
dims = 2
elif factor == 0: # not units of time: use 3D by default
dims = 3
elif lastfilter in [
'p3temporalfilter', 'arfilter', 'fftfilter',
'coherencefilter', 'coherencefftfilter'
]: # no ElementUnit info? guess based on filter name
dims = 3
else:
dims = 2
if dims == 3:
out.Signal = numpy.reshape(
out.Signal, (nChannels, nElements, nBlocks),
order='F') # nChannels - by - nElements - by - nBlocks
out.Signal = numpy.transpose(
out.Signal,
(2, 0, 1)) # nBlocks - by - nChannels - by - nElements
out.Signal = out.Signal + 0.0 # make a contiguous copy
elif dims == 2:
out.FullTime = numpy.repeat(out.Time, nElements)
factor = seconds(out)
if len(out.ElementValues):
out.FullElementValues = numpy.tile(out.ElementValues, nBlocks)
if factor > 0:
out.FullTime = out.FullTime + out.FullElementValues * factor
out.Signal = numpy.reshape(out.Signal,
(nChannels, nElements * nBlocks),
order='F') # nChannels - by - nSamples
out.Signal = numpy.transpose(out.Signal,
(1, 0)) # nSamples - by - nChannels
out.Signal = out.Signal + 0.0 # make a contiguous copy
else:
raise RuntimeError('internal error')
out.States = SigTools.sstruct()
states = [(k, int(getattr(mat.Index, k)) - 1)
for k in mat.Index._fieldnames if k != 'Signal']
for k, v in states:
setattr(out.States, k, mat.Data[v, :])
# TODO: how do the command-line tools handle event states? this seems to be set up to deliver one value per block whatever kind of state we're dealing with
out.Megabytes = megs(out)
else:
out = err
keep = True
print err
print
if os.path.isdir(tmpdir):
files = sorted([
os.path.join(tmpdir, file) for file in os.listdir(tmpdir)
if file not in ['.', '..']
])
if keep:
print 'The following commands should be executed to clean up the temporary files:'
elif verbose:
print '# removing temp files and directory ' + tmpdir
for file in files:
if keep: print " os.remove('%s')" % file
else: os.remove(file)
if keep: print " os.rmdir('%s')" % tmpdir
else: os.rmdir(tmpdir)
def TimingWindow(filename='.', ind=-1, save=None):
"""
Recreate BCI2000's timing window offline, from a saved .dat file specified by <filename>.
It is also possible to supply a directory name as <filename>, and an index <ind> (default
value -1 meaning "the last run") to choose a file automatically from that directory.
Based on BCI2000's src/shared/modules/signalsource/DataIOFilter.cpp where the timing window
content is computed in DataIOFilter::Process(), this is what appears to happen:
Begin SampleBlock #t:
Enter SignalSource module's first Process() method (DataIOFilter::Process)
Save previous SampleBlock to file
Wait to acquire new SampleBlock from hardware
+--------- Measure SourceTime in SignalSource module
| | | Make a local copy of all states (NB: all except SourceTime were set during #t-1) ---+
B| R| S| Pipe the signal through the rest of BCI2000 |
| | +- Measure StimulusTime in Application module, on leaving last Process() method |
| | |
| | |
| | Begin SampleBlock #t+1: |
| +----- Enter SignalSource module's first Process() method (DataIOFilter::Process) |
| Save data from #t, SourceTime state from #t, and other states from #t-1, to file <--+
| Wait to acquire new SampleBlock from hardware
+--------- Measure SourceTime in SignalSource module
Make a local copy of all states (NB: all except SourceTime were set during #t)
Leave DataIOFilter::Process() and pipe the signal through the rest of BCI2000
Measure StimulusTime in Application module, on leaving last Process() method
B stands for Block duration.
R stands for Roundtrip time (visible in VisualizeTiming, not reconstructable from the .dat file)
S is the filter cascade time (marked "Stimulus" in the VisualizeTiming window).
Note that, on any given SampleBlock as saved in the file, SourceTime will be *greater* than
any other timestamp states (including StimulusTime), because it is the only state that is
updated in time to be saved with the data packet it belongs to. All the others lag by one
packet. This is corrected for at the point commented with ??? in the Python code.
"""
if hasattr(filename, 'filename'): filename = filename.filename
b = bcistream(filename=filename, ind=ind)
out = SigTools.sstruct()
out.filename = b.filename
#print "decoding..."
sig,states = b.decode('all')
#print "done"
b.close()
dT,T,rT = {},{},{}
statenames = ['SourceTime', 'StimulusTime'] + ['PythonTiming%02d' % (x+1) for x in range(2)]
statenames = [s for s in statenames if s in states]
for key in statenames:
dT[key],T[key] = SigTools.unwrapdiff(states[key].flatten(), base=65536, dtype=numpy.float64)
sel, = numpy.where(dT['SourceTime'])
for key in statenames:
dT[key] = dT[key][sel[1:]]
if key == 'SourceTime': tsel = sel[:-1] # ??? why the shift
else: tsel = sel[1:] # ??? relative to here?
T[key] = T[key][tsel+1]
t0 = T['SourceTime'][0]
for key in statenames: T[key] -= t0
t = T['SourceTime'] / 1000
expected = b.samples2msec(b.params['SampleBlockSize'])
datestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(b.datestamp))
paramstr = ', '.join(['%s=%s' % (x,b.params[x]) for x in ['SampleBlockSize', 'SamplingRate', 'VisualizeTiming', 'VisualizeSource']])
chainstr = '-'.join([x for x,y in b.params['SignalSourceFilterChain']+b.params['SignalProcessingFilterChain']+b.params['ApplicationFilterChain']])
titlestr = '\n'.join([b.filename, datestamp, paramstr, chainstr])
SigTools.plot(t[[0,-1]], [expected]*2, drawnow=False)
SigTools.plot(t, dT['SourceTime'], hold=True, drawnow=False)
for key in statenames:
if key == 'SourceTime': continue
rT[key] = T[key] - T['SourceTime']
SigTools.plot(t, rT[key], hold=True, drawnow=False)
import pylab
pylab.title(titlestr)
pylab.grid(True)
pylab.xlabel('seconds')
pylab.ylabel('milliseconds')
ymin,ymax = pylab.ylim(); pylab.ylim(ymax=max(ymax,expected*2))
pylab.xlim(xmax=t[-1])
pylab.draw()
out.params = SigTools.sstruct(b.params)
out.summarystr = titlestr
out.t = t
out.SourceTime = T['SourceTime']
out.StimulusTime = T['StimulusTime']
out.BlockDuration = dT['SourceTime']
out.BlockDuration2 = dT['StimulusTime']
out.ProcessingTime = out.StimulusTime - out.SourceTime
out.ExpectedBlockDuration = expected
out.rT = rT
out.dT = dT
out.T = T
if save:
pylab.gcf().savefig(save, orientation='landscape')
return out
def StimulusTiming(filename='.', ind=None, channels=0, trigger='StimulusCode > 0', msec=200, rectify=False, threshold=0.5, use_eo=True, save=None, **kwargs):
"""
In <filename> and <ind>, give it
- a directory and ind=None: for all .dat files in the directory, in session/run order
- a directory and ind=an index or list of indices: for selected .dat files in the directory
- a dat-file name and ind=anything: for that particular file
- a list of filenames and ind=anything: for certain explicitly-specified files
<channels> may be a 0-based index, list of indices, list of channel names, or space- or comma-
delimited string of channel names
<rectify> subtracts the median and takes the abs before doing anything else
<threshold> is on the normalized scale of min=0, max=1 within the resulting image
<use_eo> uses the EventOffset state to correct timings
"""###
if hasattr(filename, 'filename'): filename = filename.filename
if ind==None:
ind = -1
if os.path.isdir(filename): filename = ListDatFiles(filename)
if not isinstance(filename, (tuple,list)): filename = [filename]
if not isinstance(ind, (tuple,list)): ind = [ind]
n = max(len(filename), len(ind))
if len(filename) == 1: filename = list(filename) * n
if len(ind) == 1: ind = list(ind) * n
if isinstance(channels, basestring): channels = channels.replace(',', ' ').split()
if not isinstance(channels, (tuple,list)): channels = [channels]
out = [SigTools.sstruct(
files=[],
events=[],
t=None,
channel=ch,
img=[],
edges=[],
threshold=None,
EventOffsets=[],
UseEventOffsets=False,
) for ch in channels]
if len(filename) == 0: raise ValueError("no data files specified")
for f,i in zip(filename, ind):
b = bcistream(filename=f, ind=i)
nsamp = b.msec2samples(msec)
sig,st = b.decode('all')
statenames = zip(*sorted([(-len(x),x) for x in st]))[1]
criterion = trigger
for x in statenames: criterion = criterion.replace(x, "st['%s']"%x)
criterion = numpy.asarray(eval(criterion)).flatten()
startind = RisingEdge(criterion).nonzero()[0] + 1
print "%d events found in %s" % (len(startind), b.filename)
for s in out:
s.files.append(b.filename)
s.events.append(len(startind))
ch = s.channel
if isinstance(ch, basestring):
chn = [x.lower() for x in b.params['ChannelNames']]
if ch.lower() in chn: ch = chn.index(ch.lower())
else: raise ValueError("could not find channel %s in %s" % (ch,b.filename))
if len(b.params['ChannelNames']) == len(sig):
s.channel = b.params['ChannelNames'][ch]
xx = numpy.asarray(sig)[ch]
if rectify: xx = numpy.abs(xx - numpy.median(xx))
xx -= xx.min()
if xx.max(): xx /= xx.max()
s.threshold = threshold
for ind in startind:
if 'EventOffset' in st:
eo = st['EventOffset'].flat[ind]
if use_eo:
ind += eo - 2**(b.statedefs['EventOffset']['length']-1)
s.UseEventOffsets = True
else:
eo = 0
s.EventOffsets.append(eo)
x = xx[ind:ind+nsamp].tolist()
x += [0.0] * (nsamp - len(x))
s.img.append(x)
for s in out:
s.img = numpy.asarray(s.img)
s.edges = [min(list(x.nonzero()[0])+[numpy.nan]) for x in (s.img > s.threshold)]
s.edges = b.samples2msec(numpy.asarray(s.edges))
s.t = b.samples2msec(numpy.arange(nsamp))
import pylab
pylab.clf()
for i,s in enumerate(out):
pylab.subplot(1, len(out), i+1)
y = y=range(1,len(s.img)+1)
SigTools.imagesc(s.img, x=s.t, y=y, aspect='auto', **kwargs)
xl,yl = pylab.xlim(),pylab.ylim()
pylab.plot(s.edges, y, 'w*', markersize=10)
pylab.xlim(xl); pylab.ylim(yl)
pylab.grid('on')
#pylab.ylim([len(s.img)+0.5,0.5]) # this corrupts the image!!
pylab.draw()
if save:
pylab.gcf().savefig(save, orientation='portrait')
return out
