def stplot(self, img, fs=None, drawnow=True, **kwargs): #kwargs['colorbar'] = kwargs.get('colorbar', True) if img.shape[0] != self.size: raise ValueError( "number of rows in image should match number of channels (=%d)" % self.size) import SigTools pylab = SigTools.Plotting.load_pylab() t = numpy.arange(img.shape[1], dtype=numpy.float64) if fs == None: x = None xlabel = 'time-sample index' else: x = t = SigTools.samples2msec(t, fs) xlabel = 'time (msec)' x = None pylab.subplot(121) h = SigTools.Plotting.imagesc(img, x=x, aspect='auto', drawnow=False, picker=5, **kwargs) ax = pylab.gca() if x == None and fs != None: xl = ax.get_xlim() xt = numpy.array( [x for x in ax.get_xticks() if min(xl) <= x <= max(xl)]) xtl = ['%g' % x for x in SigTools.samples2msec(xt, fs)] ax.set(xticks=xt, xticklabels=xtl) ax.set( xlabel=xlabel, yticks=range(img.shape[0]), yticklabels=['%d: %s' % x for x in enumerate(self.get_labels())]) ax.grid(True) h.channels = self h.img = numpy.asarray(img) h.timebase = t h.spaceax = pylab.subplot(222) h.timeax = pylab.subplot(224) def onpick(evt): col = int(round(evt.mouseevent.xdata)) row = int(round(evt.mouseevent.ydata)) h = evt.artist pylab.axes(h.spaceax) h.channels.plot(h.img[:, col], clim=h.get_clim(), drawnow=False) pylab.axes(h.timeax) SigTools.plot(h.timebase, h.img[row], drawnow=False) pylab.gca().set(ylim=h.get_clim(), xlim=(h.timebase[0], h.timebase[-1])) pylab.draw() pylab.gcf().canvas.mpl_connect('pick_event', onpick) if drawnow: pylab.draw() return h
def PlotSCD( files='*.pk' ): import pylab if isinstance( files, basestring ): if os.path.isdir( files ): files = os.path.join( files, '*.pk' ) files = glob.glob( files ) d = DataFiles.load( files ) r = SigTools.correlate( d['x'], d['y'], axis=0 ) SigTools.imagesc( r*numpy.abs(r), y=d['channels'], x=SigTools.samples2msec( range( r.shape[1] ), d['fs'] ), aspect='auto', balance=0.0, colorbar=True ) pylab.title( ', '.join( [ '%d: %d' % ( yi, ( d['y'] == yi ).sum() ) for yi in numpy.unique( d['y'] ) ] ) ) pylab.draw()
def PlotTrials( files='*.pk', channel='Cz' ): import pylab if isinstance( files, basestring ): if os.path.isdir( files ): files = os.path.join( files, '*.pk' ) files = glob.glob( files ) d = DataFiles.load( files ) chind = d[ 'channels' ].index( channel ) v = [ ( d[ 'y' ][ i ], i, vi ) for i, vi in enumerate( d[ 'x' ][ :, chind, : ] ) ] v = numpy.array( [ vi for yi, i, vi in sorted( v ) ] ) SigTools.imagesc( v, x=SigTools.samples2msec( range( v.shape[1] ), d['fs'] ), aspect='auto', balance=0.0, colorbar=True ) pylab.title( ', '.join( [ '%d: %d' % ( yi, ( d['y'] == yi ).sum() ) for yi in numpy.unique( d['y'] ) ] ) + ( ' (channel %s)' % channel ) ) pylab.draw()
def stplot(self, img, fs=None, drawnow=True, **kwargs): #kwargs['colorbar'] = kwargs.get('colorbar', True) if img.shape[0] != self.size: raise ValueError("number of rows in image should match number of channels (=%d)" % self.size) import SigTools pylab = SigTools.Plotting.load_pylab() t = numpy.arange(img.shape[1], dtype=numpy.float64) if fs == None: x = None; xlabel = 'time-sample index' else: x = t = SigTools.samples2msec(t, fs); xlabel = 'time (msec)' x = None pylab.subplot(121) h = SigTools.Plotting.imagesc(img, x=x, aspect='auto', drawnow=False, picker=5, **kwargs) ax = pylab.gca() if x == None and fs != None: xl = ax.get_xlim() xt = numpy.array([x for x in ax.get_xticks() if min(xl)<=x<=max(xl)]) xtl = ['%g' % x for x in SigTools.samples2msec(xt, fs)] ax.set(xticks=xt, xticklabels=xtl) ax.set(xlabel=xlabel, yticks=range(img.shape[0]), yticklabels=['%d: %s' % x for x in enumerate(self.get_labels())]) ax.grid(True) h.channels = self h.img = numpy.asarray(img) h.timebase = t h.spaceax = pylab.subplot(222) h.timeax = pylab.subplot(224) def onpick(evt): col = int(round(evt.mouseevent.xdata)) row = int(round(evt.mouseevent.ydata)) h = evt.artist pylab.axes(h.spaceax) h.channels.plot(h.img[:,col], clim=h.get_clim(), drawnow=False) pylab.axes(h.timeax) SigTools.plot(h.timebase, h.img[row], drawnow=False) pylab.gca().set(ylim=h.get_clim(), xlim=(h.timebase[0],h.timebase[-1])) pylab.draw() pylab.gcf().canvas.mpl_connect('pick_event', onpick) if drawnow: pylab.draw() return h
def PlotSCD(files='*.pk'): import pylab if isinstance(files, basestring): if os.path.isdir(files): files = os.path.join(files, '*.pk') files = glob.glob(files) d = DataFiles.load(files) r = SigTools.correlate(d['x'], d['y'], axis=0) SigTools.imagesc(r * numpy.abs(r), y=d['channels'], x=SigTools.samples2msec(range(r.shape[1]), d['fs']), aspect='auto', balance=0.0, colorbar=True) pylab.title(', '.join([ '%d: %d' % (yi, (d['y'] == yi).sum()) for yi in numpy.unique(d['y']) ])) pylab.draw()
def PlotTrials(files='*.pk', channel='Cz'): import pylab if isinstance(files, basestring): if os.path.isdir(files): files = os.path.join(files, '*.pk') files = glob.glob(files) d = DataFiles.load(files) chind = d['channels'].index(channel) v = [(d['y'][i], i, vi) for i, vi in enumerate(d['x'][:, chind, :])] v = numpy.array([vi for yi, i, vi in sorted(v)]) SigTools.imagesc(v, x=SigTools.samples2msec(range(v.shape[1]), d['fs']), aspect='auto', balance=0.0, colorbar=True) pylab.title(', '.join( ['%d: %d' % (yi, (d['y'] == yi).sum()) for yi in numpy.unique(d['y'])]) + (' (channel %s)' % channel)) pylab.draw()
def ClassifyERPs ( featurefiles, C = (10.0, 1.0, 0.1, 0.01), gamma = (1.0, 0.8, 0.6, 0.4, 0.2, 0.0), keepchan = (), rmchan = (), rmchan_usualsuspects = ('AUDL','AUDR','LAUD','RAUD','SYNC','VSYNC', 'VMRK', 'OLDREF'), rebias = True, save = False, select = False, description='ERPs to attended vs unattended events', maxcount=None, classes=None, folds=None, time_window=None, keeptrials=None, ): file_inventory = [] d = DataFiles.load(featurefiles, catdim=0, maxcount=maxcount, return_details=file_inventory) if isinstance(folds, basestring) and folds.lower() in ['lofo', 'loro', 'leave on run out', 'leave one file out']: n, folds = 0, [] for each in file_inventory: neach = each[1]['x'] folds.append(range(n, n+neach)) n += neach if 'x' not in d: raise ValueError("found no trial data - no 'x' variable - in the specified files") if 'y' not in d: raise ValueError("found no trial labels - no 'y' variable - in the specified files") x = d['x'] y = numpy.array(d['y'].flat) if keeptrials != None: x = x[numpy.asarray(keeptrials), :, :] y = y[numpy.asarray(keeptrials)] if time_window != None: fs = d['fs'] t = SigTools.samples2msec(numpy.arange(x.shape[2]), fs) x[:, :, t<min(time_window)] = 0 x[:, :, t>max(time_window)] = 0 if classes != None: for cl in classes: if cl not in y: raise ValueError("class %s is not in the dataset" % str(cl)) mask = numpy.array([yi in classes for yi in y]) y = y[mask] x = x[mask] discarded = sum(mask==False) if discarded: print "discarding %d trials that are outside the requested classes %s"%(discarded,str(classes)) n = len(y) uy = numpy.unique(y) if uy.size != 2: raise ValueError("expected 2 classes in dataset, found %d : %s" % (uy.size, str(uy))) for uyi in uy: nyi = sum([yi==uyi for yi in y]) nyi_min = 2 if nyi < nyi_min: raise ValueError('only %d exemplars of class %s - need at least %d' % (nyi,str(uyi),nyi_min)) y = numpy.sign(y - uy.mean()) cov,trchvar = SigTools.spcov(x=x, y=y, balance=False, return_trchvar=True) # NB: symwhitenkern would not be able to balance starttime = time.time() chlower = [ch.lower() for ch in d['channels']] if keepchan in [None,(),'',[]]: if isinstance(rmchan, basestring): rmchan = rmchan.split() if isinstance(rmchan_usualsuspects, basestring): rmchan_usualsuspects = rmchan_usualsuspects.split() allrmchan = [ch.lower() for ch in list(rmchan)+list(rmchan_usualsuspects)] unwanted = numpy.array([ch in allrmchan for ch in chlower]) notfound = [ch for ch in rmchan if ch.lower() not in chlower] else: if isinstance(keepchan, basestring): keepchan = keepchan.split() lowerkeepchan = [ch.lower() for ch in keepchan] unwanted = numpy.array([ch not in lowerkeepchan for ch in chlower]) notfound = [ch for ch in keepchan if ch.lower() not in chlower] wanted = numpy.logical_not(unwanted) print ' ' if len(notfound): print "WARNING: could not find channel%s %s\n" % ({1:''}.get(len(notfound),'s'), ', '.join(notfound)) removed = [ch for removing,ch in zip(unwanted, d['channels']) if removing] if len(removed): print "removed %d channel%s (%s)" % (len(removed), {1:''}.get(len(removed),'s'), ', '.join(removed)) print "classification will be based on %d channel%s" % (sum(wanted), {1:''}.get(sum(wanted),'s')) print "%d negatives + %d positives = %d exemplars" % (sum(y<0), sum(y>0), n) print ' ' x[:, unwanted, :] = 0 cov[:, unwanted] = 0 cov[unwanted, :] = 0 nu = numpy.asarray(cov).diagonal()[wanted].mean() for i in range(len(cov)): if cov[i,i] == 0: cov[i,i] = nu if not isinstance(C, (tuple,list,numpy.ndarray,type(None))): C = [C] if not isinstance(gamma, (tuple,list,numpy.ndarray,type(None))): gamma = [gamma] c = SigTools.klr2class(lossfunc=SigTools.balanced_loss, relcost='balance') c.varyhyper({}) if c != None: c.hyper.C=list(C) if gamma == None: c.hyper.kernel.func = SigTools.linkern else: c.varyhyper({'kernel.func':SigTools.symwhitenkern, 'kernel.cov':[cov], 'kernel.gamma':list(gamma)}) c.cvtrain(x=x, y=y, folds=folds) if rebias: c.rebias() c.calibrate() chosen = c.cv.chosen.hyper if gamma == None: Ps = None Gp = c.featureweight(x=x) else: Ps = SigTools.svd(SigTools.shrinkcov(cov, copy=True, gamma=chosen.kernel.gamma)).isqrtm xp = SigTools.spfilt(x, Ps.H, copy=True) Gp = c.featureweight(x=xp) u = SigTools.stfac(Gp, Ps) u.channels = d['channels'] u.channels_used = wanted u.fs = d['fs'] u.trchvar = trchvar try: u.channels = SigTools.ChannelSet(u.channels) except: print 'WARNING: failed to convert channels to ChannelSet' elapsed = time.time() - starttime minutes = int(elapsed/60.0) seconds = int(round(elapsed - minutes * 60.0)) print '%d min %d sec' % (minutes, seconds) datestamp = time.strftime('%Y-%m-%d %H:%M:%S') csummary = '%s (%s) trained on %d (CV %s = %.3f) at %s' % ( c.__class__.__name__, SigTools.experiment()._shortdesc(chosen), sum(c.input.istrain), c.loss.func.__name__, c.loss.train, datestamp, ) description = 'binary classification of %s: %s' % (description, csummary) u.description = description if save or select: if not isinstance(save, basestring): save = featurefiles if isinstance(save, (tuple,list)): save = save[-1] if save.lower().endswith('.gz'): save = save[:-3] if save.lower().endswith('.pk'): save = save[:-3] save = save + '_weights.prm' print "\nsaving %s\n" % save Parameters.Param(u.G.A, Name='ERPClassifierWeights', Section='PythonSig', Subsection='Epoch', Comment=csummary).write_to(save) Parameters.Param(c.model.bias, Name='ERPClassifierBias', Section='PythonSig', Subsection='Epoch', Comment=csummary).append_to(save) Parameters.Param(description, Name='SignalProcessingDescription', Section='PythonSig').append_to(save) if select: if not isinstance(select, basestring): select = 'ChosenWeights.prm' if not os.path.isabs(select): select = os.path.join(os.path.split(save)[0], select) print "saving %s\n" % select import shutil; shutil.copyfile(save, select) print description return u,c
def ClassifyERPs( featurefiles, C=(10.0, 1.0, 0.1, 0.01), gamma=(1.0, 0.8, 0.6, 0.4, 0.2, 0.0), keepchan=(), rmchan=(), rmchan_usualsuspects=('AUDL', 'AUDR', 'LAUD', 'RAUD', 'SYNC', 'VSYNC', 'VMRK', 'OLDREF'), rebias=True, save=False, select=False, description='ERPs to attended vs unattended events', maxcount=None, classes=None, folds=None, time_window=None, keeptrials=None, ): file_inventory = [] d = DataFiles.load(featurefiles, catdim=0, maxcount=maxcount, return_details=file_inventory) if isinstance(folds, basestring) and folds.lower() in [ 'lofo', 'loro', 'leave on run out', 'leave one file out' ]: n, folds = 0, [] for each in file_inventory: neach = each[1]['x'] folds.append(range(n, n + neach)) n += neach if 'x' not in d: raise ValueError( "found no trial data - no 'x' variable - in the specified files") if 'y' not in d: raise ValueError( "found no trial labels - no 'y' variable - in the specified files") x = d['x'] y = numpy.array(d['y'].flat) if keeptrials != None: x = x[numpy.asarray(keeptrials), :, :] y = y[numpy.asarray(keeptrials)] if time_window != None: fs = d['fs'] t = SigTools.samples2msec(numpy.arange(x.shape[2]), fs) x[:, :, t < min(time_window)] = 0 x[:, :, t > max(time_window)] = 0 if classes != None: for cl in classes: if cl not in y: raise ValueError("class %s is not in the dataset" % str(cl)) mask = numpy.array([yi in classes for yi in y]) y = y[mask] x = x[mask] discarded = sum(mask == False) if discarded: print "discarding %d trials that are outside the requested classes %s" % ( discarded, str(classes)) n = len(y) uy = numpy.unique(y) if uy.size != 2: raise ValueError("expected 2 classes in dataset, found %d : %s" % (uy.size, str(uy))) for uyi in uy: nyi = sum([yi == uyi for yi in y]) nyi_min = 2 if nyi < nyi_min: raise ValueError( 'only %d exemplars of class %s - need at least %d' % (nyi, str(uyi), nyi_min)) y = numpy.sign(y - uy.mean()) cov, trchvar = SigTools.spcov( x=x, y=y, balance=False, return_trchvar=True) # NB: symwhitenkern would not be able to balance starttime = time.time() chlower = [ch.lower() for ch in d['channels']] if keepchan in [None, (), '', []]: if isinstance(rmchan, basestring): rmchan = rmchan.split() if isinstance(rmchan_usualsuspects, basestring): rmchan_usualsuspects = rmchan_usualsuspects.split() allrmchan = [ ch.lower() for ch in list(rmchan) + list(rmchan_usualsuspects) ] unwanted = numpy.array([ch in allrmchan for ch in chlower]) notfound = [ch for ch in rmchan if ch.lower() not in chlower] else: if isinstance(keepchan, basestring): keepchan = keepchan.split() lowerkeepchan = [ch.lower() for ch in keepchan] unwanted = numpy.array([ch not in lowerkeepchan for ch in chlower]) notfound = [ch for ch in keepchan if ch.lower() not in chlower] wanted = numpy.logical_not(unwanted) print ' ' if len(notfound): print "WARNING: could not find channel%s %s\n" % ({ 1: '' }.get(len(notfound), 's'), ', '.join(notfound)) removed = [ch for removing, ch in zip(unwanted, d['channels']) if removing] if len(removed): print "removed %d channel%s (%s)" % (len(removed), { 1: '' }.get(len(removed), 's'), ', '.join(removed)) print "classification will be based on %d channel%s" % (sum(wanted), { 1: '' }.get(sum(wanted), 's')) print "%d negatives + %d positives = %d exemplars" % (sum(y < 0), sum(y > 0), n) print ' ' x[:, unwanted, :] = 0 cov[:, unwanted] = 0 cov[unwanted, :] = 0 nu = numpy.asarray(cov).diagonal()[wanted].mean() for i in range(len(cov)): if cov[i, i] == 0: cov[i, i] = nu if not isinstance(C, (tuple, list, numpy.ndarray, type(None))): C = [C] if not isinstance(gamma, (tuple, list, numpy.ndarray, type(None))): gamma = [gamma] c = SigTools.klr2class(lossfunc=SigTools.balanced_loss, relcost='balance') c.varyhyper({}) if c != None: c.hyper.C = list(C) if gamma == None: c.hyper.kernel.func = SigTools.linkern else: c.varyhyper({ 'kernel.func': SigTools.symwhitenkern, 'kernel.cov': [cov], 'kernel.gamma': list(gamma) }) c.cvtrain(x=x, y=y, folds=folds) if rebias: c.rebias() c.calibrate() chosen = c.cv.chosen.hyper if gamma == None: Ps = None Gp = c.featureweight(x=x) else: Ps = SigTools.svd( SigTools.shrinkcov(cov, copy=True, gamma=chosen.kernel.gamma)).isqrtm xp = SigTools.spfilt(x, Ps.H, copy=True) Gp = c.featureweight(x=xp) u = SigTools.stfac(Gp, Ps) u.channels = d['channels'] u.channels_used = wanted u.fs = d['fs'] u.trchvar = trchvar try: u.channels = SigTools.ChannelSet(u.channels) except: print 'WARNING: failed to convert channels to ChannelSet' elapsed = time.time() - starttime minutes = int(elapsed / 60.0) seconds = int(round(elapsed - minutes * 60.0)) print '%d min %d sec' % (minutes, seconds) datestamp = time.strftime('%Y-%m-%d %H:%M:%S') csummary = '%s (%s) trained on %d (CV %s = %.3f) at %s' % ( c.__class__.__name__, SigTools.experiment()._shortdesc(chosen), sum(c.input.istrain), c.loss.func.__name__, c.loss.train, datestamp, ) description = 'binary classification of %s: %s' % (description, csummary) u.description = description if save or select: if not isinstance(save, basestring): save = featurefiles if isinstance(save, (tuple, list)): save = save[-1] if save.lower().endswith('.gz'): save = save[:-3] if save.lower().endswith('.pk'): save = save[:-3] save = save + '_weights.prm' print "\nsaving %s\n" % save Parameters.Param(u.G.A, Name='ERPClassifierWeights', Section='PythonSig', Subsection='Epoch', Comment=csummary).write_to(save) Parameters.Param(c.model.bias, Name='ERPClassifierBias', Section='PythonSig', Subsection='Epoch', Comment=csummary).append_to(save) Parameters.Param(description, Name='SignalProcessingDescription', Section='PythonSig').append_to(save) if select: if not isinstance(select, basestring): select = 'ChosenWeights.prm' if not os.path.isabs(select): select = os.path.join(os.path.split(save)[0], select) print "saving %s\n" % select import shutil shutil.copyfile(save, select) print description return u, c