def subtract_residual_obs(self):
def obs_fit_error(p, y, x):
err = n.zeros((x.shape[0]), "d")
err += p[0]
for i in range(5):
err += y[:, i] * p[i + 1]
err -= x
return abs(err**2).sum()
assert self.Ya2 != None, "Not yet ready for subtracting residuals. Need to subtract average template first (self.subtract_mean_from_Ya2)"
self.res_artifacts = n.zeros(
(self._len * self.upsf, len(self._ups_ts)), n.float64)
#self.res_artifacts = n.memmap("tmp/res_art.np",mode="w+",dtype=n.float64,shape=(self._len*self.upsf,len(self._ups_ts)))
#print "Memmapping residual-artifacts"
for i, t in enumerate(self._ups_ts):
if i % 1000 == 0:
print i,
self.res_artifacts[:, i] = self.Ya2[t:t + self._len * self.upsf]
#take only 200 artifacts in for obs-calculation
idx_ups_ts = range(len(self._ups_ts))
shuffle(idx_ups_ts)
bsl = n.zeros((self.res_artifacts.shape[1]), n.bool)
for i in range(200):
bsl[idx_ups_ts[i]] = True
components = pcafilt.unmix(self.res_artifacts[:, bsl])
for i in range(5):
pylab.plot(components[:, i], label=str(i))
pylab.legend()
self.obs = components[:, :5].copy()
#Iterate over all residual artifacts and fit OBS to it
for i, t in enumerate(self._ups_ts):
p = [0, 0, 0, 0, 0, 0]
p_lsq = fmin(obs_fit_error,
p,
args=(self.obs, self.res_artifacts[:, i]),
maxiter=1e5,
maxfun=1e5) #[0]
print i, t, "p_lsq", p_lsq
#pylab.clf()
#pylab.plot(self.res_artifacts[:,i])
self.fit = n.zeros((self.obs.shape[0]), "d")
self.fit += p_lsq[0]
for j in range(5):
self.fit += self.obs[:, j] * p_lsq[j + 1]
#pylab.plot(self.fit)
#pylab.xlim([10000,15000])
#pylab.savefig("/home/thorsten/tmp/test.png")
#raw_input()
print self.res_artifacts[:, i].mean(), self.fit.mean(),
self.res_artifacts[:, i] -= self.fit
print self.res_artifacts[:, i].mean()
for i, t in enumerate(self._ups_ts):
print i, t, self.Ya2[t:t + self._len * self.upsf].mean(
), self.res_artifacts[:, i].mean(),
self.Ya2[t:t + self._len * self.upsf] = self.res_artifacts[:, i]
print self.Ya2[t:t + self._len * self.upsf].mean()
def cb_Options(self, widget):
if widget == self.bOpt["showCA"]:
self.x = self.varChIn.get_var()
if self.fTypeW["PCA"].get_active():
self.x2 = pcafilt.unmix(self.x)
elif self.fTypeW["ICA"].get_active():
self.x2 = icafilt.unmix(self.x)
if self.compCh==None:
self.compCh = ComponentChooser(self.x2)
else:
self.compCh.show_all()
def cb_Options(self, widget):
if widget == self.bOpt["showCA"]:
self.x = self.varChIn.get_var()
if self.fTypeW["PCA"].get_active():
self.x2 = pcafilt.unmix(self.x)
elif self.fTypeW["ICA"].get_active():
self.x2 = icafilt.unmix(self.x)
if self.compCh == None:
self.compCh = ComponentChooser(self.x2)
else:
self.compCh.show_all()
def subtract_residual_obs(self):
def obs_fit_error(p,y,x):
err = n.zeros((x.shape[0]),"d")
err += p[0]
for i in range(5):
err += y[:,i]*p[i+1]
err -= x
return abs(err**2).sum()
assert self.Ya2 != None, "Not yet ready for subtracting residuals. Need to subtract average template first (self.subtract_mean_from_Ya2)"
self.res_artifacts = n.zeros((self._len*self.upsf,len(self._ups_ts)),n.float64)
#self.res_artifacts = n.memmap("tmp/res_art.np",mode="w+",dtype=n.float64,shape=(self._len*self.upsf,len(self._ups_ts)))
#print "Memmapping residual-artifacts"
for i,t in enumerate(self._ups_ts):
if i%1000==0:
print i,
self.res_artifacts[:,i] = self.Ya2[t:t+self._len*self.upsf]
#take only 200 artifacts in for obs-calculation
idx_ups_ts = range(len(self._ups_ts))
shuffle(idx_ups_ts)
bsl = n.zeros((self.res_artifacts.shape[1]),n.bool)
for i in range(200):
bsl[idx_ups_ts[i]] = True
components = pcafilt.unmix(self.res_artifacts[:,bsl])
for i in range(5):
pylab.plot(components[:,i],label=str(i))
pylab.legend()
self.obs = components[:,:5].copy()
#Iterate over all residual artifacts and fit OBS to it
for i,t in enumerate(self._ups_ts):
p = [0,0,0,0,0,0]
p_lsq = fmin(obs_fit_error,p,args=(self.obs,self.res_artifacts[:,i]),maxiter=1e5,maxfun=1e5)#[0]
print i,t,"p_lsq", p_lsq
#pylab.clf()
#pylab.plot(self.res_artifacts[:,i])
self.fit = n.zeros((self.obs.shape[0]),"d")
self.fit +=p_lsq[0]
for j in range(5):
self.fit += self.obs[:,j]*p_lsq[j+1]
#pylab.plot(self.fit)
#pylab.xlim([10000,15000])
#pylab.savefig("/home/thorsten/tmp/test.png")
#raw_input()
print self.res_artifacts[:,i].mean(), self.fit.mean(),
self.res_artifacts[:,i] -= self.fit
print self.res_artifacts[:,i].mean()
for i,t in enumerate(self._ups_ts):
print i,t,self.Ya2[t:t+self._len*self.upsf].mean(), self.res_artifacts[:,i].mean(),
self.Ya2[t:t+self._len*self.upsf] = self.res_artifacts[:,i]
print self.Ya2[t:t+self._len*self.upsf].mean()
def remove_slice_components(self, width=None):
assert self._t_slices != None, "Before removing slice components, first find the slice-positions!"
if width == None:
width = self._slice_width
print "Removing slice components"
if show_progressbar:
pbar = ProgressBar(self._data.shape[1])
for cn in range(self._data.shape[1]):
if show_progressbar:
pbar.update(cn)
if debug:
print "Channel %i", cn
try:
for i in self._ts:
one_c_slices = n.zeros((width, len(self._t_slices)), "d")
for k, j in enumerate(self._t_slices):
if debug:
print k, i, j
one_c_slices[:, k] = self._data[i + j - width / 2:i +
j + width / 2, cn]
components = pcafilt.unmix(one_c_slices)
for k in range(components.shape[1]):
if debug:
print "Channel %i, Component %i, std %f, mean(abs) %f" % (
cn, k, components[:, k].std(),
abs(components[:, k]).mean()),
if components[:,
k].std() > 100.0 or k < 2: #acc > 0.03:
if debug:
print " removed"
components[:, k] = n.zeros(components[:, k].shape,
"d")
else:
if debug:
print " not removed"
#pylab.ioff()
#pylab.hist(accs,20)
#pylab.show()
one_c_slices = pcafilt.mix(components)
k = 0
for k, j in enumerate(self._t_slices):
self._data[i + j - width / 2:i + j + width / 2,
cn] = one_c_slices[:, k]
except Exception, e:
if debug:
print "Error in remove_slice_components,", e
def remove_common_components(self):
assert self._mean_data != None, "Before removing components, first calulate the mean!"
print "Removing common components"
if show_progressbar:
pbar = ProgressBar(self._data.shape[1])
for cn in range(self._data.shape[1]):
#pbar.progress(cn)
if show_progressbar:
pbar.update(cn)
one_c_data = n.zeros((self._len, len(self._ts)), "d")
for i, t in enumerate(self._ts):
one_c_data[:, i] = self._data[t:t + self._len, cn]
components = pcafilt.unmix(one_c_data)
accs = []
for i in range(components.shape[1]):
acc = abs(
n.corrcoef(self._mean_data[:, cn], components[:, i])[0, 1])
accs.append(acc)
if debug:
print "Channel %i, Component %i, corrcoef %f, std %f, mean(abs) %f" % (
cn, i, acc, components[:, i].std(),
abs(components[:, i]).mean()),
if (components[:, i].std() > 100.0
and i < 10) or i < 2: #acc > 0.03:
if debug:
print " removed"
components[:, i] = n.zeros(components[:, i].shape, "d")
else:
if debug:
print " not removed"
#pylab.ioff()
#pylab.hist(accs,20)
#pylab.show()
one_c_data = pcafilt.mix(components)
for i, t in enumerate(self._ts):
try:
self._data[t:t + self._len, cn] = one_c_data[:, i]
except ValueError, e:
print "Error at t=%i, cannot write data back after PCA, shapes are %s and %s" % (
t, self._data[t:t + self._len,
cn].shape, one_c_data[:, i].shape), e
def remove_slice_components(self,width=None):
assert self._t_slices != None, "Before removing slice components, first find the slice-positions!"
if width==None:
width=self._slice_width
print "Removing slice components"
if show_progressbar:
pbar = ProgressBar(self._data.shape[1])
for cn in range(self._data.shape[1]):
if show_progressbar:
pbar.update(cn)
if debug:
print "Channel %i", cn
try:
for i in self._ts:
one_c_slices = n.zeros((width,len(self._t_slices)),"d")
for k,j in enumerate(self._t_slices):
if debug:
print k,i,j
one_c_slices[:,k] = self._data[i+j-width/2:i+j+width/2,cn]
components = pcafilt.unmix(one_c_slices)
for k in range(components.shape[1]):
if debug:
print "Channel %i, Component %i, std %f, mean(abs) %f" % (cn,k,components[:,k].std(), abs(components[:,k]).mean()),
if components[:,k].std()>100.0 or k<2:#acc > 0.03:
if debug:
print " removed"
components[:,k] = n.zeros(components[:,k].shape,"d")
else:
if debug:
print " not removed"
#pylab.ioff()
#pylab.hist(accs,20)
#pylab.show()
one_c_slices = pcafilt.mix(components)
k=0
for k,j in enumerate(self._t_slices):
self._data[i+j-width/2:i+j+width/2,cn] = one_c_slices[:,k]
except Exception,e:
if debug:
print "Error in remove_slice_components,", e
def remove_common_components(self):
assert self._mean_data != None, "Before removing components, first calulate the mean!"
print "Removing common components"
if show_progressbar:
pbar = ProgressBar(self._data.shape[1])
for cn in range(self._data.shape[1]):
#pbar.progress(cn)
if show_progressbar:
pbar.update(cn)
one_c_data = n.zeros((self._len,len(self._ts)),"d")
for i,t in enumerate(self._ts):
one_c_data[:,i] = self._data[t:t+self._len,cn]
components = pcafilt.unmix(one_c_data)
accs = []
for i in range(components.shape[1]):
acc = abs(n.corrcoef(self._mean_data[:,cn],components[:,i])[0,1])
accs.append(acc)
if debug:
print "Channel %i, Component %i, corrcoef %f, std %f, mean(abs) %f" % (cn,i,acc, components[:,i].std(), abs(components[:,i]).mean()),
if (components[:,i].std()>100.0 and i<10) or i<2:#acc > 0.03:
if debug:
print " removed"
components[:,i] = n.zeros(components[:,i].shape,"d")
else:
if debug:
print " not removed"
#pylab.ioff()
#pylab.hist(accs,20)
#pylab.show()
one_c_data = pcafilt.mix(components)
for i,t in enumerate(self._ts):
try:
self._data[t:t+self._len,cn] = one_c_data[:,i]
except ValueError, e:
print "Error at t=%i, cannot write data back after PCA, shapes are %s and %s"%(t,self._data[t:t+self._len,cn].shape,one_c_data[:,i].shape), e
def remove_artifacts(self):
"""Tries to remove the artifacts.
Uses mixture of mean-subtraction and OBS-PCA-Subtraction
Needs self.slcs to be set.
Saves example data in self.examples to make evaluation of filter quality possible"""
def obs_fit_error(p,y,x):
err = n.zeros((x.shape[0]),"d")
err += p[0]
for i in range(self.obs_size):
err += y[:,i]*p[i+1]
err -= x
return abs(err**2).sum()
def obs_fit_error_lsq(p,y,x):
err = n.zeros((x.shape[0]),"d")
err += p[0]
for i in range(self.obs_size):
err += y[:,i]*p[i+1]
err -= x
return err
def las_fit_error_lsq(p,y,x):
"""Local artifact subtraction"""
err = p[0]*y
err -= x
return err
#Shortnames
eeg = self.eeg
slcs = self.slcs_ups
sw = self._new_slice_width *10
k = self.num_neighbors
self.examples=[]
num_examples=10
#Loop over channels
if show_progressbar:
pbar = ProgressBar().start()
for i_ch in range(eeg.num_channels):
#Make Highpass-Version of channel
#ch = eeg[:,i_ch].copy()
y , fn = upsample_to_memmap(eeg[:,i_ch],10)
y_hp,fn_hp = tmp_memmap(dtype=y.dtype,shape=y.shape,mode="w+")
y_hp[:] = filtfilt_high(1, y, Fs=10000.0)
y_out,fn_out = tmp_memmap(dtype=y.dtype,shape=y.shape,mode="w+")
y_out[:] = y[:]
#ch_hp = filtfilt_high(1.0,ch,Fs=eeg.Fs)
neighbors = n.zeros((sw+2*self._pa_zeros,k))
#Prefill
for i in range(k):
#print neighbors[:,i].shape, eeg[slcs[i]-sw/2:slcs[i]+sw/2,i_ch].shape
neighbors[:,i] = prepend_append_zeros( y_hp[slcs[i]-sw/2:slcs[i]+sw/2] , self._pa_zeros)
#Loop over slices and filter
next_subst = 0
for i,t in enumerate(slcs):
try:
if i>k/2 and i<len(slcs)-k/2:
neighbors[:,next_subst] = prepend_append_zeros( y_hp[slcs[i+k/2]-sw/2:slcs[i+k/2]+sw/2] , self._pa_zeros)
next_subst+=1
next_subst=next_subst%k
tmp = prepend_append_zeros( y[t-sw/2:t+sw/2] , self._pa_zeros)
#Subtraction
#Shift/scale template
template = neighbors.mean(axis=1)
p = [1.0, 0]
p_las = leastsq(las_fit_error_lsq,p,args=(template,tmp))[0]
#print p_las[0],
tmp -= p_las[0]*template
#Beispieldaten speichern: Teil 1
if i_ch == self.ch_for_slice_alignment:
if i%(len(slcs)/num_examples)==(len(slcs)/num_examples)/2:
print "examples, Teil 1"
example = {}
example["raw"] = prepend_append_zeros(y[t-sw/2:t+sw/2].copy() , self._pa_zeros )
example["mean"] = p_las[0]*template
#OBS-Fit
components = pcafilt.unmix(neighbors) #OBS will be first 5 components
components -= components.mean(axis=0).reshape(1,-1).repeat(components.shape[0],0) #demeaning column-wise
obs = components[:,:self.obs_size].copy()
#Fit OBS to artifact
p = [0]+[0]*self.obs_size
#p_lsq = fmin(obs_fit_error,p,args=(obs,eeg[t-sw/2:t+sw/2,i_ch]),maxiter=1e5,maxfun=1e5)#[0]
p_lsq = leastsq(obs_fit_error_lsq,p,args=(obs,tmp))[0]
#print i,t,"p_lsq", p_lsq
fit = n.zeros((obs.shape[0]),"d")
fit +=p_lsq[0]
for j in range(self.obs_size):
fit += obs[:,j]*p_lsq[j+1]
tmp -= fit
try:
#eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch] = tmp[self._pa_zeros:-self._pa_zeros][::10]
y_out[t-sw/2:t+sw/2] = tmp[self._pa_zeros:-self._pa_zeros][:]
except ValueError, ve:
print i_ch, i,t, eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch].shape, tmp[self._pa_zeros:-self._pa_zeros][::10].shape
#Beispieldaten speichern: Teil 2
if i_ch == self.ch_for_slice_alignment:
if i%(len(slcs)/num_examples)==(len(slcs)/num_examples)/2:
print "examples, Teil 2"
#example["fit"] = n.zeros(example["raw"].shape) #fit.copy()
example["fit"] = fit.copy()
example["obs"] = obs.copy()
example["filt1"] = (tmp + fit).copy()
example["filt2"] = tmp.copy()
self.examples.append(example)
if show_progressbar:
pbar.update((i_ch+i/len(slcs))*100/eeg.num_channels)
except Exception, e:
print "Error occurred at slice at t=",t,", ignored"
print e
def remove_artifacts(self):
"""Tries to remove the artifacts.
Uses mixture of mean-subtraction and OBS-PCA-Subtraction
Needs self.slcs to be set.
Saves example data in self.examples to make evaluation of filter quality possible"""
def obs_fit_error(p, y, x):
err = n.zeros((x.shape[0]), "d")
err += p[0]
for i in range(self.obs_size):
err += y[:, i] * p[i + 1]
err -= x
return abs(err**2).sum()
def obs_fit_error_lsq(p, y, x):
err = n.zeros((x.shape[0]), "d")
err += p[0]
for i in range(self.obs_size):
err += y[:, i] * p[i + 1]
err -= x
return err
def las_fit_error_lsq(p, y, x):
"""Local artifact subtraction"""
err = p[0] * y
err -= x
return err
#Shortnames
eeg = self.eeg
slcs = self.slcs_ups
sw = self._new_slice_width * 10
k = self.num_neighbors
self.examples = []
num_examples = 10
#Loop over channels
if show_progressbar:
pbar = ProgressBar().start()
for i_ch in range(eeg.num_channels):
#Make Highpass-Version of channel
#ch = eeg[:,i_ch].copy()
y, fn = upsample_to_memmap(eeg[:, i_ch], 10)
y_hp, fn_hp = tmp_memmap(dtype=y.dtype, shape=y.shape, mode="w+")
y_hp[:] = filtfilt_high(1, y, Fs=10000.0)
y_out, fn_out = tmp_memmap(dtype=y.dtype, shape=y.shape, mode="w+")
y_out[:] = y[:]
#ch_hp = filtfilt_high(1.0,ch,Fs=eeg.Fs)
neighbors = n.zeros((sw + 2 * self._pa_zeros, k))
#Prefill
for i in range(k):
#print neighbors[:,i].shape, eeg[slcs[i]-sw/2:slcs[i]+sw/2,i_ch].shape
neighbors[:, i] = prepend_append_zeros(
y_hp[slcs[i] - sw / 2:slcs[i] + sw / 2], self._pa_zeros)
#Loop over slices and filter
next_subst = 0
for i, t in enumerate(slcs):
try:
if i > k / 2 and i < len(slcs) - k / 2:
neighbors[:, next_subst] = prepend_append_zeros(
y_hp[slcs[i + k / 2] - sw / 2:slcs[i + k / 2] +
sw / 2], self._pa_zeros)
next_subst += 1
next_subst = next_subst % k
tmp = prepend_append_zeros(y[t - sw / 2:t + sw / 2],
self._pa_zeros)
#Subtraction
#Shift/scale template
template = neighbors.mean(axis=1)
p = [1.0, 0]
p_las = leastsq(las_fit_error_lsq, p,
args=(template, tmp))[0]
#print p_las[0],
tmp -= p_las[0] * template
#Beispieldaten speichern: Teil 1
if i_ch == self.ch_for_slice_alignment:
if i % (len(slcs) / num_examples) == (
len(slcs) / num_examples) / 2:
print "examples, Teil 1"
example = {}
example["raw"] = prepend_append_zeros(
y[t - sw / 2:t + sw / 2].copy(),
self._pa_zeros)
example["mean"] = p_las[0] * template
#OBS-Fit
components = pcafilt.unmix(
neighbors) #OBS will be first 5 components
components -= components.mean(axis=0).reshape(
1, -1).repeat(components.shape[0],
0) #demeaning column-wise
obs = components[:, :self.obs_size].copy()
#Fit OBS to artifact
p = [0] + [0] * self.obs_size
#p_lsq = fmin(obs_fit_error,p,args=(obs,eeg[t-sw/2:t+sw/2,i_ch]),maxiter=1e5,maxfun=1e5)#[0]
p_lsq = leastsq(obs_fit_error_lsq, p, args=(obs, tmp))[0]
#print i,t,"p_lsq", p_lsq
fit = n.zeros((obs.shape[0]), "d")
fit += p_lsq[0]
for j in range(self.obs_size):
fit += obs[:, j] * p_lsq[j + 1]
tmp -= fit
try:
#eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch] = tmp[self._pa_zeros:-self._pa_zeros][::10]
y_out[t - sw / 2:t +
sw / 2] = tmp[self._pa_zeros:-self._pa_zeros][:]
except ValueError, ve:
print i_ch, i, t, eeg[
t / 10 - sw / 10 / 2:t / 10 + sw / 10 / 2,
i_ch].shape, tmp[self._pa_zeros:-self.
_pa_zeros][::10].shape
#Beispieldaten speichern: Teil 2
if i_ch == self.ch_for_slice_alignment:
if i % (len(slcs) / num_examples) == (
len(slcs) / num_examples) / 2:
print "examples, Teil 2"
#example["fit"] = n.zeros(example["raw"].shape) #fit.copy()
example["fit"] = fit.copy()
example["obs"] = obs.copy()
example["filt1"] = (tmp + fit).copy()
example["filt2"] = tmp.copy()
self.examples.append(example)
if show_progressbar:
pbar.update(
(i_ch + i / len(slcs)) * 100 / eeg.num_channels)
except Exception, e:
print "Error occurred at slice at t=", t, ", ignored"
print e
