def remove_artifacts(self): """Tries to remove the artifacts. Uses mixture of mean-subtraction and OBS-PCA-Subtraction Needs self.slcs to be set. Saves example data in self.examples to make evaluation of filter quality possible""" def obs_fit_error(p,y,x): err = n.zeros((x.shape[0]),"d") err += p[0] for i in range(self.obs_size): err += y[:,i]*p[i+1] err -= x return abs(err**2).sum() def obs_fit_error_lsq(p,y,x): err = n.zeros((x.shape[0]),"d") err += p[0] for i in range(self.obs_size): err += y[:,i]*p[i+1] err -= x return err def las_fit_error_lsq(p,y,x): """Local artifact subtraction""" err = p[0]*y err -= x return err #Shortnames eeg = self.eeg slcs = self.slcs_ups sw = self._new_slice_width *10 k = self.num_neighbors self.examples=[] num_examples=10 #Loop over channels if show_progressbar: pbar = ProgressBar().start() for i_ch in range(eeg.num_channels): #Make Highpass-Version of channel #ch = eeg[:,i_ch].copy() y , fn = upsample_to_memmap(eeg[:,i_ch],10) y_hp,fn_hp = tmp_memmap(dtype=y.dtype,shape=y.shape,mode="w+") y_hp[:] = filtfilt_high(1, y, Fs=10000.0) y_out,fn_out = tmp_memmap(dtype=y.dtype,shape=y.shape,mode="w+") y_out[:] = y[:] #ch_hp = filtfilt_high(1.0,ch,Fs=eeg.Fs) neighbors = n.zeros((sw+2*self._pa_zeros,k)) #Prefill for i in range(k): #print neighbors[:,i].shape, eeg[slcs[i]-sw/2:slcs[i]+sw/2,i_ch].shape neighbors[:,i] = prepend_append_zeros( y_hp[slcs[i]-sw/2:slcs[i]+sw/2] , self._pa_zeros) #Loop over slices and filter next_subst = 0 for i,t in enumerate(slcs): try: if i>k/2 and i<len(slcs)-k/2: neighbors[:,next_subst] = prepend_append_zeros( y_hp[slcs[i+k/2]-sw/2:slcs[i+k/2]+sw/2] , self._pa_zeros) next_subst+=1 next_subst=next_subst%k tmp = prepend_append_zeros( y[t-sw/2:t+sw/2] , self._pa_zeros) #Subtraction #Shift/scale template template = neighbors.mean(axis=1) p = [1.0, 0] p_las = leastsq(las_fit_error_lsq,p,args=(template,tmp))[0] #print p_las[0], tmp -= p_las[0]*template #Beispieldaten speichern: Teil 1 if i_ch == self.ch_for_slice_alignment: if i%(len(slcs)/num_examples)==(len(slcs)/num_examples)/2: print "examples, Teil 1" example = {} example["raw"] = prepend_append_zeros(y[t-sw/2:t+sw/2].copy() , self._pa_zeros ) example["mean"] = p_las[0]*template #OBS-Fit components = pcafilt.unmix(neighbors) #OBS will be first 5 components components -= components.mean(axis=0).reshape(1,-1).repeat(components.shape[0],0) #demeaning column-wise obs = components[:,:self.obs_size].copy() #Fit OBS to artifact p = [0]+[0]*self.obs_size #p_lsq = fmin(obs_fit_error,p,args=(obs,eeg[t-sw/2:t+sw/2,i_ch]),maxiter=1e5,maxfun=1e5)#[0] p_lsq = leastsq(obs_fit_error_lsq,p,args=(obs,tmp))[0] #print i,t,"p_lsq", p_lsq fit = n.zeros((obs.shape[0]),"d") fit +=p_lsq[0] for j in range(self.obs_size): fit += obs[:,j]*p_lsq[j+1] tmp -= fit try: #eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch] = tmp[self._pa_zeros:-self._pa_zeros][::10] y_out[t-sw/2:t+sw/2] = tmp[self._pa_zeros:-self._pa_zeros][:] except ValueError, ve: print i_ch, i,t, eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch].shape, tmp[self._pa_zeros:-self._pa_zeros][::10].shape #Beispieldaten speichern: Teil 2 if i_ch == self.ch_for_slice_alignment: if i%(len(slcs)/num_examples)==(len(slcs)/num_examples)/2: print "examples, Teil 2" #example["fit"] = n.zeros(example["raw"].shape) #fit.copy() example["fit"] = fit.copy() example["obs"] = obs.copy() example["filt1"] = (tmp + fit).copy() example["filt2"] = tmp.copy() self.examples.append(example) if show_progressbar: pbar.update((i_ch+i/len(slcs))*100/eeg.num_channels) except Exception, e: print "Error occurred at slice at t=",t,", ignored" print e
def remove_artifacts(self): """Tries to remove the artifacts. Uses mixture of mean-subtraction and OBS-PCA-Subtraction Needs self.slcs to be set. Saves example data in self.examples to make evaluation of filter quality possible""" def obs_fit_error(p, y, x): err = n.zeros((x.shape[0]), "d") err += p[0] for i in range(self.obs_size): err += y[:, i] * p[i + 1] err -= x return abs(err**2).sum() def obs_fit_error_lsq(p, y, x): err = n.zeros((x.shape[0]), "d") err += p[0] for i in range(self.obs_size): err += y[:, i] * p[i + 1] err -= x return err def las_fit_error_lsq(p, y, x): """Local artifact subtraction""" err = p[0] * y err -= x return err #Shortnames eeg = self.eeg slcs = self.slcs_ups sw = self._new_slice_width * 10 k = self.num_neighbors self.examples = [] num_examples = 10 #Loop over channels if show_progressbar: pbar = ProgressBar().start() for i_ch in range(eeg.num_channels): #Make Highpass-Version of channel #ch = eeg[:,i_ch].copy() y, fn = upsample_to_memmap(eeg[:, i_ch], 10) y_hp, fn_hp = tmp_memmap(dtype=y.dtype, shape=y.shape, mode="w+") y_hp[:] = filtfilt_high(1, y, Fs=10000.0) y_out, fn_out = tmp_memmap(dtype=y.dtype, shape=y.shape, mode="w+") y_out[:] = y[:] #ch_hp = filtfilt_high(1.0,ch,Fs=eeg.Fs) neighbors = n.zeros((sw + 2 * self._pa_zeros, k)) #Prefill for i in range(k): #print neighbors[:,i].shape, eeg[slcs[i]-sw/2:slcs[i]+sw/2,i_ch].shape neighbors[:, i] = prepend_append_zeros( y_hp[slcs[i] - sw / 2:slcs[i] + sw / 2], self._pa_zeros) #Loop over slices and filter next_subst = 0 for i, t in enumerate(slcs): try: if i > k / 2 and i < len(slcs) - k / 2: neighbors[:, next_subst] = prepend_append_zeros( y_hp[slcs[i + k / 2] - sw / 2:slcs[i + k / 2] + sw / 2], self._pa_zeros) next_subst += 1 next_subst = next_subst % k tmp = prepend_append_zeros(y[t - sw / 2:t + sw / 2], self._pa_zeros) #Subtraction #Shift/scale template template = neighbors.mean(axis=1) p = [1.0, 0] p_las = leastsq(las_fit_error_lsq, p, args=(template, tmp))[0] #print p_las[0], tmp -= p_las[0] * template #Beispieldaten speichern: Teil 1 if i_ch == self.ch_for_slice_alignment: if i % (len(slcs) / num_examples) == ( len(slcs) / num_examples) / 2: print "examples, Teil 1" example = {} example["raw"] = prepend_append_zeros( y[t - sw / 2:t + sw / 2].copy(), self._pa_zeros) example["mean"] = p_las[0] * template #OBS-Fit components = pcafilt.unmix( neighbors) #OBS will be first 5 components components -= components.mean(axis=0).reshape( 1, -1).repeat(components.shape[0], 0) #demeaning column-wise obs = components[:, :self.obs_size].copy() #Fit OBS to artifact p = [0] + [0] * self.obs_size #p_lsq = fmin(obs_fit_error,p,args=(obs,eeg[t-sw/2:t+sw/2,i_ch]),maxiter=1e5,maxfun=1e5)#[0] p_lsq = leastsq(obs_fit_error_lsq, p, args=(obs, tmp))[0] #print i,t,"p_lsq", p_lsq fit = n.zeros((obs.shape[0]), "d") fit += p_lsq[0] for j in range(self.obs_size): fit += obs[:, j] * p_lsq[j + 1] tmp -= fit try: #eeg[t/10-sw/10/2:t/10+sw/10/2,i_ch] = tmp[self._pa_zeros:-self._pa_zeros][::10] y_out[t - sw / 2:t + sw / 2] = tmp[self._pa_zeros:-self._pa_zeros][:] except ValueError, ve: print i_ch, i, t, eeg[ t / 10 - sw / 10 / 2:t / 10 + sw / 10 / 2, i_ch].shape, tmp[self._pa_zeros:-self. _pa_zeros][::10].shape #Beispieldaten speichern: Teil 2 if i_ch == self.ch_for_slice_alignment: if i % (len(slcs) / num_examples) == ( len(slcs) / num_examples) / 2: print "examples, Teil 2" #example["fit"] = n.zeros(example["raw"].shape) #fit.copy() example["fit"] = fit.copy() example["obs"] = obs.copy() example["filt1"] = (tmp + fit).copy() example["filt2"] = tmp.copy() self.examples.append(example) if show_progressbar: pbar.update( (i_ch + i / len(slcs)) * 100 / eeg.num_channels) except Exception, e: print "Error occurred at slice at t=", t, ", ignored" print e
def find_all_slice_artifacts(self): def update_pbar(num): """Callback for find_all_maxs""" if show_progressbar: pbar.update(num/2) eeg = self.eeg #y = abs(smooth_windowed_eeg(eeg,[self.cfsa],self._slice_width))[:,0] #y = smooth_windowed_eeg_power(eeg,[self.cfsa],self._slice_width)[:,0] y = filtfilt_band(1,eeg.Fs/self._slice_width,eeg[:,self.cfsa]) #pylab.plot(y[::10]) #pylab.plot(eeg[:,14]) print y.shape, self._slice_width #import pylab #pylab.ion() #pylab.plot(y[0:20000:1]) #pylab.show() #raw_input() #pylab.plot(y[13000:20000:1]) slcs_raw = find_all_maxs(y[:1000],ratio=0.6) # First segment slcs_raw.sort() #print "t=", t #slcs_raw.append(t) offset=0 t=int(0.5*self._slice_width) while (t>0.4*self._slice_width or (y.shape[0]-offset)>10000) and (y.shape[0]-offset)>self._slice_width*2: #print (y.shape[0]-offset) #print t, offset, "-", offset = slcs_raw[-1]+self._slice_width/2 #print t, offset, "-", #pylab.plot(y[offset:offset+self._slice_width]) #pylab.show() #raw_input() t=y[offset:offset+self._slice_width].argmax() slcs_raw.append(offset+t) #print slcs_raw[-1], slcs_raw[-1]-slcs_raw[-2], " - ", #time.sleep(0.1) #print t, offset print "" #pylab.plot(y[::10]) if show_progressbar: pbar = ProgressBar(maxval=eeg.shape[0]/self._slice_width).start() #slcs_raw = find_all_maxs(y[:,0],0.3,self._slice_width,20,callback=update_pbar) print "Raw slice-positions found", len(slcs_raw), np.mean(np.diff(slcs_raw)), np.min(slcs_raw), np.max(slcs_raw) slcs_raw_diff = np.diff(slcs_raw) print "slcs_raw_diff: ", scoreatpercentile(slcs_raw_diff,5), scoreatpercentile(slcs_raw_diff,50), scoreatpercentile(slcs_raw_diff,95) #raise Exception("Abbruch") y , fn = upsample_to_memmap(eeg[:,self.cfsa],10) slcs_raw_ups = [x*10 for x in slcs_raw] t = slcs_raw_ups[len(slcs_raw)/2] template = y[t-self._slice_width*10/2:t+self._slice_width*10/2] for i in range(5): t = slcs_raw_ups[len(slcs_raw)/2+i] template += y[t-self._slice_width*10/2:t+self._slice_width*10/2] template /= 6 offsets = [] for i,t in enumerate(slcs_raw_ups): #offset = find_max_overlap(template, eeg[t-self._slice_width/2:t+self._slice_width/2,self.cfsa], 100) offset = find_max_overlap(template, y[t-self._slice_width*10/2:t+self._slice_width*10/2], 100) offsets.append(offset) self.slcs_ups = [slcs_raw_ups[i]+offsets[i]+self.slice_shift for i in range(len(slcs_raw_ups))] if show_progressbar: pbar.finish() print "Refined slice-positions found. Finished.", len(offsets), np.mean(offsets), np.median(offsets), np.min(offsets), np.max(offsets) print "Percentile 0.5,5,95,99.5 of offsets: ", scoreatpercentile(offsets,0.5), scoreatpercentile(offsets,5), scoreatpercentile(offsets,95), scoreatpercentile(offsets,99.5) #Adjusting _slice_width... print "Old slice_width:", self._slice_width self._new_slice_width = int(n.ceil(n.mean(n.diff(self.slcs_ups))))/10 self._new_slice_width += 3 # Make slice wider to have no zombie-timepoints self._new_slice_width = self._new_slice_width+self._new_slice_width%2 #self._new_slice_width = (self._new_slice_width/2)*2 # make sw%2==0 (divisible by 2) print "New slice_width:", self._new_slice_width #raise Exception("Abbruch") return [x/10 for x in self.slcs_ups]
def find_all_slice_artifacts(self): def update_pbar(num): """Callback for find_all_maxs""" if show_progressbar: pbar.update(num / 2) eeg = self.eeg #y = abs(smooth_windowed_eeg(eeg,[self.cfsa],self._slice_width))[:,0] #y = smooth_windowed_eeg_power(eeg,[self.cfsa],self._slice_width)[:,0] y = filtfilt_band(1, eeg.Fs / self._slice_width, eeg[:, self.cfsa]) #pylab.plot(y[::10]) #pylab.plot(eeg[:,14]) print y.shape, self._slice_width #import pylab #pylab.ion() #pylab.plot(y[0:20000:1]) #pylab.show() #raw_input() #pylab.plot(y[13000:20000:1]) slcs_raw = find_all_maxs(y[:1000], ratio=0.6) # First segment slcs_raw.sort() #print "t=", t #slcs_raw.append(t) offset = 0 t = int(0.5 * self._slice_width) while (t > 0.4 * self._slice_width or (y.shape[0] - offset) > 10000 ) and (y.shape[0] - offset) > self._slice_width * 2: #print (y.shape[0]-offset) #print t, offset, "-", offset = slcs_raw[-1] + self._slice_width / 2 #print t, offset, "-", #pylab.plot(y[offset:offset+self._slice_width]) #pylab.show() #raw_input() t = y[offset:offset + self._slice_width].argmax() slcs_raw.append(offset + t) #print slcs_raw[-1], slcs_raw[-1]-slcs_raw[-2], " - ", #time.sleep(0.1) #print t, offset print "" #pylab.plot(y[::10]) if show_progressbar: pbar = ProgressBar(maxval=eeg.shape[0] / self._slice_width).start() #slcs_raw = find_all_maxs(y[:,0],0.3,self._slice_width,20,callback=update_pbar) print "Raw slice-positions found", len(slcs_raw), np.mean( np.diff(slcs_raw)), np.min(slcs_raw), np.max(slcs_raw) slcs_raw_diff = np.diff(slcs_raw) print "slcs_raw_diff: ", scoreatpercentile( slcs_raw_diff, 5), scoreatpercentile(slcs_raw_diff, 50), scoreatpercentile(slcs_raw_diff, 95) #raise Exception("Abbruch") y, fn = upsample_to_memmap(eeg[:, self.cfsa], 10) slcs_raw_ups = [x * 10 for x in slcs_raw] t = slcs_raw_ups[len(slcs_raw) / 2] template = y[t - self._slice_width * 10 / 2:t + self._slice_width * 10 / 2] for i in range(5): t = slcs_raw_ups[len(slcs_raw) / 2 + i] template += y[t - self._slice_width * 10 / 2:t + self._slice_width * 10 / 2] template /= 6 offsets = [] for i, t in enumerate(slcs_raw_ups): #offset = find_max_overlap(template, eeg[t-self._slice_width/2:t+self._slice_width/2,self.cfsa], 100) offset = find_max_overlap( template, y[t - self._slice_width * 10 / 2:t + self._slice_width * 10 / 2], 100) offsets.append(offset) self.slcs_ups = [ slcs_raw_ups[i] + offsets[i] + self.slice_shift for i in range(len(slcs_raw_ups)) ] if show_progressbar: pbar.finish() print "Refined slice-positions found. Finished.", len( offsets), np.mean(offsets), np.median(offsets), np.min( offsets), np.max(offsets) print "Percentile 0.5,5,95,99.5 of offsets: ", scoreatpercentile( offsets, 0.5), scoreatpercentile(offsets, 5), scoreatpercentile( offsets, 95), scoreatpercentile(offsets, 99.5) #Adjusting _slice_width... print "Old slice_width:", self._slice_width self._new_slice_width = int(n.ceil(n.mean(n.diff(self.slcs_ups)))) / 10 self._new_slice_width += 3 # Make slice wider to have no zombie-timepoints self._new_slice_width = self._new_slice_width + self._new_slice_width % 2 #self._new_slice_width = (self._new_slice_width/2)*2 # make sw%2==0 (divisible by 2) print "New slice_width:", self._new_slice_width #raise Exception("Abbruch") return [x / 10 for x in self.slcs_ups]