def correlationCoefficient(x, y, mask=None): """ calcualate the correlation coefficient of two numpys """ if x.shape != y.shape: apDisplay.printError( "images are not the same shape in correlation calc") if mask != None: if x.shape != mask.shape: apDisplay.printError( "mask is not the same shape as images in correlation calc") tot = mask.sum() if tot == 0: return 0.0 x = imagenorm.normStdevMask(x, mask) y = imagenorm.normStdevMask(y, mask) else: tot = float(x.shape[0] * x.shape[1]) x = imagenorm.normStdev(x) y = imagenorm.normStdev(y) z = x * y if mask != None: z = z * mask sm = z.sum() return sm / tot
def msd(x,y,mask=None): if mask != None: tot = float(mask.sum()) if tot == 0: return 1.0e13 x = imagenorm.normStdevMask(x,mask) y = imagenorm.normStdevMask(y,mask) else: tot = float(x.shape[0]*x.shape[1]) x = imagenorm.normStdev(x) y = imagenorm.normStdev(y) z = (x-y)**2 if mask != None: z = z*mask sm = z.sum() return sm/tot
def msd(x, y, mask=None): if mask != None: tot = float(mask.sum()) if tot == 0: return 1.0e13 x = imagenorm.normStdevMask(x, mask) y = imagenorm.normStdevMask(y, mask) else: tot = float(x.shape[0] * x.shape[1]) x = imagenorm.normStdev(x) y = imagenorm.normStdev(y) z = (x - y)**2 if mask != None: z = z * mask sm = z.sum() return sm / tot
def getImageFiles(imgtree, rawdir, link, copy): #This function should replace linkImageFiles in all calls (i.e. in tomoaligner.py and everywhere else) filenamelist = [] newimgtree = [] for imagedata in imgtree: #set up names imgpath = imagedata['session']['image path'] presetname = imagedata['preset']['name'] imgprefix = presetname + imagedata['filename'].split(presetname)[-1] imgname = imgprefix + '.mrc' filenamelist.append(imgprefix) destpath = os.path.join(rawdir, imgname) newimgtree.append(destpath) imgfullpath = os.path.join(imgpath, imagedata['filename'] + '.mrc') if link == "True": #create symlinks to files if os.path.islink(destpath): os.remove(destpath) if not os.path.isfile(destpath): os.symlink(imgfullpath, destpath) elif copy == "True": shutil.copy(imgfullpath, destpath) #Y-flip raw images, normalize them, and convert them to float32 because Protomo image = mrc.read(destpath) image = numpy.flipud(image) image = imagenorm.normStdev(image) image = numpy.float32(image) mrc.write(image, destpath) #else: just return values return filenamelist, newimgtree
def getImageFiles(imgtree, rawdir, link, copy): #This function should replace linkImageFiles in all calls (i.e. in tomoaligner.py and everywhere else) filenamelist = [] newimgtree=[] for imagedata in imgtree: #set up names imgpath=imagedata['session']['image path'] presetname=imagedata['preset']['name'] imgprefix=presetname+imagedata['filename'].split(presetname)[-1] imgname=imgprefix+'.mrc' filenamelist.append(imgprefix) destpath = os.path.join(rawdir,imgname) newimgtree.append(destpath) imgfullpath = os.path.join(imgpath,imagedata['filename']+'.mrc') if link == "True": #create symlinks to files if os.path.islink(destpath): os.remove(destpath) if not os.path.isfile(destpath): os.symlink(imgfullpath,destpath) elif copy == "True": shutil.copy(imgfullpath,destpath) #Y-flip raw images, normalize them, and convert them to float32 because Protomo image=mrc.read(destpath) image=numpy.flipud(image) image=imagenorm.normStdev(image) image=numpy.float32(image) mrc.write(image,destpath) #else: just return values return filenamelist, newimgtree
def correlationCoefficient(x,y,mask=None): """ calcualate the correlation coefficient of two numpys """ if x.shape != y.shape: apDisplay.printError("images are not the same shape in correlation calc") if mask != None: if x.shape != mask.shape: apDisplay.printError("mask is not the same shape as images in correlation calc") tot = mask.sum() if tot == 0: return 0.0 x = imagenorm.normStdevMask(x,mask) y = imagenorm.normStdevMask(y,mask) else: tot = float(x.shape[0]*x.shape[1]) x = imagenorm.normStdev(x) y = imagenorm.normStdev(y) z = x*y if mask != None: z = z*mask sm = z.sum() return sm/tot
def doseCompensate(seriesname, rundir, sessionname, tiltseriesnumber, frame_aligned_images, raw_path, pixelsize, dose_presets, dose_a, dose_b, dose_c): """ Images will be lowpass filtered using equation (3) from Grant & Grigorieff, 2015. No changes to the database are made. No backups are made. """ sessiondata = apDatabase.getSessionDataFromSessionName(sessionname) tiltseriesdata = apDatabase.getTiltSeriesDataFromTiltNumAndSessionId(tiltseriesnumber,sessiondata) tiltdata = apTomo.getImageList([tiltseriesdata]) frame_tiltdata, non_frame_tiltdata = frameOrNonFrameTiltdata(tiltdata) tilts, ordered_imagelist, accumulated_dose_list, ordered_mrc_files, refimg = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned="False") if frame_aligned_images == "True": #For different image filenames a, ordered_imagelist, c, d, e = apTomo.orderImageList(frame_tiltdata, non_frame_tiltdata, frame_aligned=frame_aligned_images) newfilenames, new_ordered_imagelist = apProTomo.getImageFiles(ordered_imagelist, raw_path, link=False, copy=False) if (dose_presets == "Light"): dose_a = 0.245 dose_b = -1.6 dose_c = 12 elif (dose_presets == "Moderate"): dose_a = 0.245 dose_b = -1.665 dose_c = 2.81 elif (dose_presets == "Heavy"): dose_a = 0.245 dose_b = -1.4 dose_c = 2 apDisplay.printMsg('Dose compensating all tilt images with a=%s, b=%s, and c=%s...' % (dose_a, dose_b, dose_c)) for image, j in zip(new_ordered_imagelist, range(len(new_ordered_imagelist))): lowpass = float(np.real(complex(dose_a/(accumulated_dose_list[j] - dose_c))**(1/dose_b))) #equation (3) from Grant & Grigorieff, 2015 if lowpass < 0.0: lowpass = 0.0 im = mrc.read(image) im = imagefilter.lowPassFilter(im, apix=pixelsize, radius=lowpass, msg=False) im=imagenorm.normStdev(im) mrc.write(im, image) #Make plots apProTomo2Aligner.makeDosePlots(rundir, seriesname, tilts, accumulated_dose_list, dose_a, dose_b, dose_c) apDisplay.printMsg("Dose compensation finished for tilt-series #%s!" % tiltseriesnumber) return
def createDots(angle=0.0, noiselevel=3.0, shape=(128,128), shift=None): a = numpy.zeros(shape, dtype=numpy.float64) a30 = int(min(shape)*.30) #25% a40 = int(2*min(shape)/5.0) #40% a50 = int(min(shape)/2.0) #50% a60 = int(3*min(shape)/5.0) #60% a70 = int(min(shape)*.70) #75% a[a30:a40,a30:a40] = 1 a[a50:a60,a30:a40] = 1 a[a60:a70,a40:a50] = 1 a[a40:a50,a50:a60] = 1 a[a30:a40,a60:a70] = 1 a[a60:a70,a60:a70] = 1 b = a b = ndimage.rotate(b, angle, reshape=False, order=1) if shift is not None: b = ndimage.shift(b, shift=shift, mode='wrap', order=1) bnoise = b + noiselevel*numpy.random.random(shape) bnoise = ndimage.median_filter(bnoise, size=2) bnoise = imagenorm.normStdev(bnoise) return bnoise
def mergeImageStackIntoBigStack(self, imgstackfile, imgdata): t0 = time.time() apDisplay.printMsg("filtering particles and adding to stack") # if applying a boxmask, write to a temporary file before adding to main stack bigimgstack = os.path.join(self.params['rundir'], self.params['single']) if self.params['boxmask'] is not None: bigimgstack = os.path.splitext(imgstackfile)[0]+"-premask.hed" ### here is the craziness ### step 1: read imgstackfile into memory imgstackmemmap = imagic.read(imgstackfile) ### when only particle is read it defaults to a 2D array instead of 3D array if len(imgstackmemmap.shape) < 3: imgstackmemmap = imgstackmemmap.reshape(1, imgstackmemmap.shape[0], imgstackmemmap.shape[1]) if self.params['debug'] is True: print "imgstackmemmap.shape", imgstackmemmap.shape apix = self.params['apix'] #apDatabase.getPixelSize(imgdata) boxshape = (self.boxsize, self.boxsize) processedParticles = [] for particle in imgstackmemmap: ### step 2: filter particles ### high / low pass filtering #if self.params['pixlimit']: # particle = imagefilter.pixelLimitFilter(particle, self.params['pixlimit']) if self.params['lowpass']: particle = imagefilter.lowPassFilter(particle, apix=apix, radius=self.params['lowpass']) if self.params['highpass']: particle = imagefilter.highPassFilter2(particle, self.params['highpass'], apix=apix) ### unless specified, invert the images if self.params['inverted'] is True: particle = -1.0 * particle if particle.shape != boxshape: if self.boxsize <= particle.shape[0] and self.boxsize <= particle.shape[1]: particle = imagefilter.frame_cut(particle, boxshape) else: apDisplay.printError("particle shape (%dx%d) is smaller than boxsize (%d)" %(particle.shape[0], particle.shape[1], self.boxsize)) ### step 3: normalize particles #self.normoptions = ('none', 'boxnorm', 'edgenorm', 'rampnorm', 'parabolic') #normalizemethod if self.params['normalizemethod'] == 'boxnorm': particle = imagenorm.normStdev(particle) elif self.params['normalizemethod'] == 'edgenorm': particle = imagenorm.edgeNorm(particle) elif self.params['normalizemethod'] == 'rampnorm': particle = imagenorm.rampNorm(particle) elif self.params['normalizemethod'] == 'parabolic': particle = imagenorm.parabolicNorm(particle) ### step 4: decimate/bin particles if specified ### binning is last, so we maintain most detail and do not have to deal with binned apix if self.params['bin'] > 1: particle = imagefun.bin2(particle, self.params['bin']) #from scipy.misc import toimage #toimage(particle).show() processedParticles.append(particle) ### step 5: merge particle list with larger stack apImagicFile.appendParticleListToStackFile(processedParticles, bigimgstack, msg=self.params['debug']) #remove original image stack from memory del imgstackmemmap del processedParticles t0 = time.time() # if applying boxmask, now mask the particles & append to stack if self.params['boxmask'] is not None: # normalize particles before boxing, since zeros in mask # can affect subsequent processing if not properly normalized apEMAN.executeEmanCmd("proc2d %s %s edgenorm inplace"%(bigimgstack,bigimgstack),showcmd=False) imgstack = apImagicFile.readImagic(bigimgstack, msg=False) maskstack = apImagicFile.readImagic(self.params['boxmaskf'],msg=False) for i in range(len(imgstack['images'])): imgstack['images'][i]*=maskstack['images'][i] maskedpartstack = os.path.splitext(imgstackfile)[0]+"-aftermask.hed" apImagicFile.writeImagic(imgstack['images'], maskedpartstack) bigimgstack = os.path.join(self.params['rundir'], self.params['single']) apEMAN.executeEmanCmd("proc2d %s %s flip"%(maskedpartstack,bigimgstack)) ### count particles bigcount = apFile.numImagesInStack(bigimgstack, self.boxsize/self.params['bin']) imgcount = apFile.numImagesInStack(imgstackfile, self.boxsize) ### append to particle log file partlogfile = os.path.join(self.params['rundir'], self.timestamp+"-particles.info") f = open(partlogfile, 'a') for i in range(imgcount): partnum = self.particleNumber + i + 1 line = str(partnum)+'\t'+os.path.join(imgdata['session']['image path'], imgdata['filename']+".mrc") f.write(line+"\n") f.close() self.mergestacktimes.append(time.time()-t0) return bigcount
def start(self): ### Works # read from MRC image/stack # read from HED/IMG stack # write to MRC image # write to MRC stack # write to HED/IMG stack # append to MRC stack # append to HED/IMG stack # filter images # implement binning # implement normalization ### needs more testing # write pixelsize to new MRC file # get apix from MRC header # implement proc2d --average # implement proc2d --list feature ### TODO # read from SPIDER stack # read from SPIDER image # read from EMAN2/HDF stack # write to SPIDER image # write to SPIDER stack # write to EMAN2/HDF stack # get apix from HED/IMG header # implement proc2d --rotavg # implement proc2d --clip # determine numParticles to add if self.params['last'] is None: self.params['last'] = self.inputNumParticles - 1 #stacks start at 0 elif self.params['last'] > self.inputNumParticles: apDisplay.printWarning("Last particle requested (%d) is larger than available particles (%d)" %(self.params['last'], self.inputNumParticles)) self.params['last'] = self.inputNumParticles - 1 #stacks start at 0 addNumParticles = self.params['last'] - self.params['first'] + 1 ### prepare for an existing file existNumParticles = self.outFileStatus() self.totalParticles = existNumParticles + addNumParticles indata = self.readFileData(self.params['infile']) #it more efficient to process X particles and write them to disk rather than # write each particle to disk each time. #particles are read using a memory map (numpy.memmap), so we can pretend to # continuously read all into memory particlesPerCycle = self.getParticlesPerCycle(self.params['infile']) if self.params['average'] is True: summedPartice = numpy.zeros((self.boxsize,self.boxsize)) processedParticles = [] if self.params['list']: partlist = self.readKeepList() else: partlist = range(self.params['first'], self.params['first']+addNumParticles) count = 0 for partnum in partlist: count += 1 particle = indata[partnum] if self.params['debug'] is True: print "---------" print "Particle Number: %d of %d"%(partnum, addNumParticles) if self.params['pixlimit']: self.message("pixlimit: %s"%(self.params['pixlimit'])) particle = imagefilter.pixelLimitFilter(particle, self.params['pixlimit']) if self.params['lowpass']: self.message("lowpass: %s"%(self.params['lowpass'])) particle = imagefilter.lowPassFilter(particle, apix=self.params['apix'], radius=self.params['lowpass']) if self.params['highpass']: self.message("highpass: %s"%(self.params['highpass'])) particle = imagefilter.highPassFilter2(particle, self.params['highpass'], apix=self.params['apix']) ### unless specified, invert the images if self.params['inverted'] is True: self.message("inverted: %s"%(self.params['inverted'])) particle = -1.0 * particle ### clipping """ if particle.shape != boxshape: if boxsize <= particle.shape[0] and boxsize <= particle.shape[1]: particle = imagefilter.frame_cut(particle, boxshape) else: apDisplay.printError("particle shape (%dx%d) is smaller than boxsize (%d)" %(particle.shape[0], particle.shape[1], boxsize)) """ ### step 3: normalize particles #self.normoptions = ('none', 'boxnorm', 'edgenorm', 'rampnorm', 'parabolic') #normalizemethod self.message("normalize method: %s"%(self.params['normalizemethod'])) if self.params['normalizemethod'] == 'boxnorm': particle = imagenorm.normStdev(particle) elif self.params['normalizemethod'] == 'edgenorm': particle = imagenorm.edgeNorm(particle) elif self.params['normalizemethod'] == 'rampnorm': particle = imagenorm.rampNorm(particle) elif self.params['normalizemethod'] == 'parabolic': particle = imagenorm.parabolicNorm(particle) ### step 4: decimate/bin particles if specified ### binning is last, so we maintain most detail and do not have to deal with binned apix if self.params['bin'] > 1: particle = imagefun.bin2(particle, self.params['bin']) ### working above this line if self.params['average'] is True: summedPartice += particle else: processedParticles.append(particle) if len(processedParticles) == particlesPerCycle: ### step 5: merge particle list with larger stack self.appendParticleListToStackFile(processedParticles, self.params['outfile']) sys.stderr.write("%d of %d"%(count, len(partlist))) processedParticles = [] if len(processedParticles) > 0: self.appendParticleListToStackFile(processedParticles, self.params['outfile']) if self.params['average'] is True: avgParticle = summedPartice/count self.appendParticleListToStackFile([avgParticle,], self.params['outfile']) print "Wrote %d particles to file "%(self.particlesWritten)