def createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] #defocus=15000 + 5000* random.random() for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i]#+500. defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(Transform(a)) self.partSet.append(p) self.partSet.write()
def importCTF(self, mic, fileName): ctf = CTFModel() ctf.setMicrograph(mic) readCtfModel(ctf, fileName, ctf4=ctffindOutputVersion(fileName) == 4) for suffix in ["_psd.mrc", ".mrc"]: if os.path.exists(pwutils.removeExt(fileName) + suffix): ctf.setPsdFile(pwutils.removeExt(fileName) + suffix) return ctf
def _getCTFModel(self, defocusU, defocusV, defocusAngle, resol, psdFile): ctf = CTFModel() ctf.setStandardDefocus(defocusU, defocusV, defocusAngle) ctf.setResolution(resol) ctf.setPsdFile(psdFile) return ctf
def importCTF(self, mic, fileName): defocusU, defocusV, defocusAngle = parseCtffindOutput(fileName) ctf = CTFModel() ctf.copyObjId(mic) ctf.setStandardDefocus(defocusU, defocusV, defocusAngle) ctf.setMicrograph(mic) ctf.setPsdFile(removeExt(fileName) + "_psd.mrc") return ctf
def importCTF(self, mic, fileName): ctf = CTFModel() ctf.setMicrograph(mic) readCtfModel(ctf, fileName, ctf4=ctffindOutputVersion(fileName) == 4) # Try to find the given PSD file associated with the cttfind log file # we handle special cases of .ctf extension and _ctffindX prefix for Relion runs fnBase = pwutils.removeExt(fileName) for suffix in ["_psd.mrc", ".mrc", ".ctf"]: psdPrefixes = [fnBase, fnBase.replace("_ctffind3", ""), fnBase.replace("_ctffind4", "")] for prefix in psdPrefixes: psdFile = prefix + suffix if os.path.exists(psdFile): if psdFile.endswith(".ctf"): psdFile += ":mrc" ctf.setPsdFile(psdFile) return ctf
def _micrographsFromEmx(protocol, emxData, emxFile, outputDir, acquisition, samplingRate, copyOrLink): """ Create the output SetOfMicrographs given an EMXData object. If there is information of the CTF, also the SetOfCTF will be registered as output of the protocol. """ emxMic = emxData.getFirstObject(emxlib.MICROGRAPH) if emxMic is not None: micSet = protocol._createSetOfMicrographs() mic = Micrograph() mic.setAcquisition(acquisition) if _hasCtfLabels(emxMic): mic.setCTF(CTFModel()) ctfSet = protocol._createSetOfCTF() else: ctfSet = None _micrographFromEmx(emxMic, mic) acq = mic.getAcquisition().clone() micSet.setAcquisition(acq) micSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) if not samplingRate: samplingRate = mic.getSamplingRate() micSet.setSamplingRate(samplingRate) micDir = dirname(emxFile) for emxMic in emxData.iterClasses(emxlib.MICROGRAPH): _micrographFromEmx(emxMic, mic) _, fn = mic.getLocation() micFn = join(micDir, fn) if copyOrLink is not None: micBase = basename(micFn) newFn = join(outputDir, micBase) copyOrLink(micFn, newFn) mic.setLocation(newFn) else: mic.setLocation(micFn) _fillMicName(mic, fn) micSet.append(mic) emxMic._micId = mic.getObjId() mic.cleanObjId() if ctfSet is not None: ctf = mic.getCTF().clone() ctf.setMicrograph(mic) #TODO I do not think next line is needed #ctf.setMicFile(newFn) ctfSet.append(ctf) if emxMic is not None: protocol._defineOutputs(outputMicrographs=micSet) if ctfSet is not None: protocol._defineOutputs(outputCTF=ctfSet) ctfSet.setMicrographs(micSet) protocol._defineCtfRelation(micSet, ctfSet)
def importCTF(self, mic, fileName): ctf = CTFModel() ctf.setMicrograph(mic) readCtfModel(ctf, fileName, ctf4=False) # Try to find the given PSD file associated with the cttfind log file # we handle special cases of .ctf extension and _ctffindX prefix for Relion runs fnBase = pwutils.removeExt(fileName) for suffix in ['_psd.mrc', '.mrc', '.ctf']: psdPrefixes = [fnBase, fnBase.replace('_ctffind3', ''), fnBase.replace('_gctf', '')] for prefix in psdPrefixes: psdFile = prefix + suffix if os.path.exists(psdFile): if psdFile.endswith('.ctf'): psdFile += ':mrc' ctf.setPsdFile(psdFile) return ctf
def aaatest_particlesToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles( filename=self.getOutputPath("particles.sqlite")) n = 10 fn = self.particles ctfs = [ CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles.star') fnStk = self.getOutputPath('particles.stk') print ">>> Writing to file: %s" % fnStar relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def importCtfStep(self, micsId, pattern): """ Copy movies matching the filename pattern Register other parameters. """ inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) ctfFiles = self._getFilePaths(pattern) ctfDict = {} for fn in ctfFiles: # Try to match the micrograph id from filename # this is set by the user by using #### format in the pattern match = self._idRegex.match(fn) if match is None: raise Exception("File '%s' doesn't match the pattern '%s'" % (fn, self.pattern.get())) ctfId = int(match.group(1)) ctfDict[ctfId] = fn from pyworkflow.em.packages.grigoriefflab.convert import parseCtffindOutput for mic in inputMics: if mic.getObjId() in ctfDict: defocusU, defocusV, defocusAngle = parseCtffindOutput( ctfDict[mic.getObjId()]) else: self.warning("CTF for micrograph id %d was not found." % mic.getObjId()) defocusU, defocusV, defocusAngle = -999, -1, -999 # save the values of defocus for each micrograph in a list ctf = CTFModel() ctf.copyObjId(mic) ctf.setStandardDefocus(defocusU, defocusV, defocusAngle) ctf.setMicrograph(mic) ctfSet.append(ctf) self._defineOutputs(outputCTF=ctfSet) self._defineCtfRelation(inputMics, ctfSet)
def importCtfStep(self, micsId, pattern): """ Copy movies matching the filename pattern Register other parameters. """ inputMics = self.inputMicrographs.get() ctfSet = self._createSetOfCTF() ctfSet.setMicrographs(inputMics) ctfFiles = self._getFilePaths(pattern) ctfDict = {} for fn in ctfFiles: # Try to match the micrograph id from filename # this is set by the user by using #### format in the pattern match = self._idRegex.match(fn) if match is None: raise Exception("File '%s' doesn't match the pattern '%s'" % (fn, self.pattern.get())) ctfId = int(match.group(1)) ctfDict[ctfId] = fn from pyworkflow.em.packages.grigoriefflab.convert import parseCtffindOutput for mic in inputMics: if mic.getObjId() in ctfDict: defocusU, defocusV, defocusAngle = parseCtffindOutput(ctfDict[mic.getObjId()]) else: self.warning("CTF for micrograph id %d was not found." % mic.getObjId()) defocusU, defocusV, defocusAngle = -999, -1, -999 # save the values of defocus for each micrograph in a list ctf = CTFModel() ctf.copyObjId(mic) ctf.setStandardDefocus(defocusU, defocusV, defocusAngle) ctf.setMicrograph(mic) ctfSet.append(ctf) self._defineOutputs(outputCTF=ctfSet) self._defineCtfRelation(inputMics, ctfSet)
def importCTF(self, mic, fileName): ctf = CTFModel() ctf.setMicrograph(mic) readCtfModel(ctf, fileName, ctf4=False) # Try to find the given PSD file associated with the cttfind log file # we handle special cases of .ctf extension and _ctffindX prefix for Relion runs fnBase = pwutils.removeExt(fileName) for suffix in ['_psd.mrc', '.mrc', '.ctf']: psdPrefixes = [ fnBase, fnBase.replace('_ctffind3', ''), fnBase.replace('_gctf', '') ] for prefix in psdPrefixes: psdFile = prefix + suffix if os.path.exists(psdFile): if psdFile.endswith('.ctf'): psdFile += ':mrc' ctf.setPsdFile(psdFile) return ctf
def _getCTFModel(self, defocusU, defocusV, defocusAngle, psdFile): ctf = CTFModel() ctf.setStandardDefocus(defocusU, defocusV, defocusAngle) ctf.setPsdFile(psdFile) return ctf
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def _particlesFromEmx(protocol, emxData, emxFile, outputDir, acquisition, samplingRate, copyOrLink, alignType=ALIGN_NONE): """ Create the output SetOfCoordinates or SetOfParticles given an EMXData object. Add CTF information to the particles if present. """ emxParticle = emxData.getFirstObject(emxlib.PARTICLE) partDir = dirname(emxFile) micSet = getattr(protocol, 'outputMicrographs', None) if emxParticle is not None: # Check if there are particles or coordinates fn = emxParticle.get(emxlib.FILENAME) if exists(join( partDir, fn)): # if the particles has binary data, means particles case partSet = protocol._createSetOfParticles() partSet.setAcquisition( acquisition) # this adquisition is per project #we need one per particle part = Particle() if _hasCtfLabels(emxParticle) or _hasCtfLabels( emxParticle.getMicrograph()): part.setCTF(CTFModel()) partSet.setHasCTF(True) if emxParticle.has('centerCoord__X'): part.setCoordinate(Coordinate()) #if emxParticle.has('transformationMatrix__t11'): # _particleFromEmx(emxParticle, part) # #partSet.setAlignment3D() # partSet.setAlignment(alignType) #else: # partSet.setAlignment(alignType) partSet.setAlignment(alignType) if not samplingRate: samplingRate = part.getSamplingRate() partSet.setSamplingRate(samplingRate) partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) particles = True else: # if not binary data, the coordinate case if micSet is None: raise Exception( 'Could not import Coordinates from EMX, micrographs not imported.' ) partSet = protocol._createSetOfCoordinates(micSet) part = Coordinate() particles = False copiedFiles = {} # copied or linked for emxParticle in emxData.iterClasses(emxlib.PARTICLE): if particles: _particleFromEmx(emxParticle, part) i, fn = part.getLocation() partFn = join(partDir, fn) newFn = join(outputDir, basename(partFn)) newLoc = (i, newFn) if not partFn in copiedFiles: copyOrLink(partFn, newFn) copiedFiles[partFn] = newFn part.setLocation(newLoc) if partSet.hasAlignment(): transform = Transform() _transformFromEmx(emxParticle, part, transform, alignType) part.setTransform(transform) else: _coordinateFromEmx(emxParticle, part) partSet.append(part) part.cleanObjId() if particles: protocol._defineOutputs(outputParticles=partSet) else: protocol._defineOutputs(outputCoordinates=partSet) if micSet is not None: protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter +=1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol(ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport1) protImport2 = self.newProtocol(ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def importCTF(self, mic, fileName): ctf = CTFModel() ctf.setMicrograph(mic) readCTFModel(ctf, fileName) return ctf