def createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] #defocus=15000 + 5000* random.random() for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i]#+500. defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(Transform(a)) self.partSet.append(p) self.partSet.write()
def readPartsFromMics(self, micList, outputParts): """ Read the particles extract for the given list of micrographs and update the outputParts set with new items. """ p = Particle() for mic in micList: # We need to make this dict because there is no ID in the .xmd file coordDict = {} for coord in self.coordDict[mic.getObjId()]: pos = self._getPos(coord) if pos in coordDict: print( "WARNING: Ignoring duplicated coordinate: %s, id=%s" % (coord.getObjId(), pos)) coordDict[pos] = coord added = set() # Keep track of added coords to avoid duplicates for row in md.iterRows(self._getMicXmd(mic)): pos = (row.getValue(md.MDL_XCOOR), row.getValue(md.MDL_YCOOR)) coord = coordDict.get(pos, None) if coord is not None and coord.getObjId() not in added: # scale the coordinates according to particles dimension. coord.scale(self.getBoxScale()) p.copyObjId(coord) p.setLocation(xmippToLocation(row.getValue(md.MDL_IMAGE))) p.setCoordinate(coord) p.setMicId(mic.getObjId()) p.setCTF(mic.getCTF()) # adding the variance and Gini coeff. value of the mic zone setXmippAttributes(p, row, md.MDL_SCORE_BY_VAR) setXmippAttributes(p, row, md.MDL_SCORE_BY_GINI) if row.containsLabel(md.MDL_ZSCORE_DEEPLEARNING1): setXmippAttributes(p, row, md.MDL_ZSCORE_DEEPLEARNING1) # disabled particles (in metadata) should not add to the # final set if row.getValue(md.MDL_ENABLED) > 0: outputParts.append(p) added.add(coord.getObjId()) # Release the list of coordinates for this micrograph since it # will not be longer needed del self.coordDict[mic.getObjId()]
def aaatest_particlesToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles( filename=self.getOutputPath("particles.sqlite")) n = 10 fn = self.particles ctfs = [ CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles.star') fnStk = self.getOutputPath('particles.stk') print ">>> Writing to file: %s" % fnStar relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def test_particlesWithPhaseShiftToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite")) n = 10 fn = self.getFile('particles_binary') ctfs = [CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15, phaseShift=90), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25, phaseShift=60) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles_ph_sh.star') fnStk = self.getOutputPath('particles.stk') print (">>> Writing to file: %s" % fnStar) relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE)) self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def _particlesFromEmx(protocol, emxData, emxFile, outputDir, acquisition, samplingRate, copyOrLink, alignType=ALIGN_NONE): """ Create the output SetOfCoordinates or SetOfParticles given an EMXData object. Add CTF information to the particles if present. """ emxParticle = emxData.getFirstObject(emxlib.PARTICLE) partDir = dirname(emxFile) micSet = getattr(protocol, 'outputMicrographs', None) if emxParticle is not None: # Check if there are particles or coordinates fn = emxParticle.get(emxlib.FILENAME) if exists(join( partDir, fn)): # if the particles has binary data, means particles case partSet = protocol._createSetOfParticles() partSet.setAcquisition( acquisition) # this adquisition is per project #we need one per particle part = Particle() if _hasCtfLabels(emxParticle) or _hasCtfLabels( emxParticle.getMicrograph()): part.setCTF(CTFModel()) partSet.setHasCTF(True) if emxParticle.has('centerCoord__X'): part.setCoordinate(Coordinate()) #if emxParticle.has('transformationMatrix__t11'): # _particleFromEmx(emxParticle, part) # #partSet.setAlignment3D() # partSet.setAlignment(alignType) #else: # partSet.setAlignment(alignType) partSet.setAlignment(alignType) if not samplingRate: samplingRate = part.getSamplingRate() partSet.setSamplingRate(samplingRate) partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) particles = True else: # if not binary data, the coordinate case if micSet is None: raise Exception( 'Could not import Coordinates from EMX, micrographs not imported.' ) partSet = protocol._createSetOfCoordinates(micSet) part = Coordinate() particles = False copiedFiles = {} # copied or linked for emxParticle in emxData.iterClasses(emxlib.PARTICLE): if particles: _particleFromEmx(emxParticle, part) i, fn = part.getLocation() partFn = join(partDir, fn) newFn = join(outputDir, basename(partFn)) newLoc = (i, newFn) if not partFn in copiedFiles: copyOrLink(partFn, newFn) copiedFiles[partFn] = newFn part.setLocation(newLoc) if partSet.hasAlignment(): transform = Transform() _transformFromEmx(emxParticle, part, transform, alignType) part.setTransform(transform) else: _coordinateFromEmx(emxParticle, part) partSet.append(part) part.cleanObjId() if particles: protocol._defineOutputs(outputParticles=partSet) else: protocol._defineOutputs(outputCoordinates=partSet) if micSet is not None: protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
def _particlesFromEmx(protocol , emxData , emxFile , outputDir , acquisition , samplingRate , copyOrLink , alignType=ALIGN_NONE): """ Create the output SetOfCoordinates or SetOfParticles given an EMXData object. Add CTF information to the particles if present. """ emxParticle = emxData.getFirstObject(emxlib.PARTICLE) partDir = dirname(emxFile) micSet = getattr(protocol, 'outputMicrographs', None) if emxParticle is not None: # Check if there are particles or coordinates fn = emxParticle.get(emxlib.FILENAME) if exists(join(partDir, fn)): # if the particles has binary data, means particles case partSet = protocol._createSetOfParticles() partSet.setAcquisition(acquisition)# this adquisition is per project #we need one per particle part = Particle() if _hasCtfLabels(emxParticle) or _hasCtfLabels(emxParticle.getMicrograph()): part.setCTF(CTFModel()) partSet.setHasCTF(True) if emxParticle.has('centerCoord__X'): part.setCoordinate(Coordinate()) #if emxParticle.has('transformationMatrix__t11'): # _particleFromEmx(emxParticle, part) # #partSet.setAlignment3D() # partSet.setAlignment(alignType) #else: # partSet.setAlignment(alignType) partSet.setAlignment(alignType) if not samplingRate: samplingRate = part.getSamplingRate() partSet.setSamplingRate(samplingRate) partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get()) particles = True else: # if not binary data, the coordinate case if micSet is None: raise Exception('Could not import Coordinates from EMX, micrographs not imported.') partSet = protocol._createSetOfCoordinates(micSet) part = Coordinate() particles = False copiedFiles = {} # copied or linked for emxParticle in emxData.iterClasses(emxlib.PARTICLE): if particles: _particleFromEmx(emxParticle, part) i, fn = part.getLocation() partFn = join(partDir, fn) newFn = join(outputDir, basename(partFn)) newLoc = (i, newFn) if not partFn in copiedFiles: copyOrLink(partFn, newFn) copiedFiles[partFn] = newFn part.setLocation(newLoc) if partSet.hasAlignment(): transform = Transform() _transformFromEmx(emxParticle, part, transform, alignType) part.setTransform(transform) else: _coordinateFromEmx(emxParticle, part) partSet.append(part) part.cleanObjId() if particles: protocol._defineOutputs(outputParticles=partSet) else: protocol._defineOutputs(outputCoordinates=partSet) if micSet is not None: protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter +=1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol(ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport1) protImport2 = self.newProtocol(ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1