def getParticles(self, protImport, classid): dbPartSet = protImport._getPath("particles_class-%d.sqlite" % classid) class3D = protImport.outputClasses[classid] if os.path.exists(dbPartSet): os.remove(dbPartSet) partSet = SetOfParticles(filename=dbPartSet) partSet.copyInfo(class3D) for part in class3D: partSet.append(part) partSet.write() partSet.close() protImportCls1 = self.newProtocol( ProtImportParticles, objLabel='particles class-%d' % classid, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=dbPartSet, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImportCls1) self.assertIsNotNone(protImportCls1.outputParticles.getFileName(), "There was a problem with the import") return protImportCls1
def _createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) if doCtf: self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i] defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(emobj.Transform(a)) self.partSet.append(p) self.partSet.write()
def _getIterData(self, it): data_sqlite = self._getFileName('data_scipion', iter=it) if not os.path.exists(data_sqlite): iterImgSet = SetOfParticles(filename=data_sqlite) iterImgSet.copyInfo(self._getInputParticles()) self._fillDataFromIter(iterImgSet, it) iterImgSet.write() iterImgSet.close() return data_sqlite
class TestSets(pwtests.BaseTest): """Run different tests related to the editor set protocol.""" @classmethod def setUpClass(cls): pwtests.setupTestProject(cls) def _createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) if doCtf: self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i] defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(emobj.Transform(a)) self.partSet.append(p) self.partSet.write() def importData(self, baseFn, objLabel, protType, importFrom): prot = self.newProtocol(protType, objLabel=objLabel, filesPath=baseFn, maskPath=baseFn, sqliteFile=baseFn, haveDataBeenPhaseFlipped=False, magnification=10000, samplingRate=samplingRate, importFrom=importFrom ) self.launchProtocol(prot) return prot def testOperation(self): """Make a trivial operation. Increment defocusU by one""" setPartSqliteName = self.proj.getTmpPath("particles_operate.sqlite") setPartName = self.proj.getTmpPath('particles.stk') self._createSetOfParticles(setPartSqliteName, setPartName, doCtf=True) protImportProj = self.importData(setPartSqliteName, "import projection\n operation", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) #launch operate set protocol protSetEditor = self.newProtocol(ProtSetEditor, objLabel="operate") protSetEditor.inputSet.set(protImportProj) protSetEditor.inputSet.setExtended("outputParticles") protSetEditor.operation.set(ProtSetEditor.CHOICE_FORMULA) protSetEditor.formula.set('item._ctfModel._defocusU.set(item._ctfModel._defocusU.get() + 1. )') self.launchProtocol(protSetEditor) for item1, item2 in zip(protSetEditor.outputParticles, protImportProj.outputParticles): self.assertAlmostEqual(item1._ctfModel._defocusU.get() , item2._ctfModel._defocusU.get() + 1) def testRotation(self): """Rotate projections alignments so in the reconstruction vecor (0,0,1) y rotated to (1,0,0)""" setPartSqliteName = self.proj.getTmpPath("particles_rot_vec.sqlite") setPartName = self.proj.getTmpPath('particles.stk') self._createSetOfParticles(setPartSqliteName, setPartName, doCtf=True) protImportProj = self.importData(setPartSqliteName, "import projection\n rot vector", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) #launch operate set protocol protSetEditor = self.newProtocol(ProtSetEditor, objLabel="rotate") protSetEditor.inputSet.set(protImportProj) protSetEditor.inputSet.setExtended("outputParticles") protSetEditor.operation.set(ProtSetEditor.CHOICE_ROTATE_VECTOR) protSetEditor.xs.set(0.) protSetEditor.ys.set(0.) protSetEditor.zs.set(1.) protSetEditor.xt.set(1.) protSetEditor.yt.set(0.) protSetEditor.zt.set(0.) self.launchProtocol(protSetEditor) for controlPartSet, outPartSet in zip( tOut, protSetEditor.outputParticles, ): self.assertTrue(np.allclose(controlPartSet , outPartSet.getTransform().getMatrix())) def testRotationIco(self): """Rotate projections alignments between icosahedral symmetries""" setPartSqliteName = self.proj.getTmpPath("particles_rot_ico.sqlite") setPartName = self.proj.getTmpPath('particles.stk') self._createSetOfParticles(setPartSqliteName, setPartName, doCtf=True) protImportProj = self.importData(setPartSqliteName, "import projection\n ico sym", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) #launch operate set protocol protSetEditor = self.newProtocol(ProtSetEditor, objLabel="rotate") protSetEditor.inputSet.set(protImportProj) protSetEditor.inputSet.setExtended("outputParticles") protSetEditor.operation.set(ProtSetEditor.CHOICE_ROTATE_ICOSAHEDRAL) protSetEditor.originSymmetryGroup.set(SYM_I222 - SYM_I222) protSetEditor.targetSymmetryGroup.set(SYM_I222r - SYM_I222) self.launchProtocol(protSetEditor) for controlPartSet, outPartSet in zip(tIcoOut, protSetEditor.outputParticles, ): self.assertTrue(np.allclose(controlPartSet , outPartSet.getTransform().getMatrix()))
def testOrderBy(self): """ create set of particles and orderby a given attribute """ # This function was written by Roberto. It does things # differently, so let's keep it for reference. # create set of particles inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite') inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk') imgSet = SetOfParticles(filename=inFileNameMetadata) imgSet.setSamplingRate(1.5) acq = Acquisition() acq.setAmplitudeContrast(0.1) acq.setMagnification(10000) acq.setVoltage(200) acq.setSphericalAberration(2.0) imgSet.setAcquisition(acq) img = Particle() for i in range(1, 10): img.setLocation(i, inFileNameData) img.setMicId(i % 3) img.setClassId(i % 5) imgSet.append(img) img.cleanObjId() imgSet.write() # now import the dataset prot1 = self.newProtocol( emprot.ProtImportParticles, importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata, magnification=10000, samplingRate=1.5) prot1.setObjLabel('from sqlite (test-sets)') self.launchProtocol(prot1) if prot1.outputParticles is None: raise Exception( 'Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata) protSplitSet = self.newProtocol(emprot.ProtSplitSet, inputSet=prot1.outputParticles, numberOfSets=2, randomize=True) self.launchProtocol(protSplitSet) inputSets = [ protSplitSet.outputParticles01, protSplitSet.outputParticles02 ] outputSet = SetOfParticles( filename=self.proj.getTmpPath('gold.sqlite')) for itemSet in inputSets: for obj in itemSet: outputSet.append(obj) for item1, item2 in izip(imgSet, outputSet): if not item1.equalAttributes(item2): print("Items differ:") prettyDict(item1.getObjDict()) prettyDict(item2.getObjDict()) self.assertTrue(item1.equalAttributes(item2), )
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" # create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') # Start ids 4 img1 = Particle(objId=4) img2 = Particle(objId=4) attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): # Increment Id img1.setObjId(img1.getObjId() + 1) img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): # Increment Id img2.setObjId(img2.getObjId() + 1) img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() # import them protImport1 = self.newProtocol( emprot.ProtImportParticles, objLabel='import set1', importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( emprot.ProtImportParticles, objLabel='import set2', importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) # create merge protocol p_union = self.newProtocol(emprot.ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") # Assert ids self.assertEqual(counter + 5, img.getObjId(), "Object id's not kept.") counter += 1
def testMergeAlternateColumn(self): """Test that the union operation works as expected. Even if the order of the columns do not match. That is, M1(a,b,c) U M2(a,c,b)""" # Create two sets of particles inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() # import them protImport1 = self.newProtocol( emprot.ProtImportParticles, objLabel='import set1', importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( emprot.ProtImportParticles, objLabel='import set2', importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) # create merge protocol p_union = self.newProtocol(emprot.ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) # assert counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) counter += 1