def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = MicrographsTiltPair( filename=self._getPath('micrographs_pairs.sqlite')) modifiedSet = MicrographsTiltPair(filename=self._dbName, prefix=self._dbPrefix) inputU = micrographsTiltPair.getUntilted() inputT = micrographsTiltPair.getTilted() outputU = SetOfMicrographs( filename=self._getPath('mics_untilted.sqlite')) outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) for micPairI in modifiedSet: untilted = micPairI.getUntilted() tilted = micPairI.getTilted() if micPairI.isEnabled(): micPairO = TiltPair() micPairO.setUntilted(untilted) micPairO.setTilted(tilted) output.append(micPairO) outputU.append(untilted) outputT.append(tilted) output.setUntilted(outputU) output.setTilted(outputT) # Register outputs outputDict = {'outputMicrographsTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(micrographsTiltPair, output) return output
def _createCluster(self): """ Create the cluster with the selected particles from the cluster. This method will be called when the button 'Create Cluster' is pressed. """ # Write the particles prot = self.protocol project = prot.getProject() inputSet = prot.getInputParticles() fnSqlite = prot._getTmpPath('cluster_particles.sqlite') cleanPath(fnSqlite) partSet = SetOfParticles(filename=fnSqlite) partSet.copyInfo(inputSet) for point in self.getData(): if point.getState() == Point.SELECTED: particle = inputSet[point.getId()] partSet.append(particle) partSet.write() partSet.close() from protocol_batch_cluster import BatchProtNMACluster newProt = project.newProtocol(BatchProtNMACluster) clusterName = self.clusterWindow.getClusterName() if clusterName: newProt.setObjLabel(clusterName) newProt.inputNmaDimred.set(prot) newProt.sqliteFile.set(fnSqlite) project.launchProtocol(newProt)
def _createSubSetFromParticlesTiltPair(self, particlesTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = ParticlesTiltPair( filename=self._getPath('particles_pairs.sqlite')) inputU = particlesTiltPair.getUntilted() inputT = particlesTiltPair.getTilted() outputU = SetOfParticles( filename=self._getPath('particles_untilted.sqlite')) outputT = SetOfParticles( filename=self._getPath('particles_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) modifiedSet = ParticlesTiltPair(filename=self._dbName, prefix=self._dbPrefix) for pair, u, t in izip(modifiedSet, inputU, inputT): if pair.isEnabled(): output.append(pair) outputU.append(u) outputT.append(t) # Register outputs output.setUntilted(outputU) output.setTilted(outputT) outputDict = {'outputParticlesTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(particlesTiltPair, output) return output
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = MicrographsTiltPair(filename=self._getPath('micrographs_pairs.sqlite')) print "self._dbName=%s" % self._dbName modifiedSet = MicrographsTiltPair(filename=self._dbName, prefix=self._dbPrefix) inputU = micrographsTiltPair.getUntilted() inputT = micrographsTiltPair.getTilted() outputU = SetOfMicrographs(filename=self._getPath('mics_untilted.sqlite')) outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) for micPairI in modifiedSet: untilted = micPairI.getUntilted() tilted = micPairI.getTilted() if micPairI.isEnabled(): micPairO = TiltPair() micPairO.setUntilted(untilted) micPairO.setTilted(tilted) output.append(micPairO) outputU.append(untilted) outputT.append(tilted) output.setUntilted(outputU) output.setTilted(outputT) # Register outputs outputDict = {'outputMicrographsTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(micrographsTiltPair, output) return output
def _createSubSetFromParticlesTiltPair(self, particlesTiltPair): print 'create subset from particles tilt pair' """ Create a subset of Micrographs Tilt Pair. """ output = ParticlesTiltPair(filename=self._getPath('particles_pairs.sqlite')) inputU = particlesTiltPair.getUntilted() inputT = particlesTiltPair.getTilted() outputU = SetOfParticles(filename=self._getPath('particles_untilted.sqlite')) outputT = SetOfParticles(filename=self._getPath('particles_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) modifiedSet = ParticlesTiltPair(filename=self._dbName, prefix=self._dbPrefix) for pair, u, t in izip(modifiedSet, inputU, inputT): if pair.isEnabled(): output.append(pair) outputU.append(u) outputT.append(t) # Register outputs output.setUntilted(outputU) output.setTilted(outputT) outputDict = {'outputParticlesTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(particlesTiltPair, output) return output
def getParticles(self, protImport, classid): dbPartSet = protImport._getPath("particles_class-%d.sqlite" % classid) class3D = protImport.outputClasses[classid] if os.path.exists(dbPartSet): os.remove(dbPartSet) partSet = SetOfParticles(filename=dbPartSet) partSet.copyInfo(class3D) for part in class3D: partSet.append(part) partSet.write() partSet.close() protImportCls1 = self.newProtocol( ProtImportParticles, objLabel='particles class-%d' % classid, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=dbPartSet, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImportCls1) self.assertIsNotNone(protImportCls1.outputParticles.getFileName(), "There was a problem with the import") return protImportCls1
def test_mrcsLink(self): """ In this case just a link with .mrcs extension should be created """ stackFile = self.dsEmx.getFile('particles/particles.mrc') partSet = SetOfParticles(filename=':memory:') for i in range(1, 10): particle = Particle() particle.setLocation(i, stackFile) partSet.append(particle) outputDir = self.getOutputPath() filesDict = convertBinaryFiles(partSet, outputDir) print filesDict
def test_particlesWithPhaseShiftToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite")) n = 10 fn = self.getFile('particles_binary') ctfs = [CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15, phaseShift=90), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25, phaseShift=60) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles_ph_sh.star') fnStk = self.getOutputPath('particles.stk') print (">>> Writing to file: %s" % fnStar) relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE)) self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
def aaatest_particlesToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles( filename=self.getOutputPath("particles.sqlite")) n = 10 fn = self.particles ctfs = [ CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles.star') fnStk = self.getOutputPath('particles.stk') print ">>> Writing to file: %s" % fnStar relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def test_hdfToStk(self): """ In this case the hdf stack files should be converted to .stk spider files for Relion. """ stackFiles = [ 'BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf' ] partSet = SetOfParticles(filename=':memory:') for fn in stackFiles: particle = Particle() particle.setLocation(1, self.ds.getFile('particles/%s' % fn)) partSet.append(particle) outputDir = self.getOutputPath() filesDict = convertBinaryFiles(partSet, outputDir) partSet.close() print filesDict
def test_hdfToStk(self): """ In this case the hdf stack files should be converted to .stk spider files for Relion. """ stackFiles = ['BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf'] partSet = SetOfParticles(filename=':memory:') for fn in stackFiles: particle = Particle() particle.setLocation(1, self.ds.getFile('particles/%s' % fn)) partSet.append(particle) outputDir = self.getOutputPath() filesDict = convertBinaryFiles(partSet, outputDir) partSet.close() print filesDict
def testMergeAlternateColumn(self): """Test that the union operation works as expected. Even if the order of the columns do not match. That is, M1(a,b,c) U M2(a,c,b)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) imgSet1.append(img1) counter +=1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1= Float(attrb1[counter]) img2._attrb2= Float(attrb2[counter]) imgSet2.append(img2) counter +=1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol(ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport1) protImport2 = self.newProtocol(ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) #assert counter=0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter],img._attrb1,4) self.assertAlmostEqual(attrb2[counter],img._attrb2,4) counter += 1
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter +=1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol(ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport1) protImport2 = self.newProtocol(ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True ) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def launchTest(self, fileKey, mList, alignType=None, **kwargs): """ Helper function to launch similar alignment tests give the EMX transformation matrix. Params: fileKey: the file where to grab the input stack images. mList: the matrix list of transformations (should be the same length of the stack of images) """ print ("\n") print ("*" * 80) print ("* Launching test: ", fileKey) print ("*" * 80) is2D = alignType == ALIGN_2D stackFn = self.dataset.getFile(fileKey) partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite") mdFn = self.getOutputPath(fileKey + "_particles.star") partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite") if self.IS_ALIGNMENT: outputFn = self.getOutputPath(fileKey + "_output.mrcs") outputFnRelion = self.getOutputPath(fileKey + "_output") goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs') else: outputFn = self.getOutputPath(fileKey + "_output.vol") goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol') if PRINT_FILES: print("BINARY DATA: ", stackFn) print("SET1: ", partFn1) print(" MD: ", mdFn) print("SET2: ", partFn2) print("OUTPUT: ", outputFn) print("GOLD: ", goldFn) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: partSet = SetOfParticles(filename=partFn1) else: partSet = SetOfVolumes(filename=partFn1) partSet.setAlignment(alignType) partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) # Populate the SetOfParticles with images # taken from images.mrc file # and setting the previous alignment parameters aList = [numpy.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() p.setLocation(i + 1, stackFn) p.setTransform(Transform(a)) partSet.append(p) # Write out the .sqlite file and check that are correctly aligned print ("Parset", partFn1) partSet.printAll() partSet.write() # Convert to a Xmipp metadata and also check that the images are # aligned correctly if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.writeSetOfParticles(partSet, mdFn,"/tmp", alignType=alignType) partSet2 = SetOfParticles(filename=partFn2) else: relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType) partSet2 = SetOfVolumes(filename=partFn2) # Let's create now another SetOfImages reading back the written # Xmipp metadata and check one more time. partSet2.copyInfo(partSet) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.readSetOfParticles(mdFn, partSet2, alignType=alignType) else: relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType) partSet2.write() if PRINT_MATRIX: for i, img in enumerate(partSet2): m1 = aList[i] m2 = img.getTransform().getMatrix() print ("-" * 5) print (img.getFileName(), img.getIndex()) print ('m1:\n', m1, relion.geometryFromMatrix(m1, False)) print ('m2:\n', m2, relion.geometryFromMatrix(m2, False)) # self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2)) # Launch apply transformation and check result images runRelionProgram(self.CMD % locals()) if SHOW_IMAGES: runRelionProgram('scipion show %(outputFn)s' % locals()) if os.path.exists(goldFn): self.assertTrue( ImageHandler().compareData(goldFn, outputFn, tolerance=0.001), "Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testOrderBy(self): """ create set of particles and orderby a given attribute """ # This function was written by Roberto. It does things # differently, so let's keep it for reference. #create set of particles inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite') inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk') imgSet = SetOfParticles(filename=inFileNameMetadata) imgSet.setSamplingRate(1.5) acq = Acquisition() acq.setAmplitudeContrast(0.1) acq.setMagnification(10000) acq.setVoltage(200) acq.setSphericalAberration(2.0) imgSet.setAcquisition(acq) img = Particle() for i in range(1, 10): img.setLocation(i, inFileNameData) img.setMicId(i%3) img.setClassId(i%5) imgSet.append(img) img.cleanObjId() imgSet.write() #now import the dataset prot1 = self.newProtocol(ProtImportParticles, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata, magnification=10000, samplingRate=1.5 ) prot1.setObjLabel('from sqlite (test-sets)') self.launchProtocol(prot1) if prot1.outputParticles is None: raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata) protSplitSet = self.newProtocol(ProtSplitSet, inputSet=prot1.outputParticles, numberOfSets=2, randomize=True) self.launchProtocol(protSplitSet) inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02] outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite')) for itemSet in inputSets: for obj in itemSet: outputSet.append(obj) for item1, item2 in izip(imgSet, outputSet): if not item1.equalAttributes(item2): print "Items differ:" prettyDict(item1.getObjDict()) prettyDict(item2.getObjDict()) self.assertTrue(item1.equalAttributes(item2), )
def testOrderBy(self): """ create set of particles and orderby a given attribute """ # This function was written by Roberto. It does things # differently, so let's keep it for reference. #create set of particles inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite') inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk') imgSet = SetOfParticles(filename=inFileNameMetadata) imgSet.setSamplingRate(1.5) acq = Acquisition() acq.setAmplitudeContrast(0.1) acq.setMagnification(10000) acq.setVoltage(200) acq.setSphericalAberration(2.0) imgSet.setAcquisition(acq) img = Particle() for i in range(1, 10): img.setLocation(i, inFileNameData) img.setMicId(i % 3) img.setClassId(i % 5) imgSet.append(img) img.cleanObjId() imgSet.write() #now import the dataset prot1 = self.newProtocol( ProtImportParticles, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata, magnification=10000, samplingRate=1.5) prot1.setObjLabel('from sqlite (test-sets)') self.launchProtocol(prot1) if prot1.outputParticles is None: raise Exception( 'Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata) protSplitSet = self.newProtocol(ProtSplitSet, inputSet=prot1.outputParticles, numberOfSets=2, randomize=True) self.launchProtocol(protSplitSet) inputSets = [ protSplitSet.outputParticles01, protSplitSet.outputParticles02 ] outputSet = SetOfParticles( filename=self.proj.getTmpPath('gold.sqlite')) for itemSet in inputSets: for obj in itemSet: outputSet.append(obj) for item1, item2 in izip(imgSet, outputSet): if not item1.equalAttributes(item2): print "Items differ:" prettyDict(item1.getObjDict()) prettyDict(item2.getObjDict()) self.assertTrue(item1.equalAttributes(item2), )
def testMergeAlternateColumn(self): """Test that the union operation works as expected. Even if the order of the columns do not match. That is, M1(a,b,c) U M2(a,c,b)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) #assert counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) counter += 1
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def launchTest(self, fileKey, mList, alignType=None, **kwargs): """ Helper function to launch similar alignment tests give the EMX transformation matrix. Params: fileKey: the file where to grab the input stack images. mList: the matrix list of transformations (should be the same length of the stack of images) """ print "\n" print "*" * 80 print "* Launching test: ", fileKey print "*" * 80 is2D = alignType == ALIGN_2D stackFn = self.dataset.getFile(fileKey) partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite") mdFn = self.getOutputPath(fileKey + "_particles.star") partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite") if self.IS_ALIGNMENT: outputFn = self.getOutputPath(fileKey + "_output.mrcs") outputFnRelion = self.getOutputPath(fileKey + "_output") goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs') else: outputFn = self.getOutputPath(fileKey + "_output.vol") goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol') if PRINT_FILES: print "BINARY DATA: ", stackFn print "SET1: ", partFn1 print " MD: ", mdFn print "SET2: ", partFn2 print "OUTPUT: ", outputFn print "GOLD: ", goldFn if alignType == ALIGN_2D or alignType == ALIGN_PROJ: partSet = SetOfParticles(filename=partFn1) else: partSet = SetOfVolumes(filename=partFn1) partSet.setAlignment(alignType) partSet.setAcquisition( Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) # Populate the SetOfParticles with images # taken from images.mrc file # and setting the previous alignment parameters aList = [numpy.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() p.setLocation(i + 1, stackFn) p.setTransform(Transform(a)) partSet.append(p) # Write out the .sqlite file and check that are correctly aligned print "Parset", partFn1 partSet.printAll() partSet.write() # Convert to a Xmipp metadata and also check that the images are # aligned correctly if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.writeSetOfParticles(partSet, mdFn, "/tmp", alignType=alignType) partSet2 = SetOfParticles(filename=partFn2) else: relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType) partSet2 = SetOfVolumes(filename=partFn2) # Let's create now another SetOfImages reading back the written # Xmipp metadata and check one more time. partSet2.copyInfo(partSet) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.readSetOfParticles(mdFn, partSet2, alignType=alignType) else: relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType) partSet2.write() if PRINT_MATRIX: for i, img in enumerate(partSet2): m1 = aList[i] m2 = img.getTransform().getMatrix() print "-" * 5 print img.getFileName(), img.getIndex() print 'm1:\n', m1, relion.geometryFromMatrix(m1, False) print 'm2:\n', m2, relion.geometryFromMatrix(m2, False) # self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2)) # Launch apply transformation and check result images runRelionProgram(self.CMD % locals()) if SHOW_IMAGES: runRelionProgram('scipion show %(outputFn)s' % locals()) if os.path.exists(goldFn): self.assertTrue( ImageHandler().compareData(goldFn, outputFn, tolerance=0.001), "Different data files:\n>%s\n<%s" % (goldFn, outputFn))
class TestSubProj(BaseTest): @classmethod def setUpClass(cls): setupTestProject(cls) def createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] #defocus=15000 + 5000* random.random() for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i]#+500. defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(Transform(a)) self.partSet.append(p) self.partSet.write() def createProjection(self, proj, num, baseName): img = xmipp.Image() img.setDataType(xmipp.DT_FLOAT) img.resize(projSize, projSize) #img.initRandom(0., 1., xmipp.XMIPP_RND_GAUSSIAN) img.initConstant(0.) for coor in proj: value = img.getPixel(coor[0], coor[1], coor[2], coor[3]) img.setPixel(coor[0], coor[1], coor[2], coor[3], coor[4]+value) # coor4 is the pixel value img.write("%d@"%num + baseName) def createVol(self, volume): vol = xmipp.Image() vol.setDataType(xmipp.DT_FLOAT) vol.resize(projSize, projSize, projSize) #vol.initRandom(0., .5, xmipp.XMIPP_RND_UNIFORM) vol.initConstant(0.) for coor in volume: vol.setPixel(coor[0], coor[1], coor[2], coor[3], coor[4]) # coor4 is the pixel value vol.write(self.volBaseFn) def createMask(self, _maskName): vol = xmipp.Image() vol.setDataType(xmipp.DT_FLOAT) vol.resize(projSize, projSize, projSize) vol.initConstant(0.0)#ROB: not sure this is needed halfDim = int(projSize/2) maskRadius2 = maskRadius * maskRadius for i in range(-halfDim,halfDim): for j in range(-halfDim,halfDim): for k in range(-halfDim,halfDim): if(i*i+j*j+k*k) < maskRadius2: vol.setPixel(0, k+halfDim, i+halfDim, j+halfDim, 1.) # coor4 is the pixel value vol.write(_maskName) def applyCTF(self, setPartMd): writeSetOfParticles(self.partSet,setPartMd) md1 = xmipp.MetaData() md1.setColumnFormat(False) idctf = md1.addObject() _acquisition = self.partSet.getAcquisition() for part in self.partSet: baseFnCtf = self.proj.getTmpPath("kk")#self._getTmpPath("ctf_%d.param"%mic) md1.setValue(xmipp.MDL_CTF_SAMPLING_RATE, samplingRate, idctf) md1.setValue(xmipp.MDL_CTF_VOLTAGE, 200., idctf); ctf = part.getCTF() udefocus = ctf.getDefocusU() vdefocus = ctf.getDefocusV() angle = ctf.getDefocusAngle() md1.setValue(xmipp.MDL_CTF_DEFOCUSU, udefocus, idctf); md1.setValue(xmipp.MDL_CTF_DEFOCUSV, vdefocus, idctf); md1.setValue(xmipp.MDL_CTF_DEFOCUS_ANGLE, 180.0 * random.random(), idctf); md1.setValue(xmipp.MDL_CTF_CS, 2., idctf); md1.setValue(xmipp.MDL_CTF_Q0, 0.07, idctf); md1.setValue(xmipp.MDL_CTF_K, 1., idctf); md1.write(baseFnCtf) ##writeSetOfParticles(self.partSet, setPartMd) #apply ctf args = " -i %s"%setPartMd args += " -o %s"%self.proj.getTmpPath(setPartCtfName) args += " -f ctf %s"%baseFnCtf args += " --sampling %f"%samplingRate runXmippProgram("xmipp_transform_filter", args) args = " -i %s"%setPartMd args += " -o %s"%self.proj.getTmpPath(setPartCtfPosName) args += " -f ctfpos %s"%baseFnCtf args += " --sampling %f"%samplingRate runXmippProgram("xmipp_transform_filter", args) def importData(self, baseFn, objLabel, protType, importFrom): prot = self.newProtocol(protType, objLabel=objLabel, filesPath=baseFn, maskPath=baseFn, sqliteFile=baseFn, haveDataBeenPhaseFlipped=False, magnification=10000, samplingRate=samplingRate, importFrom=importFrom ) self.launchProtocol(prot) return prot def test_pattern(self): #1) create fake protocol so I have a place to save data #prot = self.launchFakeProtocol() #output stack self.setPartName = self.proj.getTmpPath(setPartName) self.setPartSqliteName = self.proj.getTmpPath(setPartSqliteName) self.setPartSqliteCtfName = self.proj.getTmpPath(setPartSqliteCtfName) self.setPartSqliteCTfPosName = self.proj.getTmpPath(setPartSqliteCTfPosName) self.kksqlite = self.proj.getTmpPath("kk.sqlite") self.setPartMd = self.proj.getTmpPath(setPartNameMd) self.volBaseFn = self.proj.getTmpPath(volName) self.maskName = self.proj.getTmpPath(maskName) #2) create projections and sets of particles self.createProjection(proj1, 1, self.setPartName) self.createProjection(proj2, 2, self.setPartName) self.createProjection(proj3, 3, self.setPartName) self.createSetOfParticles(self.setPartSqliteCTfPosName, self.proj.getTmpPath(setPartCtfPosName), True) self.createSetOfParticles(self.setPartSqliteCtfName, self.proj.getTmpPath(setPartCtfName), True) self.createSetOfParticles(self.setPartSqliteName, self.setPartName, False) #create auxiliary setofparticles self.createSetOfParticles(self.kksqlite, self.setPartName, True) #4) apply CTF self.applyCTF(self.setPartMd) #5) create volume self.createVol(vol1) #6) create mask self.createMask(self.maskName) #import three projection datasets, volume and mask protPlainProj = self.importData(self.setPartSqliteName, "plain projection", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) protCTFProj = self.importData(self.setPartSqliteCtfName,"ctf projection", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) protCTFposProj = self.importData(self.setPartSqliteCTfPosName, "pos ctf projection", ProtImportParticles, ProtImportParticles.IMPORT_FROM_SCIPION) _protImportVol = self.importData(os.path.abspath(self.proj.getTmpPath(volName)), "3D reference", ProtImportVolumes, ProtImportParticles.IMPORT_FROM_FILES) _protImportMask = self.importData(self.proj.getTmpPath(maskName), "3D mask", ProtImportMask, ProtImportParticles.IMPORT_FROM_FILES) mask = VolumeMask() mask.setFileName(self.proj.getTmpPath(maskName)) mask.setSamplingRate(samplingRate) #launch substract protocol <<<<<<<<<<<<<<<<<<<<<<<<<< protSubtract = self.newProtocol(XmippProtSubtractProjection) protSubtract.inputParticles.set(protPlainProj.outputParticles) protSubtract.inputVolume.set(_protImportVol.outputVolume) protSubtract.refMask.set(_protImportMask.outputMask) protSubtract.projType.set(XmippProtSubtractProjection.CORRECT_NONE) self.launchProtocol(protSubtract) protSubtractCTF = self.newProtocol(XmippProtSubtractProjection) protSubtractCTF.inputParticles.set(protCTFProj.outputParticles) protSubtractCTF.inputVolume.set(_protImportVol.outputVolume) protSubtractCTF.refMask.set(_protImportMask.outputMask) protSubtractCTF.projType.set(XmippProtSubtractProjection.CORRECT_FULL_CTF) self.launchProtocol(protSubtractCTF) protSubtractCTFpos = self.newProtocol(XmippProtSubtractProjection) protSubtractCTFpos.inputParticles.set(protCTFposProj.outputParticles) protSubtractCTFpos.inputVolume.set(_protImportVol.outputVolume) protSubtractCTFpos.refMask.set(_protImportMask.outputMask) protSubtractCTFpos.projType.set(XmippProtSubtractProjection.CORRECT_PHASE_FLIP) self.launchProtocol(protSubtractCTFpos) protSubtractCTFRelion = self.newProtocol(ProtRelionSubtract) protSubtractCTFRelion.inputParticles.set(protCTFProj.outputParticles) protSubtractCTFRelion.inputVolume.set(_protImportVol.outputVolume) self.launchProtocol(protSubtractCTFRelion) self.assertIsNotNone(protSubtract.outputParticles, "There was a problem with subtract projection") self.assertTrue(True)
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) #assert counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) if hasattr(img, '_attrb3'): self.assertTrue(False, "join should not have attrb3") if not hasattr(img, '_attrb2'): self.assertTrue(False, "join should have attrb2") counter += 1