def _createSubSetFromParticlesTiltPair(self, particlesTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = ParticlesTiltPair( filename=self._getPath('particles_pairs.sqlite')) inputU = particlesTiltPair.getUntilted() inputT = particlesTiltPair.getTilted() outputU = SetOfParticles( filename=self._getPath('particles_untilted.sqlite')) outputT = SetOfParticles( filename=self._getPath('particles_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) modifiedSet = ParticlesTiltPair(filename=self._dbName, prefix=self._dbPrefix) for pair, u, t in izip(modifiedSet, inputU, inputT): if pair.isEnabled(): output.append(pair) outputU.append(u) outputT.append(t) # Register outputs output.setUntilted(outputU) output.setTilted(outputT) outputDict = {'outputParticlesTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(particlesTiltPair, output) return output
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair): """ Create a subset of Micrographs Tilt Pair. """ output = MicrographsTiltPair( filename=self._getPath('micrographs_pairs.sqlite')) modifiedSet = MicrographsTiltPair(filename=self._dbName, prefix=self._dbPrefix) inputU = micrographsTiltPair.getUntilted() inputT = micrographsTiltPair.getTilted() outputU = SetOfMicrographs( filename=self._getPath('mics_untilted.sqlite')) outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) for micPairI in modifiedSet: untilted = micPairI.getUntilted() tilted = micPairI.getTilted() if micPairI.isEnabled(): micPairO = TiltPair() micPairO.setUntilted(untilted) micPairO.setTilted(tilted) output.append(micPairO) outputU.append(untilted) outputT.append(tilted) output.setUntilted(outputU) output.setTilted(outputT) # Register outputs outputDict = {'outputMicrographsTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(micrographsTiltPair, output) return output
def eliminationStep(self, fnInputMd): self.inputImages = self.inputParticles.get() partsFile = self.inputImages.getFileName() self.partsSet = SetOfParticles(filename=partsFile) self.partsSet.loadAllProperties() self.streamClosed = self.partsSet.isStreamClosed() if self.check == None: writeSetOfParticles(self.partsSet, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation') else: writeSetOfParticles(self.partsSet, fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') for p in self.partsSet.iterItems(orderBy='creation', direction='DESC'): self.check = p.getObjCreation() break self.partsSet.close() self.lenPartsSet = len(self.partsSet) print("os.path.exists(fnInputMd)): %s" % os.path.exists(fnInputMd)) args = "-i %s -o %s -e %s -t %f" % ( fnInputMd, self.fnOutputMd, self.fnElimMd, self.threshold.get()) if self.addFeatures: args += " --addFeatures" if self.useDenoising: args += " --useDenoising -d %f" % self.denoising.get() self.runJob("xmipp_image_eliminate_empty_particles", args) cleanPath(fnInputMd)
def _createCluster(self): """ Create the cluster with the selected particles from the cluster. This method will be called when the button 'Create Cluster' is pressed. """ # Write the particles prot = self.protocol project = prot.getProject() inputSet = prot.getInputParticles() fnSqlite = prot._getTmpPath('cluster_particles.sqlite') cleanPath(fnSqlite) partSet = SetOfParticles(filename=fnSqlite) partSet.copyInfo(inputSet) for point in self.getData(): if point.getState() == Point.SELECTED: particle = inputSet[point.getId()] partSet.append(particle) partSet.write() partSet.close() from protocol_batch_cluster import BatchProtNMACluster newProt = project.newProtocol(BatchProtNMACluster) clusterName = self.clusterWindow.getClusterName() if clusterName: newProt.setObjLabel(clusterName) newProt.inputNmaDimred.set(prot) newProt.sqliteFile.set(fnSqlite) project.launchProtocol(newProt)
def _checkProcessedData(self): if self.setof == SET_OF_MOVIES: objSet = SetOfMovies(filename=self._getPath('movies.sqlite')) elif self.setof == SET_OF_MICROGRAPHS: objSet = \ SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_RANDOM_MICROGRAPHS: objSet = \ SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_PARTICLES: objSet = SetOfParticles(filename=self._getPath('particles.sqlite')) else: raise Exception('Unknown data type') newObjSet, newObj = self._checkNewItems(objSet) if self.setof == SET_OF_MOVIES: newObjSet.setFramesRange(self.inputMovies.get().getFramesRange()) # check if end .... if self.setof != SET_OF_PARTICLES: self.nDims = self.nDim.get() endObjs = newObjSet.getSize() == self.nDims if newObj: if endObjs: newObjSet.setStreamState(newObjSet.STREAM_CLOSED) self._updateOutput(newObjSet) newObjSet.close()
def _loadInput(self): partsFile = self.inputParticles.get().getFileName() inPartsSet = SetOfParticles(filename=partsFile) inPartsSet.loadAllProperties() if self.check == None: writeSetOfParticles(inPartsSet, self.fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation') else: writeSetOfParticles(inPartsSet, self.fnInputMd, alignType=em.ALIGN_NONE, orderBy='creation', where='creation>"' + str(self.check) + '"') writeSetOfParticles(inPartsSet, self.fnInputOldMd, alignType=em.ALIGN_NONE, orderBy='creation', where='creation<"' + str(self.check) + '"') for p in inPartsSet.iterItems(orderBy='creation', direction='DESC'): self.check = p.getObjCreation() break streamClosed = inPartsSet.isStreamClosed() inputSize = inPartsSet.getSize() inPartsSet.close() return inputSize, streamClosed
def loadInputs(self): micsFn = self.getInputMicrographs().getFileName() micsSet = SetOfMicrographs(filename=micsFn) micsSet.loadAllProperties() availableMics = [] for mic in micsSet: availableMics.append(mic.getObjId()) micsSetClosed = micsSet.isStreamClosed() micsSet.close() partsFn = self.getInputParticles().getFileName() partsSet = SetOfParticles(filename=partsFn) partsSet.loadAllProperties() newParts = [] newMics = [] for item in partsSet: micKey = item.getCoordinate().getMicId() if micKey not in self.micsDone and micKey in availableMics: newParts.append(item.getObjId()) if not micKey in self.micsDone: newMics.append(micKey) self.micsDone += newMics self.inputSize = partsSet.getSize() partSetClosed = partsSet.isStreamClosed() partsSet.close() return newParts, micsSetClosed and partSetClosed
def getParticles(self, protImport, classid): dbPartSet = protImport._getPath("particles_class-%d.sqlite" % classid) class3D = protImport.outputClasses[classid] if os.path.exists(dbPartSet): os.remove(dbPartSet) partSet = SetOfParticles(filename=dbPartSet) partSet.copyInfo(class3D) for part in class3D: partSet.append(part) partSet.write() partSet.close() protImportCls1 = self.newProtocol( ProtImportParticles, objLabel='particles class-%d' % classid, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=dbPartSet, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImportCls1) self.assertIsNotNone(protImportCls1.outputParticles.getFileName(), "There was a problem with the import") return protImportCls1
def importParticles(self): """ Import particles from Frealign. Params: parFile: the filename of the parameter file with the alignment in Frealign. stackFile: single stack file with the images. """ partSet = self.protocol._createSetOfParticles() partSet.setObjComment('Particles imported from Frealign parfile:\n%s' % self.parFile) # Create a local link to the input stack file localStack = self.protocol._getExtraPath( os.path.basename(self.stackFile)) pwutils.createLink(self.stackFile, localStack) # Create a temporary set only with location tmpSet = SetOfParticles(filename=':memory:') tmpSet.readStack(localStack) self._setupSet(tmpSet) # Update both samplingRate and acquisition with parameters # selected in the protocol form self._setupSet(partSet) # Now read the alignment parameters from par file readSetOfParticles(tmpSet, partSet, self.parFile) partSet.setHasCTF(True) # Register the output set of particles self.protocol._defineOutputs(outputParticles=partSet)
def createSetOfParticles(self, setPartSqliteName, partFn, doCtf=False): # create a set of particles self.partSet = SetOfParticles(filename=setPartSqliteName) self.partSet.setAlignment(ALIGN_PROJ) self.partSet.setAcquisition(Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) self.partSet.setSamplingRate(samplingRate) self.partSet.setHasCTF(True) aList = [np.array(m) for m in mList] #defocus=15000 + 5000* random.random() for i, a in enumerate(aList): p = Particle() if doCtf: defocusU = defocusList[i]#+500. defocusV = defocusList[i] ctf = CTFModel(defocusU=defocusU, defocusV=defocusV, defocusAngle=defocusAngle[i]) ctf.standardize() p.setCTF(ctf) p.setLocation(i + 1, partFn) p.setTransform(Transform(a)) self.partSet.append(p) self.partSet.write()
def createOutputStep(self): # Create the SetOfImages object on the database #imgSet = XmippSetOfParticles(self._getPath('images.xmd')) fnImages = self._getOutputImgMd() # Create output SetOfParticles imgSet = self._createSetOfParticles() imgSet.copyInfo(self.inputMics) if self.doFlip: # Check if self.inputMics are phase flipped. if self.inputMics.isPhaseFlipped(): imgSet.setIsPhaseFlipped(False) else: imgSet.setIsPhaseFlipped(True) #imgSet.setHasCTF(self.fnCTF is not None) if self.downsampleType == OTHER: imgSet.setSamplingRate(self.inputMics.getSamplingRate() * self.downFactor.get()) imgSet.setCoordinates(self.inputCoords) # Create a temporary set to read from the metadata file # and later create the good one with the coordinates # properly set. We need this because the .update is not # working in the mapper when new attributes are added. imgSet.setHasCTF(self.ctfRelations.hasValue()) auxSet = SetOfParticles(filename=':memory:') auxSet.copyInfo(imgSet) readSetOfParticles(fnImages, auxSet) if self.downsampleType != SAME_AS_PICKING: factor = self.samplingInput / self.samplingFinal # For each particle retrieve micId from SetOFCoordinates and set it on the CTFModel for img in auxSet: #FIXME: This can be slow to make a query to grab the coord, maybe use zip(imgSet, coordSet)??? coord = self.inputCoords[img.getObjId()] if self.downsampleType != SAME_AS_PICKING: x, y = coord.getPosition() coord.setPosition(x * factor, y * factor) ctfModel = img.getCTF() if ctfModel is not None: ctfModel.setObjId(coord.getMicId()) ##img.setCTF(ctfModel)####JM img.setMicId(coord.getMicId()) img.setCoordinate(coord) imgSet.append(img) self._storeMethodsInfo(fnImages) self._defineOutputs(outputParticles=imgSet) self._defineSourceRelation(self.inputCoordinates, imgSet) if self.ctfRelations.hasValue(): self._defineSourceRelation(self.ctfRelations.get(), imgSet)
def convertInputStep(self, imagesMd): # It is unusual to create the output in the convertInputStep, # but just to avoid reading twice the sqlite with particles inputSet = self.inputNmaDimred.get().getInputParticles() partSet = self._createSetOfParticles() partSet.copyInfo(inputSet) tmpSet = SetOfParticles(filename=self.sqliteFile.get()) partSet.appendFromImages(tmpSet) # Register outputs partSet.setAlignmentProj() self._defineOutputs(outputParticles=partSet) self._defineTransformRelation(inputSet, partSet) writeSetOfParticles(partSet, imagesMd)
def loadAcquisitionInfo(self): """ Return a dictionary with acquisition values and the sampling rate information. In the case of Xmipp, they are stored in files: acquisition_info.xmd and microscope.xmd """ acquisitionDict = OrderedDict() inputSet = SetOfParticles(filename=self._sqliteFile) acquisitionDict['samplingRate'] = inputSet.getProperty('_samplingRate') acquisitionDict['voltage'] = inputSet.getProperty( '_acquisition._voltage') acquisitionDict['amplitudeContrast'] = inputSet.getProperty( '_acquisition._amplitudeContrast') acquisitionDict['magnification'] = int( float(inputSet.getProperty('_acquisition._magnification'))) return acquisitionDict
def test_mrcsLink(self): """ In this case just a link with .mrcs extension should be created """ stackFile = self.dsEmx.getFile('particles/particles.mrc') partSet = SetOfParticles(filename=':memory:') for i in range(1, 10): particle = Particle() particle.setLocation(i, stackFile) partSet.append(particle) outputDir = self.getOutputPath() filesDict = convertBinaryFiles(partSet, outputDir) print filesDict
def _allParticles(self, iterate=False): # A handler function to iterate over the particles inputSet = self.inputSet.get() if self.isInputClasses(): iterParticles = inputSet.iterClassItems() if iterate: return iterParticles else: particles = SetOfParticles(filename=":memory:") particles.copyInfo(inputSet.getFirstItem()) particles.copyItems(iterParticles) return particles else: if iterate: return inputSet.iterItems() else: return inputSet
def _checkNewInput(self): # Check if there are new particles to process from the input set partsFile = self.inputParticles.get().getFileName() partsSet = SetOfParticles(filename=partsFile) partsSet.loadAllProperties() self.SetOfParticles = [m.clone() for m in partsSet] self.streamClosed = partsSet.isStreamClosed() partsSet.close() partsSet = self._createSetOfParticles() readSetOfParticles(self._getExtraPath("allDone.xmd"), partsSet) newParts = any(m.getObjId() not in partsSet for m in self.SetOfParticles) outputStep = self._getFirstJoinStep() if newParts: fDeps = self._insertNewPartsSteps(self.insertedDict, self.SetOfParticles) if outputStep is not None: outputStep.addPrerequisites(*fDeps) self.updateSteps()
def loadAcquisitionInfo(self): """ Return a dictionary with acquisition values and the sampling rate information. In the case of Scipion, these values will be read from the 'Properties' table of the particles.sqlite file. """ acq = OrderedDict() inputSet = SetOfParticles(filename=self._sqliteFile) def _get(key): return inputSet.getProperty(key) acq['samplingRate'] = _get('_samplingRate') acq['voltage'] = _get('_acquisition._voltage') acq['amplitudeContrast'] = _get('_acquisition._amplitudeContrast') acq['sphericalAberration'] = float( _get('_acquisition._sphericalAberration')) return acq
def importParticles(self): """ Import particles from a metadata 'images.xmd' """ inputSet = SetOfParticles(filename=self._sqliteFile) inputSet.loadProperty('_alignment', ALIGN_NONE) inputSet.loadProperty('_hasCtf', False) self._findImagesPath(inputSet) partSet = self.protocol._createSetOfParticles() partSet.copyInfo(inputSet) partSet.setObjComment( 'Particles imported from Scipion sqlite file:\n%s' % self._sqliteFile) partSet.copyItems(inputSet, updateItemCallback=self._updateParticle) # Update both samplingRate and acquisition with parameters # selected in the protocol form self.protocol.setSamplingRate(partSet) self.protocol.fillAcquisition(partSet.getAcquisition()) partSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get()) self.protocol._defineOutputs(outputParticles=partSet)
def aaatest_particlesToStar(self): """ Write a SetOfParticles to Relion star input file. """ imgSet = SetOfParticles( filename=self.getOutputPath("particles.sqlite")) n = 10 fn = self.particles ctfs = [ CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15), CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25) ] acquisition = Acquisition(magnification=60000, voltage=300, sphericalAberration=2., amplitudeContrast=0.07) imgSet.setAcquisition(acquisition) coord = Coordinate() coord.setMicId(1) for i in range(n): p = Particle() p.setLocation(i + 1, fn) ctf = ctfs[i % 2] p.setCTF(ctf) p.setAcquisition(acquisition) p._xmipp_zScore = Float(i) coord.setX(i * 10) coord.setY(i * 10) p.setCoordinate(coord) imgSet.append(p) fnStar = self.getOutputPath('particles.star') fnStk = self.getOutputPath('particles.stk') print ">>> Writing to file: %s" % fnStar relion.writeSetOfParticles(imgSet, fnStar, fnStk) mdAll = md.MetaData(fnStar) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X)) self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y)) self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def test_hdfToStk(self): """ In this case the hdf stack files should be converted to .stk spider files for Relion. """ stackFiles = [ 'BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf' ] partSet = SetOfParticles(filename=':memory:') for fn in stackFiles: particle = Particle() particle.setLocation(1, self.ds.getFile('particles/%s' % fn)) partSet.append(particle) outputDir = self.getOutputPath() filesDict = convertBinaryFiles(partSet, outputDir) partSet.close() print filesDict
def testOrderBy(self): """ create set of particles and orderby a given attribute """ # This function was written by Roberto. It does things # differently, so let's keep it for reference. #create set of particles inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite') inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk') imgSet = SetOfParticles(filename=inFileNameMetadata) imgSet.setSamplingRate(1.5) acq = Acquisition() acq.setAmplitudeContrast(0.1) acq.setMagnification(10000) acq.setVoltage(200) acq.setSphericalAberration(2.0) imgSet.setAcquisition(acq) img = Particle() for i in range(1, 10): img.setLocation(i, inFileNameData) img.setMicId(i % 3) img.setClassId(i % 5) imgSet.append(img) img.cleanObjId() imgSet.write() #now import the dataset prot1 = self.newProtocol( ProtImportParticles, importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata, magnification=10000, samplingRate=1.5) prot1.setObjLabel('from sqlite (test-sets)') self.launchProtocol(prot1) if prot1.outputParticles is None: raise Exception( 'Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata) protSplitSet = self.newProtocol(ProtSplitSet, inputSet=prot1.outputParticles, numberOfSets=2, randomize=True) self.launchProtocol(protSplitSet) inputSets = [ protSplitSet.outputParticles01, protSplitSet.outputParticles02 ] outputSet = SetOfParticles( filename=self.proj.getTmpPath('gold.sqlite')) for itemSet in inputSets: for obj in itemSet: outputSet.append(obj) for item1, item2 in izip(imgSet, outputSet): if not item1.equalAttributes(item2): print "Items differ:" prettyDict(item1.getObjDict()) prettyDict(item2.getObjDict()) self.assertTrue(item1.equalAttributes(item2), )
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 # Test the join handles different attributes at a second level ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0) ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0) ctf2._myOwnQuality = Float(1.) img1.setCTF(ctf1) img2.setCTF(ctf2) for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join different attrs', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) self.assertFalse(hasattr(img, '_attrb3'), "join should not have attrb3") self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2") ctf = img.getCTF() self.assertIsNotNone(ctf, "Image should have CTF after join") self.assertFalse(hasattr(ctf, '_myOwnQuality'), "CTF should not have non common attributes") counter += 1
def testMergeAlternateColumn(self): """Test that the union operation works as expected. Even if the order of the columns do not match. That is, M1(a,b,c) U M2(a,c,b)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) #assert counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) counter += 1
def createOutputStep(self): fnTilted = self._getExtraPath("images_tilted.xmd") fnUntilted = self._getExtraPath("images_untilted.xmd") # Create outputs SetOfParticles both for tilted and untilted imgSetU = self._createSetOfParticles(suffix="Untilted") imgSetU.copyInfo(self.uMics) imgSetT = self._createSetOfParticles(suffix="Tilted") imgSetT.copyInfo(self.tMics) sampling = self.getMicSampling() if self._micsOther( ) else self.getCoordSampling() if self._doDownsample(): sampling *= self.downFactor.get() imgSetU.setSamplingRate(sampling) imgSetT.setSamplingRate(sampling) # set coords from the input, will update later if needed imgSetU.setCoordinates( self.inputCoordinatesTiltedPairs.get().getUntilted()) imgSetT.setCoordinates( self.inputCoordinatesTiltedPairs.get().getTilted()) # Read untilted and tilted particles on a temporary object (also disabled particles) imgSetAuxU = SetOfParticles(filename=':memory:') imgSetAuxU.copyInfo(imgSetU) readSetOfParticles(fnUntilted, imgSetAuxU, removeDisabled=False) imgSetAuxT = SetOfParticles(filename=':memory:') imgSetAuxT.copyInfo(imgSetT) readSetOfParticles(fnTilted, imgSetAuxT, removeDisabled=False) # calculate factor for coords scaling factor = 1 / self.samplingFactor if self._doDownsample(): factor /= self.downFactor.get() coordsT = self.getCoords().getTilted() # For each untilted particle retrieve micId from SetOfCoordinates untilted for imgU, coordU in izip(imgSetAuxU, self.getCoords().getUntilted()): # FIXME: Remove this check when sure that objIds are equal id = imgU.getObjId() if id != coordU.getObjId(): raise Exception( 'ObjIds in untilted img and coord are not the same!!!!') imgT = imgSetAuxT[id] coordT = coordsT[id] # If both particles are enabled append them if imgU.isEnabled() and imgT.isEnabled(): if self._micsOther() or self._doDownsample(): coordU.scale(factor) coordT.scale(factor) imgU.setCoordinate(coordU) imgSetU.append(imgU) imgT.setCoordinate(coordT) imgSetT.append(imgT) if self.doFlip: imgSetU.setIsPhaseFlipped(self.ctfUntilt.hasValue()) imgSetU.setHasCTF(self.ctfUntilt.hasValue()) imgSetT.setIsPhaseFlipped(self.ctfTilt.hasValue()) imgSetT.setHasCTF(self.ctfTilt.hasValue()) imgSetU.write() imgSetT.write() # Define output ParticlesTiltPair outputset = ParticlesTiltPair( filename=self._getPath('particles_pairs.sqlite')) outputset.setTilted(imgSetT) outputset.setUntilted(imgSetU) for imgU, imgT in izip(imgSetU, imgSetT): outputset.append(TiltPair(imgU, imgT)) outputset.setCoordsPair(self.inputCoordinatesTiltedPairs.get()) self._defineOutputs(outputParticlesTiltPair=outputset) self._defineSourceRelation(self.inputCoordinatesTiltedPairs, outputset)
def launchTest(self, fileKey, mList, alignType=None, **kwargs): """ Helper function to launch similar alignment tests give the EMX transformation matrix. Params: fileKey: the file where to grab the input stack images. mList: the matrix list of transformations (should be the same length of the stack of images) """ print "\n" print "*" * 80 print "* Launching test: ", fileKey print "*" * 80 is2D = alignType == ALIGN_2D stackFn = self.dataset.getFile(fileKey) partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite") mdFn = self.getOutputPath(fileKey + "_particles.star") partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite") if self.IS_ALIGNMENT: outputFn = self.getOutputPath(fileKey + "_output.mrcs") outputFnRelion = self.getOutputPath(fileKey + "_output") goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs') else: outputFn = self.getOutputPath(fileKey + "_output.vol") goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol') if PRINT_FILES: print "BINARY DATA: ", stackFn print "SET1: ", partFn1 print " MD: ", mdFn print "SET2: ", partFn2 print "OUTPUT: ", outputFn print "GOLD: ", goldFn if alignType == ALIGN_2D or alignType == ALIGN_PROJ: partSet = SetOfParticles(filename=partFn1) else: partSet = SetOfVolumes(filename=partFn1) partSet.setAlignment(alignType) partSet.setAcquisition( Acquisition(voltage=300, sphericalAberration=2, amplitudeContrast=0.1, magnification=60000)) # Populate the SetOfParticles with images # taken from images.mrc file # and setting the previous alignment parameters aList = [numpy.array(m) for m in mList] for i, a in enumerate(aList): p = Particle() p.setLocation(i + 1, stackFn) p.setTransform(Transform(a)) partSet.append(p) # Write out the .sqlite file and check that are correctly aligned print "Parset", partFn1 partSet.printAll() partSet.write() # Convert to a Xmipp metadata and also check that the images are # aligned correctly if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.writeSetOfParticles(partSet, mdFn, "/tmp", alignType=alignType) partSet2 = SetOfParticles(filename=partFn2) else: relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType) partSet2 = SetOfVolumes(filename=partFn2) # Let's create now another SetOfImages reading back the written # Xmipp metadata and check one more time. partSet2.copyInfo(partSet) if alignType == ALIGN_2D or alignType == ALIGN_PROJ: relion.readSetOfParticles(mdFn, partSet2, alignType=alignType) else: relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType) partSet2.write() if PRINT_MATRIX: for i, img in enumerate(partSet2): m1 = aList[i] m2 = img.getTransform().getMatrix() print "-" * 5 print img.getFileName(), img.getIndex() print 'm1:\n', m1, relion.geometryFromMatrix(m1, False) print 'm2:\n', m2, relion.geometryFromMatrix(m2, False) # self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2)) # Launch apply transformation and check result images runRelionProgram(self.CMD % locals()) if SHOW_IMAGES: runRelionProgram('scipion show %(outputFn)s' % locals()) if os.path.exists(goldFn): self.assertTrue( ImageHandler().compareData(goldFn, outputFn, tolerance=0.001), "Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testMergeDifferentAttrs(self): """ Test merge from subsets with different attritubes. That is, M1(a,b,c) U M2(a,b,c,d)""" #create two set of particles inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite') inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite') imgSet1 = SetOfParticles(filename=inFileNameMetadata1) imgSet2 = SetOfParticles(filename=inFileNameMetadata2) inFileNameData = self.proj.getTmpPath('particles.stk') img1 = Particle() img2 = Particle() attrb1 = [11, 12, 13, 14] attrb2 = [21, 22, 23, 24] attrb3 = [31, 32] counter = 0 for i in range(1, 3): img1.cleanObjId() img1.setLocation(i, inFileNameData) img1.setMicId(i % 3) img1.setClassId(i % 5) img1.setSamplingRate(1.) img1._attrb1 = Float(attrb1[counter]) img1._attrb2 = Float(attrb2[counter]) img1._attrb3 = Float(attrb3[counter]) imgSet1.append(img1) counter += 1 for i in range(1, 3): img2.cleanObjId() img2.setLocation(i, inFileNameData) img2.setClassId(i % 5) img2.setMicId(i % 3) img2.setSamplingRate(2.) img2._attrb1 = Float(attrb1[counter]) img2._attrb2 = Float(attrb2[counter]) imgSet2.append(img2) counter += 1 imgSet1.write() imgSet2.write() #import them protImport1 = self.newProtocol( ProtImportParticles, objLabel='import set1', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata1, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport1) protImport2 = self.newProtocol( ProtImportParticles, objLabel='import set2', importFrom=ProtImportParticles.IMPORT_FROM_SCIPION, sqliteFile=inFileNameMetadata2, magnification=10000, samplingRate=7.08, haveDataBeenPhaseFlipped=True) self.launchProtocol(protImport2) #create merge protocol p_union = self.newProtocol(ProtUnionSet, objLabel='join diff column order', ignoreExtraAttributes=True) p_union.inputSets.append(protImport1.outputParticles) p_union.inputSets.append(protImport2.outputParticles) self.proj.launchProtocol(p_union, wait=True) #assert counter = 0 for img in p_union.outputSet: self.assertAlmostEqual(attrb1[counter], img._attrb1, 4) self.assertAlmostEqual(attrb2[counter], img._attrb2, 4) if hasattr(img, '_attrb3'): self.assertTrue(False, "join should not have attrb3") if not hasattr(img, '_attrb2'): self.assertTrue(False, "join should have attrb2") counter += 1