def _createSubSetFromParticlesTiltPair(self, particlesTiltPair): """ Create a subset of Particles Tilt Pair. """ output = ParticlesTiltPair( filename=self._getPath('particles_pairs.sqlite')) inputU = particlesTiltPair.getUntilted() inputT = particlesTiltPair.getTilted() outputU = emobj.SetOfParticles( filename=self._getPath('particles_untilted.sqlite')) outputT = emobj.SetOfParticles( filename=self._getPath('particles_tilted.sqlite')) outputU.copyInfo(inputU) outputT.copyInfo(inputT) modifiedSet = ParticlesTiltPair(filename=self._dbName, prefix=self._dbPrefix) # noinspection DuplicatedCode for pair, u, t in izip(modifiedSet, inputU, inputT): if pair.isEnabled(): output.append(pair) outputU.append(u) outputT.append(t) # Register outputs output.setUntilted(outputU) output.setTilted(outputT) # Link output to the same coordinates pairs than input output.setCoordsPair(particlesTiltPair.getCoordsPair()) outputDict = {'outputParticlesTiltPair': output} self._defineOutputs(**outputDict) self._defineTransformRelation(particlesTiltPair, output) return output
def _checkProcessedData(self): if self.setof == SET_OF_MOVIES: objSet = emobj.SetOfMovies(filename=self._getPath('movies.sqlite')) elif self.setof == SET_OF_MICROGRAPHS: objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_RANDOM_MICROGRAPHS: objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite')) elif self.setof == SET_OF_PARTICLES: objSet = emobj.SetOfParticles(filename=self._getPath('particles.sqlite')) else: raise Exception('Unknown data type') newObjSet, newObj = self._checkNewItems(objSet) if self.setof == SET_OF_MOVIES: newObjSet.setFramesRange(self.inputMovies.get().getFramesRange()) # check if end .... if self.setof != SET_OF_PARTICLES: self.nDims = self.nDim.get() endObjs = newObjSet.getSize() == self.nDims if newObj: if endObjs: newObjSet.setStreamState(newObjSet.STREAM_CLOSED) self._updateOutput(newObjSet) newObjSet.close()
def loadInputs(self): micsFn = self.getInputMicrographs().getFileName() micsSet = emobj.SetOfMicrographs(filename=micsFn) micsSet.loadAllProperties() availableMics = [] for mic in micsSet: availableMics.append(mic.getObjId()) micsSetClosed = micsSet.isStreamClosed() micsSet.close() partsFn = self.getInputParticles().getFileName() partsSet = emobj.SetOfParticles(filename=partsFn) partsSet.loadAllProperties() newParts = [] newMics = [] for item in partsSet: micKey = item.getCoordinate().getMicId() if micKey not in self.micsDone and micKey in availableMics: newParts.append(item.getObjId()) if micKey not in self.micsDone: newMics.append(micKey) self.micsDone += newMics self.inputSize = partsSet.getSize() partSetClosed = partsSet.isStreamClosed() partsSet.close() return newParts, micsSetClosed and partSetClosed
def test_subsetsFromSelection(self): """ From a sqlite file of a SetOfClasses2D, with some classes and element disabled, we want to create a subset of images and classes. """ classes2DSet = emobj.SetOfClasses2D(filename=self.selectionFn) imgSet = emobj.SetOfParticles(filename=':memory:') # We are going to iterate over the enabled items and create # a new set of images imgSet.appendFromClasses(classes2DSet) # Since we have disabled two classes (6 images) and 1 images # from class 1 and 2 (2 images), the final size of the set # should be 68 sizes = [32, 36] self.assertEqual(imgSet.getSize(), sum(sizes)) imgSet.clear() # Close db connection and clean data # Now create a subset of classes and check the number # of images per class clsSet = emobj.SetOfClasses2D(filename=':memory:') clsSet.appendFromClasses(classes2DSet) for i, cls in enumerate(clsSet): self.assertEqual(cls.getSize(), sizes[i]) clsSet.clear() # Close db connection and clean data
def test_getFiles(self): # create setofImages dbFn = self.getOutputPath('multistack_set.sqlite') # dbFn = ':memory:' n = 10 m = 3 img = emobj.Particle() imgSet = emobj.SetOfParticles(filename=dbFn) imgSet.setSamplingRate(1.0) goldStacks = set() for j in range(1, m + 1): stackName = 'stack%02d.stk' % j goldStacks.add(stackName) for i in range(1, n + 1): img.setLocation(i, stackName) imgSet.append(img) img.cleanObjId() self.assertEquals(goldStacks, imgSet.getFiles()) imgSet.close() # It should automatically call load # before accessing items db imgSet.getFirstItem()
def test_hugeSet(self): """ Create a set of a big number of particles to measure creation time with sqlite operations. """ # Allow what huge means to be defined with environment var n = int(os.environ.get('SCIPION_TEST_HUGE', 10000)) print(">>>> Creating a set of %d particles." % n) print(" (set SCIPION_TEST_HUGE environment var to other value)") dbFn = self.getOutputPath('huge_set.sqlite') # dbFn = ':memory:' img = emobj.Particle() imgSet = emobj.SetOfParticles(filename=dbFn) imgSet.setSamplingRate(1.0) for i in range(1, n + 1): # Creating object inside the loop significantly # decrease performance # img = Particle() img.setLocation(i, "images.stk") imgSet.append(img) img.cleanObjId() # img.setObjId(None) imgSet.write()
def test_orderBy(self): # create setofProjections sorted imgSet1 = emobj.SetOfParticles(filename=':memory:', prefix='set1') imgSet2 = emobj.SetOfParticles(filename=':memory:', prefix='set2') imgSet1.setSamplingRate(1.5) imgSet2.setSamplingRate(1.5) img = emobj.Particle() for i in range(1, 10): img.setLocation(i, 'mystack.stk') img.setMicId(10 - i) img.setClassId(i % 5) imgSet1.append(img) img.setMicId(i) imgSet2.append(img) img.cleanObjId() # orderby for item1, item2 in zip( imgSet1.iterItems(orderBy='_micId', direction='ASC'), imgSet2.iterItems(orderBy='_micId', direction='ASC')): self.assertEquals(item1.getMicId(), item2.getMicId())
def test_readStack(self): """ Read an stack of 29 particles from .hdf file. Particles should be of 500x500 pixels. """ size = 29 xdim = 500 inStack = self.dataset.getFile('particles1') outFn = self.getOutputPath('particles.sqlite') imgSet = emobj.SetOfParticles(filename=outFn) imgSet.setSamplingRate(1.0) imgSet.readStack(inStack) # This should add 29 new items to the set self.assertEquals(size, imgSet.getSize()) # Check same size self.assertEquals(xdim, imgSet.getDim()[0]) # Check same dimensions print("writing particles to: ", outFn) imgSet.write() imgSet2 = emobj.SetOfParticles(filename=':memory:') imgSet2.copyInfo(imgSet) self.assertAlmostEqual(imgSet.getSamplingRate(), imgSet2.getSamplingRate())
def test_listAttribute(self): """ Test the use of copyItems function where the items have a property that is a list. """ inFn = self.dataset.getFile('particles/particles_nma.sqlite') outFn = self.getOutputPath('particles.sqlite') inputSet = emobj.SetOfParticles(filename=inFn) inputSet.setSamplingRate(1.0) outputSet = emobj.SetOfParticles(filename=outFn) outputSet.copyInfo(inputSet) outputSet.copyItems(inputSet, updateItemCallback=self._updateItem) # print("writing particles to: ", outFn) outputSet.write() outputSet.close() # Read again the written set of particles checkSet = emobj.SetOfParticles(filename=outFn) # check that the first item (as the rest) # have the added _list attribute with the correct values particle = checkSet.getFirstItem() self.assertTrue(particle.hasAttribute('_list')) for v1, v2 in zip([1.0, 2.0], particle._list): self.assertAlmostEqual(v1, float(v2)) # check that copied items have exactly # the same attribes than the input set # except for the _list for i1, i2 in zip(inputSet, checkSet): self.assertTrue(i2.equalAttributes(i1, ignore=['_list']))
def _getIterParticles(self, it, clean=False): import pwem.objects as em """ Return a classes .sqlite file for this iteration. If the file doesn't exists, it will be created by converting from this iteration data.star file. """ dataParticles = self._getFileName('particlesScipion', iter=it) if clean: cleanPath(dataParticles) if not exists(dataParticles): partSet = em.SetOfParticles(filename=dataParticles) self._fillParticlesFromIter(partSet, it) partSet.write() partSet.close() else: partSet = em.SetOfParticles(filename=dataParticles) imgSet = self.inputParticles.get() partSet.copyInfo(imgSet) partSet.setAlignmentProj() return partSet
def loadAcquisitionInfo(self): """ Return a dictionary with acquisition values and the sampling rate information. In the case of Scipion, these values will be read from the 'Properties' table of the particles.sqlite file. """ acq = OrderedDict() inputSet = emobj.SetOfParticles(filename=self._sqliteFile) def _get(key): return inputSet.getProperty(key) acq['samplingRate'] = _get('_samplingRate') acq['voltage'] = _get('_acquisition._voltage') acq['amplitudeContrast'] = _get('_acquisition._amplitudeContrast') acq['sphericalAberration'] = float(_get('_acquisition._sphericalAberration')) return acq
def importParticles(self): """ Import particles from a metadata 'images.xmd' """ inputSet = emobj.SetOfParticles(filename=self._sqliteFile) inputSet.loadProperty('_alignment', emcts.ALIGN_NONE) inputSet.loadProperty('_hasCtf', False) self._findImagesPath(inputSet) partSet = self.protocol._createSetOfParticles() partSet.copyInfo(inputSet) partSet.setObjComment('Particles imported from Scipion sqlite file:\n%s' % self._sqliteFile) partSet.copyItems(inputSet, updateItemCallback=self._updateParticle) # Update both samplingRate and acquisition with parameters # selected in the protocol form self.protocol.setSamplingRate(partSet) self.protocol.fillAcquisition(partSet.getAcquisition()) partSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get()) self.protocol._defineOutputs(outputParticles=partSet)