def _createSubSetFromParticlesTiltPair(self, particlesTiltPair):
""" Create a subset of Particles Tilt Pair. """
output = ParticlesTiltPair(
filename=self._getPath('particles_pairs.sqlite'))
inputU = particlesTiltPair.getUntilted()
inputT = particlesTiltPair.getTilted()
outputU = emobj.SetOfParticles(
filename=self._getPath('particles_untilted.sqlite'))
outputT = emobj.SetOfParticles(
filename=self._getPath('particles_tilted.sqlite'))
outputU.copyInfo(inputU)
outputT.copyInfo(inputT)
modifiedSet = ParticlesTiltPair(filename=self._dbName,
prefix=self._dbPrefix)
# noinspection DuplicatedCode
for pair, u, t in izip(modifiedSet, inputU, inputT):
if pair.isEnabled():
output.append(pair)
outputU.append(u)
outputT.append(t)
# Register outputs
output.setUntilted(outputU)
output.setTilted(outputT)
# Link output to the same coordinates pairs than input
output.setCoordsPair(particlesTiltPair.getCoordsPair())
outputDict = {'outputParticlesTiltPair': output}
self._defineOutputs(**outputDict)
self._defineTransformRelation(particlesTiltPair, output)
return output
def _checkProcessedData(self):
if self.setof == SET_OF_MOVIES:
objSet = emobj.SetOfMovies(filename=self._getPath('movies.sqlite'))
elif self.setof == SET_OF_MICROGRAPHS:
objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
objSet = emobj.SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_PARTICLES:
objSet = emobj.SetOfParticles(filename=self._getPath('particles.sqlite'))
else:
raise Exception('Unknown data type')
newObjSet, newObj = self._checkNewItems(objSet)
if self.setof == SET_OF_MOVIES:
newObjSet.setFramesRange(self.inputMovies.get().getFramesRange())
# check if end ....
if self.setof != SET_OF_PARTICLES:
self.nDims = self.nDim.get()
endObjs = newObjSet.getSize() == self.nDims
if newObj:
if endObjs:
newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
self._updateOutput(newObjSet)
newObjSet.close()
def loadInputs(self):
micsFn = self.getInputMicrographs().getFileName()
micsSet = emobj.SetOfMicrographs(filename=micsFn)
micsSet.loadAllProperties()
availableMics = []
for mic in micsSet:
availableMics.append(mic.getObjId())
micsSetClosed = micsSet.isStreamClosed()
micsSet.close()
partsFn = self.getInputParticles().getFileName()
partsSet = emobj.SetOfParticles(filename=partsFn)
partsSet.loadAllProperties()
newParts = []
newMics = []
for item in partsSet:
micKey = item.getCoordinate().getMicId()
if micKey not in self.micsDone and micKey in availableMics:
newParts.append(item.getObjId())
if micKey not in self.micsDone:
newMics.append(micKey)
self.micsDone += newMics
self.inputSize = partsSet.getSize()
partSetClosed = partsSet.isStreamClosed()
partsSet.close()
return newParts, micsSetClosed and partSetClosed
def test_subsetsFromSelection(self):
""" From a sqlite file of a SetOfClasses2D, with some
classes and element disabled, we want to create a
subset of images and classes.
"""
classes2DSet = emobj.SetOfClasses2D(filename=self.selectionFn)
imgSet = emobj.SetOfParticles(filename=':memory:')
# We are going to iterate over the enabled items and create
# a new set of images
imgSet.appendFromClasses(classes2DSet)
# Since we have disabled two classes (6 images) and 1 images
# from class 1 and 2 (2 images), the final size of the set
# should be 68
sizes = [32, 36]
self.assertEqual(imgSet.getSize(), sum(sizes))
imgSet.clear() # Close db connection and clean data
# Now create a subset of classes and check the number
# of images per class
clsSet = emobj.SetOfClasses2D(filename=':memory:')
clsSet.appendFromClasses(classes2DSet)
for i, cls in enumerate(clsSet):
self.assertEqual(cls.getSize(), sizes[i])
clsSet.clear() # Close db connection and clean data
def test_getFiles(self):
# create setofImages
dbFn = self.getOutputPath('multistack_set.sqlite')
# dbFn = ':memory:'
n = 10
m = 3
img = emobj.Particle()
imgSet = emobj.SetOfParticles(filename=dbFn)
imgSet.setSamplingRate(1.0)
goldStacks = set()
for j in range(1, m + 1):
stackName = 'stack%02d.stk' % j
goldStacks.add(stackName)
for i in range(1, n + 1):
img.setLocation(i, stackName)
imgSet.append(img)
img.cleanObjId()
self.assertEquals(goldStacks, imgSet.getFiles())
imgSet.close()
# It should automatically call load
# before accessing items db
imgSet.getFirstItem()
def test_hugeSet(self):
""" Create a set of a big number of particles to measure
creation time with sqlite operations.
"""
# Allow what huge means to be defined with environment var
n = int(os.environ.get('SCIPION_TEST_HUGE', 10000))
print(">>>> Creating a set of %d particles." % n)
print(" (set SCIPION_TEST_HUGE environment var to other value)")
dbFn = self.getOutputPath('huge_set.sqlite')
# dbFn = ':memory:'
img = emobj.Particle()
imgSet = emobj.SetOfParticles(filename=dbFn)
imgSet.setSamplingRate(1.0)
for i in range(1, n + 1):
# Creating object inside the loop significantly
# decrease performance
# img = Particle()
img.setLocation(i, "images.stk")
imgSet.append(img)
img.cleanObjId()
# img.setObjId(None)
imgSet.write()
def test_orderBy(self):
# create setofProjections sorted
imgSet1 = emobj.SetOfParticles(filename=':memory:', prefix='set1')
imgSet2 = emobj.SetOfParticles(filename=':memory:', prefix='set2')
imgSet1.setSamplingRate(1.5)
imgSet2.setSamplingRate(1.5)
img = emobj.Particle()
for i in range(1, 10):
img.setLocation(i, 'mystack.stk')
img.setMicId(10 - i)
img.setClassId(i % 5)
imgSet1.append(img)
img.setMicId(i)
imgSet2.append(img)
img.cleanObjId()
# orderby
for item1, item2 in zip(
imgSet1.iterItems(orderBy='_micId', direction='ASC'),
imgSet2.iterItems(orderBy='_micId', direction='ASC')):
self.assertEquals(item1.getMicId(), item2.getMicId())
def test_readStack(self):
""" Read an stack of 29 particles from .hdf file.
Particles should be of 500x500 pixels.
"""
size = 29
xdim = 500
inStack = self.dataset.getFile('particles1')
outFn = self.getOutputPath('particles.sqlite')
imgSet = emobj.SetOfParticles(filename=outFn)
imgSet.setSamplingRate(1.0)
imgSet.readStack(inStack) # This should add 29 new items to the set
self.assertEquals(size, imgSet.getSize()) # Check same size
self.assertEquals(xdim, imgSet.getDim()[0]) # Check same dimensions
print("writing particles to: ", outFn)
imgSet.write()
imgSet2 = emobj.SetOfParticles(filename=':memory:')
imgSet2.copyInfo(imgSet)
self.assertAlmostEqual(imgSet.getSamplingRate(),
imgSet2.getSamplingRate())
def test_listAttribute(self):
"""
Test the use of copyItems function where the items have
a property that is a list.
"""
inFn = self.dataset.getFile('particles/particles_nma.sqlite')
outFn = self.getOutputPath('particles.sqlite')
inputSet = emobj.SetOfParticles(filename=inFn)
inputSet.setSamplingRate(1.0)
outputSet = emobj.SetOfParticles(filename=outFn)
outputSet.copyInfo(inputSet)
outputSet.copyItems(inputSet, updateItemCallback=self._updateItem)
# print("writing particles to: ", outFn)
outputSet.write()
outputSet.close()
# Read again the written set of particles
checkSet = emobj.SetOfParticles(filename=outFn)
# check that the first item (as the rest)
# have the added _list attribute with the correct values
particle = checkSet.getFirstItem()
self.assertTrue(particle.hasAttribute('_list'))
for v1, v2 in zip([1.0, 2.0], particle._list):
self.assertAlmostEqual(v1, float(v2))
# check that copied items have exactly
# the same attribes than the input set
# except for the _list
for i1, i2 in zip(inputSet, checkSet):
self.assertTrue(i2.equalAttributes(i1, ignore=['_list']))
def _getIterParticles(self, it, clean=False):
import pwem.objects as em
""" Return a classes .sqlite file for this iteration.
If the file doesn't exists, it will be created by
converting from this iteration data.star file.
"""
dataParticles = self._getFileName('particlesScipion', iter=it)
if clean:
cleanPath(dataParticles)
if not exists(dataParticles):
partSet = em.SetOfParticles(filename=dataParticles)
self._fillParticlesFromIter(partSet, it)
partSet.write()
partSet.close()
else:
partSet = em.SetOfParticles(filename=dataParticles)
imgSet = self.inputParticles.get()
partSet.copyInfo(imgSet)
partSet.setAlignmentProj()
return partSet
def loadAcquisitionInfo(self):
""" Return a dictionary with acquisition values and
the sampling rate information.
In the case of Scipion, these values will be read from the
'Properties' table of the particles.sqlite file.
"""
acq = OrderedDict()
inputSet = emobj.SetOfParticles(filename=self._sqliteFile)
def _get(key):
return inputSet.getProperty(key)
acq['samplingRate'] = _get('_samplingRate')
acq['voltage'] = _get('_acquisition._voltage')
acq['amplitudeContrast'] = _get('_acquisition._amplitudeContrast')
acq['sphericalAberration'] = float(_get('_acquisition._sphericalAberration'))
return acq
def importParticles(self):
""" Import particles from a metadata 'images.xmd' """
inputSet = emobj.SetOfParticles(filename=self._sqliteFile)
inputSet.loadProperty('_alignment', emcts.ALIGN_NONE)
inputSet.loadProperty('_hasCtf', False)
self._findImagesPath(inputSet)
partSet = self.protocol._createSetOfParticles()
partSet.copyInfo(inputSet)
partSet.setObjComment('Particles imported from Scipion sqlite file:\n%s'
% self._sqliteFile)
partSet.copyItems(inputSet, updateItemCallback=self._updateParticle)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form
self.protocol.setSamplingRate(partSet)
self.protocol.fillAcquisition(partSet.getAcquisition())
partSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get())
self.protocol._defineOutputs(outputParticles=partSet)
