def getParticles(self, protImport, classid):
dbPartSet = protImport._getPath("particles_class-%d.sqlite" % classid)
class3D = protImport.outputClasses[classid]
if os.path.exists(dbPartSet):
os.remove(dbPartSet)
partSet = SetOfParticles(filename=dbPartSet)
partSet.copyInfo(class3D)
for part in class3D:
partSet.append(part)
partSet.write()
partSet.close()
protImportCls1 = self.newProtocol(
ProtImportParticles,
objLabel='particles class-%d' % classid,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=dbPartSet,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImportCls1)
self.assertIsNotNone(protImportCls1.outputParticles.getFileName(),
"There was a problem with the import")
return protImportCls1
def _createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
if doCtf:
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(emobj.Transform(a))
self.partSet.append(p)
self.partSet.write()
def _getIterData(self, it):
data_sqlite = self._getFileName('data_scipion', iter=it)
if not os.path.exists(data_sqlite):
iterImgSet = SetOfParticles(filename=data_sqlite)
iterImgSet.copyInfo(self._getInputParticles())
self._fillDataFromIter(iterImgSet, it)
iterImgSet.write()
iterImgSet.close()
return data_sqlite
class TestSets(pwtests.BaseTest):
"""Run different tests related to the editor set protocol."""
@classmethod
def setUpClass(cls):
pwtests.setupTestProject(cls)
def _createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
if doCtf:
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(emobj.Transform(a))
self.partSet.append(p)
self.partSet.write()
def importData(self, baseFn, objLabel, protType, importFrom):
prot = self.newProtocol(protType,
objLabel=objLabel,
filesPath=baseFn,
maskPath=baseFn,
sqliteFile=baseFn,
haveDataBeenPhaseFlipped=False,
magnification=10000,
samplingRate=samplingRate,
importFrom=importFrom
)
self.launchProtocol(prot)
return prot
def testOperation(self):
"""Make a trivial operation. Increment defocusU by one"""
setPartSqliteName = self.proj.getTmpPath("particles_operate.sqlite")
setPartName = self.proj.getTmpPath('particles.stk')
self._createSetOfParticles(setPartSqliteName, setPartName,
doCtf=True)
protImportProj = self.importData(setPartSqliteName,
"import projection\n operation",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
#launch operate set protocol
protSetEditor = self.newProtocol(ProtSetEditor,
objLabel="operate")
protSetEditor.inputSet.set(protImportProj)
protSetEditor.inputSet.setExtended("outputParticles")
protSetEditor.operation.set(ProtSetEditor.CHOICE_FORMULA)
protSetEditor.formula.set('item._ctfModel._defocusU.set(item._ctfModel._defocusU.get() + 1. )')
self.launchProtocol(protSetEditor)
for item1, item2 in zip(protSetEditor.outputParticles, protImportProj.outputParticles):
self.assertAlmostEqual(item1._ctfModel._defocusU.get() ,
item2._ctfModel._defocusU.get() + 1)
def testRotation(self):
"""Rotate projections alignments so in the reconstruction
vecor (0,0,1) y rotated to (1,0,0)"""
setPartSqliteName = self.proj.getTmpPath("particles_rot_vec.sqlite")
setPartName = self.proj.getTmpPath('particles.stk')
self._createSetOfParticles(setPartSqliteName, setPartName,
doCtf=True)
protImportProj = self.importData(setPartSqliteName,
"import projection\n rot vector",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
#launch operate set protocol
protSetEditor = self.newProtocol(ProtSetEditor, objLabel="rotate")
protSetEditor.inputSet.set(protImportProj)
protSetEditor.inputSet.setExtended("outputParticles")
protSetEditor.operation.set(ProtSetEditor.CHOICE_ROTATE_VECTOR)
protSetEditor.xs.set(0.)
protSetEditor.ys.set(0.)
protSetEditor.zs.set(1.)
protSetEditor.xt.set(1.)
protSetEditor.yt.set(0.)
protSetEditor.zt.set(0.)
self.launchProtocol(protSetEditor)
for controlPartSet, outPartSet in zip( tOut,
protSetEditor.outputParticles, ):
self.assertTrue(np.allclose(controlPartSet ,
outPartSet.getTransform().getMatrix()))
def testRotationIco(self):
"""Rotate projections alignments between icosahedral
symmetries"""
setPartSqliteName = self.proj.getTmpPath("particles_rot_ico.sqlite")
setPartName = self.proj.getTmpPath('particles.stk')
self._createSetOfParticles(setPartSqliteName, setPartName,
doCtf=True)
protImportProj = self.importData(setPartSqliteName,
"import projection\n ico sym",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
#launch operate set protocol
protSetEditor = self.newProtocol(ProtSetEditor, objLabel="rotate")
protSetEditor.inputSet.set(protImportProj)
protSetEditor.inputSet.setExtended("outputParticles")
protSetEditor.operation.set(ProtSetEditor.CHOICE_ROTATE_ICOSAHEDRAL)
protSetEditor.originSymmetryGroup.set(SYM_I222 - SYM_I222)
protSetEditor.targetSymmetryGroup.set(SYM_I222r - SYM_I222)
self.launchProtocol(protSetEditor)
for controlPartSet, outPartSet in zip(tIcoOut,
protSetEditor.outputParticles, ):
self.assertTrue(np.allclose(controlPartSet ,
outPartSet.getTransform().getMatrix()))
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
# create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i % 3)
img.setClassId(i % 5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
# now import the dataset
prot1 = self.newProtocol(
emprot.ProtImportParticles,
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception(
'Import of images: %s, failed. outputParticles is None.' %
inFileNameMetadata)
protSplitSet = self.newProtocol(emprot.ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [
protSplitSet.outputParticles01, protSplitSet.outputParticles02
]
outputSet = SetOfParticles(
filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print("Items differ:")
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
# create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
# Start ids 4
img1 = Particle(objId=4)
img2 = Particle(objId=4)
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
# Increment Id
img1.setObjId(img1.getObjId() + 1)
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
# Increment Id
img2.setObjId(img2.getObjId() + 1)
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
# import them
protImport1 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set1',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set2',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
# create merge protocol
p_union = self.newProtocol(emprot.ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
# Assert ids
self.assertEqual(counter + 5, img.getObjId(),
"Object id's not kept.")
counter += 1
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
# Create two sets of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
# import them
protImport1 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set1',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set2',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
# create merge protocol
p_union = self.newProtocol(emprot.ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
# assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
counter += 1