def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = Particle()
for mic in micList:
# We need to make this dict because there is no ID in the .xmd file
coordDict = {}
for coord in self.coordDict[mic.getObjId()]:
pos = self._getPos(coord)
if pos in coordDict:
print(
"WARNING: Ignoring duplicated coordinate: %s, id=%s" %
(coord.getObjId(), pos))
coordDict[pos] = coord
added = set() # Keep track of added coords to avoid duplicates
for row in md.iterRows(self._getMicXmd(mic)):
pos = (row.getValue(md.MDL_XCOOR), row.getValue(md.MDL_YCOOR))
coord = coordDict.get(pos, None)
if coord is not None and coord.getObjId() not in added:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
p.setLocation(xmippToLocation(row.getValue(md.MDL_IMAGE)))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
# adding the variance and Gini coeff. value of the mic zone
setXmippAttributes(p, row, md.MDL_SCORE_BY_VAR)
setXmippAttributes(p, row, md.MDL_SCORE_BY_GINI)
if row.containsLabel(md.MDL_ZSCORE_DEEPLEARNING1):
setXmippAttributes(p, row, md.MDL_ZSCORE_DEEPLEARNING1)
# disabled particles (in metadata) should not add to the
# final set
if row.getValue(md.MDL_ENABLED) > 0:
outputParts.append(p)
added.add(coord.getObjId())
# Release the list of coordinates for this micrograph since it
# will not be longer needed
del self.coordDict[mic.getObjId()]
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i % 3)
img.setClassId(i % 5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(
ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception(
'Import of images: %s, failed. outputParticles is None.' %
inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [
protSplitSet.outputParticles01, protSplitSet.outputParticles02
]
outputSet = SetOfParticles(
filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
counter += 1
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i%3)
img.setClassId(i%5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5
)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter +=1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'),
"join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter +=1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1= Float(attrb1[counter])
img2._attrb2= Float(attrb2[counter])
imgSet2.append(img2)
counter +=1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter=0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter],img._attrb1,4)
self.assertAlmostEqual(attrb2[counter],img._attrb2,4)
counter += 1
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
if hasattr(img, '_attrb3'):
self.assertTrue(False, "join should not have attrb3")
if not hasattr(img, '_attrb2'):
self.assertTrue(False, "join should have attrb2")
counter += 1
