def readSetOfParticles(lstFile, partSet, copyOrLink, direc):
for index, fn in iterLstFile(lstFile):
item = Particle()
# set full path to particles stack file
abspath = os.path.abspath(lstFile)
fn = abspath.replace('sets/%s' % os.path.basename(lstFile), '') + fn
newFn = pwutils.join(direc, os.path.basename(fn))
if not pwutils.exists(newFn):
copyOrLink(fn, newFn)
item.setLocation(index, newFn)
partSet.append(item)
def createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
#defocus=15000 + 5000* random.random()
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]#+500.
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(Transform(a))
self.partSet.append(p)
self.partSet.write()
def _fillClasses(self, outputClasses):
""" Create the SetOfClasses2D """
inputSet = self.inputClasses.get().getImages()
myRep = md.MetaData('classes@' +
self._getExtraPath('final_classes.xmd'))
for row in md.iterRows(myRep):
fn = row.getValue(md.MDL_IMAGE)
rep = Particle(fn)
repId = row.getObjId()
newClass = Class2D(objId=repId)
newClass.setAlignment2D()
newClass.copyInfo(inputSet)
newClass.setAcquisition(inputSet.getAcquisition())
newClass.setRepresentative(rep)
outputClasses.append(newClass)
i = 1
mdBlocks = md.getBlocksInMetaDataFile(
self._getExtraPath('final_classes.xmd'))
for block in mdBlocks:
if block.startswith('class00'):
mdClass = md.MetaData(block + "@" +
self._getExtraPath('final_classes.xmd'))
imgClassId = i
newClass = outputClasses[imgClassId]
newClass.enableAppend()
for row in md.iterRows(mdClass):
part = rowToParticle(row)
newClass.append(part)
i += 1
newClass.setAlignment2D()
outputClasses.update(newClass)
def test_mrcsLink(self):
""" In this case just a link with .mrcs extension
should be created
"""
stackFile = self.dsEmx.getFile('particles/particles.mrc')
partSet = SetOfParticles(filename=':memory:')
for i in range(1, 10):
particle = Particle()
particle.setLocation(i, stackFile)
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convertBinaryFiles(partSet, outputDir)
print filesDict
def test_mrcsLink(self):
""" In this case just a link with .mrcs extension
should be created
"""
stackFile = self.dsEmx.getFile('particles/particles.mrc')
partSet = SetOfParticles(filename=':memory:')
for i in range(1, 10):
particle = Particle()
particle.setLocation(i, stackFile)
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convertBinaryFiles(partSet, outputDir)
print filesDict
def test_particlesWithPhaseShiftToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite"))
n = 10
fn = self.getFile('particles_binary')
ctfs = [CTFModel(defocusU=10000, defocusV=15000,
defocusAngle=15, phaseShift=90),
CTFModel(defocusU=20000, defocusV=25000,
defocusAngle=25, phaseShift=60)
]
acquisition = Acquisition(magnification=60000, voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles_ph_sh.star')
fnStk = self.getOutputPath('particles.stk')
print (">>> Writing to file: %s" % fnStar)
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
def test_hdfToStk(self):
""" In this case the hdf stack files should be converted
to .stk spider files for Relion.
"""
stackFiles = [
'BPV_1386_ptcls.hdf', 'BPV_1387_ptcls.hdf', 'BPV_1388_ptcls.hdf'
]
partSet = SetOfParticles(filename=':memory:')
for fn in stackFiles:
particle = Particle()
particle.setLocation(1, self.ds.getFile('particles/%s' % fn))
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convertBinaryFiles(partSet, outputDir)
partSet.close()
print filesDict
def test_hdfToStk(self):
""" In this case the hdf stack files should be converted
to .stk spider files for Relion.
"""
stackFiles = ['BPV_1386_ptcls.hdf',
'BPV_1387_ptcls.hdf',
'BPV_1388_ptcls.hdf']
partSet = SetOfParticles(filename=':memory:')
for fn in stackFiles:
particle = Particle()
particle.setLocation(1, self.ds.getFile('particles/%s' % fn))
partSet.append(particle)
outputDir = self.getOutputPath()
filesDict = convertBinaryFiles(partSet, outputDir)
partSet.close()
print filesDict
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
p = Particle()
for mic in micList:
# We need to make this dict because there is no ID in the .xmd file
coordDict = {}
for coord in self.coordDict[mic.getObjId()]:
pos = self._getPos(coord)
if pos in coordDict:
print(
"WARNING: Ignoring duplicated coordinate: %s, id=%s" %
(coord.getObjId(), pos))
coordDict[pos] = coord
added = set() # Keep track of added coords to avoid duplicates
for row in md.iterRows(self._getMicXmd(mic)):
pos = (row.getValue(md.MDL_XCOOR), row.getValue(md.MDL_YCOOR))
coord = coordDict.get(pos, None)
if coord is not None and coord.getObjId() not in added:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
p.setLocation(xmippToLocation(row.getValue(md.MDL_IMAGE)))
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
# adding the variance and Gini coeff. value of the mic zone
setXmippAttributes(p, row, md.MDL_SCORE_BY_VAR)
setXmippAttributes(p, row, md.MDL_SCORE_BY_GINI)
if row.containsLabel(md.MDL_ZSCORE_DEEPLEARNING1):
setXmippAttributes(p, row, md.MDL_ZSCORE_DEEPLEARNING1)
# disabled particles (in metadata) should not add to the
# final set
if row.getValue(md.MDL_ENABLED) > 0:
outputParts.append(p)
added.add(coord.getObjId())
# Release the list of coordinates for this micrograph since it
# will not be longer needed
del self.coordDict[mic.getObjId()]
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i % 3)
img.setClassId(i % 5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(
ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception(
'Import of images: %s, failed. outputParticles is None.' %
inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [
protSplitSet.outputParticles01, protSplitSet.outputParticles02
]
outputSet = SetOfParticles(
filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
counter += 1
def _particlesFromEmx(protocol,
emxData,
emxFile,
outputDir,
acquisition,
samplingRate,
copyOrLink,
alignType=ALIGN_NONE):
""" Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
Add CTF information to the particles if present.
"""
emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
partDir = dirname(emxFile)
micSet = getattr(protocol, 'outputMicrographs', None)
if emxParticle is not None:
# Check if there are particles or coordinates
fn = emxParticle.get(emxlib.FILENAME)
if exists(join(
partDir,
fn)): # if the particles has binary data, means particles case
partSet = protocol._createSetOfParticles()
partSet.setAcquisition(
acquisition) # this adquisition is per project
#we need one per particle
part = Particle()
if _hasCtfLabels(emxParticle) or _hasCtfLabels(
emxParticle.getMicrograph()):
part.setCTF(CTFModel())
partSet.setHasCTF(True)
if emxParticle.has('centerCoord__X'):
part.setCoordinate(Coordinate())
#if emxParticle.has('transformationMatrix__t11'):
# _particleFromEmx(emxParticle, part)
# #partSet.setAlignment3D()
# partSet.setAlignment(alignType)
#else:
# partSet.setAlignment(alignType)
partSet.setAlignment(alignType)
if not samplingRate:
samplingRate = part.getSamplingRate()
partSet.setSamplingRate(samplingRate)
partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
particles = True
else: # if not binary data, the coordinate case
if micSet is None:
raise Exception(
'Could not import Coordinates from EMX, micrographs not imported.'
)
partSet = protocol._createSetOfCoordinates(micSet)
part = Coordinate()
particles = False
copiedFiles = {} # copied or linked
for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
if particles:
_particleFromEmx(emxParticle, part)
i, fn = part.getLocation()
partFn = join(partDir, fn)
newFn = join(outputDir, basename(partFn))
newLoc = (i, newFn)
if not partFn in copiedFiles:
copyOrLink(partFn, newFn)
copiedFiles[partFn] = newFn
part.setLocation(newLoc)
if partSet.hasAlignment():
transform = Transform()
_transformFromEmx(emxParticle, part, transform, alignType)
part.setTransform(transform)
else:
_coordinateFromEmx(emxParticle, part)
partSet.append(part)
part.cleanObjId()
if particles:
protocol._defineOutputs(outputParticles=partSet)
else:
protocol._defineOutputs(outputCoordinates=partSet)
if micSet is not None:
protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
give the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print ("\n")
print ("*" * 80)
print ("* Launching test: ", fileKey)
print ("*" * 80)
is2D = alignType == ALIGN_2D
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [numpy.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print ("Parset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.writeSetOfParticles(partSet, mdFn,"/tmp", alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print ("-" * 5)
print (img.getFileName(), img.getIndex())
print ('m1:\n', m1, relion.geometryFromMatrix(m1, False))
print ('m2:\n', m2, relion.geometryFromMatrix(m2, False))
# self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i%3)
img.setClassId(i%5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5
)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
give the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print "\n"
print "*" * 80
print "* Launching test: ", fileKey
print "*" * 80
is2D = alignType == ALIGN_2D
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')
if PRINT_FILES:
print "BINARY DATA: ", stackFn
print "SET1: ", partFn1
print " MD: ", mdFn
print "SET2: ", partFn2
print "OUTPUT: ", outputFn
print "GOLD: ", goldFn
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
partSet.setAcquisition(
Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [numpy.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print "Parset", partFn1
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.writeSetOfParticles(partSet,
mdFn,
"/tmp",
alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print "-" * 5
print img.getFileName(), img.getIndex()
print 'm1:\n', m1, relion.geometryFromMatrix(m1, False)
print 'm2:\n', m2, relion.geometryFromMatrix(m2, False)
# self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter +=1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1= Float(attrb1[counter])
img2._attrb2= Float(attrb2[counter])
imgSet2.append(img2)
counter +=1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter=0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter],img._attrb1,4)
self.assertAlmostEqual(attrb2[counter],img._attrb2,4)
counter += 1
def _particlesFromEmx(protocol
, emxData
, emxFile
, outputDir
, acquisition
, samplingRate
, copyOrLink
, alignType=ALIGN_NONE):
""" Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
Add CTF information to the particles if present.
"""
emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
partDir = dirname(emxFile)
micSet = getattr(protocol, 'outputMicrographs', None)
if emxParticle is not None:
# Check if there are particles or coordinates
fn = emxParticle.get(emxlib.FILENAME)
if exists(join(partDir, fn)): # if the particles has binary data, means particles case
partSet = protocol._createSetOfParticles()
partSet.setAcquisition(acquisition)# this adquisition is per project
#we need one per particle
part = Particle()
if _hasCtfLabels(emxParticle) or _hasCtfLabels(emxParticle.getMicrograph()):
part.setCTF(CTFModel())
partSet.setHasCTF(True)
if emxParticle.has('centerCoord__X'):
part.setCoordinate(Coordinate())
#if emxParticle.has('transformationMatrix__t11'):
# _particleFromEmx(emxParticle, part)
# #partSet.setAlignment3D()
# partSet.setAlignment(alignType)
#else:
# partSet.setAlignment(alignType)
partSet.setAlignment(alignType)
if not samplingRate:
samplingRate = part.getSamplingRate()
partSet.setSamplingRate(samplingRate)
partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
particles = True
else: # if not binary data, the coordinate case
if micSet is None:
raise Exception('Could not import Coordinates from EMX, micrographs not imported.')
partSet = protocol._createSetOfCoordinates(micSet)
part = Coordinate()
particles = False
copiedFiles = {} # copied or linked
for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
if particles:
_particleFromEmx(emxParticle, part)
i, fn = part.getLocation()
partFn = join(partDir, fn)
newFn = join(outputDir, basename(partFn))
newLoc = (i, newFn)
if not partFn in copiedFiles:
copyOrLink(partFn, newFn)
copiedFiles[partFn] = newFn
part.setLocation(newLoc)
if partSet.hasAlignment():
transform = Transform()
_transformFromEmx(emxParticle, part, transform, alignType)
part.setTransform(transform)
else:
_coordinateFromEmx(emxParticle, part)
partSet.append(part)
part.cleanObjId()
if particles:
protocol._defineOutputs(outputParticles=partSet)
else:
protocol._defineOutputs(outputCoordinates=partSet)
if micSet is not None:
protocol._defineSourceRelation(protocol.outputMicrographs,
partSet)
def _checkNewItems(self, objSet):
""" Check for already computed micrograph/movie and
update the output set. """
objDict = {}
newObj = False
for obj in objSet:
objDict[obj.getFileName()] = True
if objDict:
if objSet.getSize():
objSet.enableAppend()
objSet.loadAllProperties()
else:
objSet.setStreamState(objSet.STREAM_OPEN)
acquisition = Acquisition()
if self.setof == SET_OF_MICROGRAPHS:
acquisition.setMagnification(
self.inputMics.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMics.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(self.inputMics.get(
).getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(self.inputMics.get(
).getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(self.inputMics.get().getSamplingRate())
elif self.setof == SET_OF_MOVIES:
acquisition.setMagnification(
self.inputMovies.get().getAcquisition().getMagnification())
acquisition.setVoltage(
self.inputMovies.get().getAcquisition().getVoltage())
acquisition.setSphericalAberration(self.inputMovies.get(
).getAcquisition().getSphericalAberration())
acquisition.setAmplitudeContrast(self.inputMovies.get(
).getAcquisition().getAmplitudeContrast())
objSet.setAcquisition(acquisition)
objSet.setSamplingRate(
self.inputMovies.get().getSamplingRate())
else:
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
objSet.setAcquisition(acquisition)
if self.setof == SET_OF_PARTICLES:
objSet.setSamplingRate(
self.inputParticles.get().getSamplingRate())
else:
objSet.setSamplingRate(self.samplingRate.get())
if self.setof == SET_OF_MOVIES:
obj = Movie()
elif self.setof == SET_OF_MICROGRAPHS:
obj = Micrograph()
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
obj = Micrograph()
elif self.setof == SET_OF_PARTICLES:
obj = Particle()
else:
raise Exception('Unknown data type')
for k, v in self.dictObj.iteritems():
if (k not in objDict):
self.counter += 1
obj.setFileName(k)
if self.setof != SET_OF_PARTICLES:
obj.setMicName(basename(k))
obj.setObjId(self.counter)
objSet.append(obj)
newObj = True
return objSet, newObj # why a dictionary, a boolean may be enough
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter +=1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'),
"join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
if hasattr(img, '_attrb3'):
self.assertTrue(False, "join should not have attrb3")
if not hasattr(img, '_attrb2'):
self.assertTrue(False, "join should have attrb2")
counter += 1
def aaatest_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles.star')
fnStk = self.getOutputPath('particles.stk')
print ">>> Writing to file: %s" % fnStar
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))