def aaatest_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles.star')
fnStk = self.getOutputPath('particles.stk')
print ">>> Writing to file: %s" % fnStar
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def test_particlesWithPhaseShiftToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(filename=self.getOutputPath("particles_ph_sh.sqlite"))
n = 10
fn = self.getFile('particles_binary')
ctfs = [CTFModel(defocusU=10000, defocusV=15000,
defocusAngle=15, phaseShift=90),
CTFModel(defocusU=20000, defocusV=25000,
defocusAngle=25, phaseShift=60)
]
acquisition = Acquisition(magnification=60000, voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles_ph_sh.star')
fnStk = self.getOutputPath('particles.stk')
print (">>> Writing to file: %s" % fnStar)
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
self.assertTrue(mdAll.containsLabel(md.RLN_CTF_PHASESHIFT))
class TestSubProj(BaseTest):
@classmethod
def setUpClass(cls):
setupTestProject(cls)
def createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
#defocus=15000 + 5000* random.random()
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]#+500.
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(Transform(a))
self.partSet.append(p)
self.partSet.write()
def createProjection(self, proj, num, baseName):
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
#img.initRandom(0., 1., xmipp.XMIPP_RND_GAUSSIAN)
img.initConstant(0.)
for coor in proj:
value = img.getPixel(coor[0], coor[1], coor[2], coor[3])
img.setPixel(coor[0], coor[1], coor[2], coor[3], coor[4]+value) # coor4 is the pixel value
img.write("%d@"%num + baseName)
def createVol(self, volume):
vol = xmipp.Image()
vol.setDataType(xmipp.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
#vol.initRandom(0., .5, xmipp.XMIPP_RND_UNIFORM)
vol.initConstant(0.)
for coor in volume:
vol.setPixel(coor[0], coor[1], coor[2], coor[3], coor[4]) # coor4 is the pixel value
vol.write(self.volBaseFn)
def createMask(self, _maskName):
vol = xmipp.Image()
vol.setDataType(xmipp.DT_FLOAT)
vol.resize(projSize, projSize, projSize)
vol.initConstant(0.0)#ROB: not sure this is needed
halfDim = int(projSize/2)
maskRadius2 = maskRadius * maskRadius
for i in range(-halfDim,halfDim):
for j in range(-halfDim,halfDim):
for k in range(-halfDim,halfDim):
if(i*i+j*j+k*k) < maskRadius2:
vol.setPixel(0, k+halfDim, i+halfDim, j+halfDim, 1.) # coor4 is the pixel value
vol.write(_maskName)
def applyCTF(self, setPartMd):
writeSetOfParticles(self.partSet,setPartMd)
md1 = xmipp.MetaData()
md1.setColumnFormat(False)
idctf = md1.addObject()
_acquisition = self.partSet.getAcquisition()
for part in self.partSet:
baseFnCtf = self.proj.getTmpPath("kk")#self._getTmpPath("ctf_%d.param"%mic)
md1.setValue(xmipp.MDL_CTF_SAMPLING_RATE, samplingRate, idctf)
md1.setValue(xmipp.MDL_CTF_VOLTAGE, 200., idctf);
ctf = part.getCTF()
udefocus = ctf.getDefocusU()
vdefocus = ctf.getDefocusV()
angle = ctf.getDefocusAngle()
md1.setValue(xmipp.MDL_CTF_DEFOCUSU, udefocus, idctf);
md1.setValue(xmipp.MDL_CTF_DEFOCUSV, vdefocus, idctf);
md1.setValue(xmipp.MDL_CTF_DEFOCUS_ANGLE, 180.0 * random.random(), idctf);
md1.setValue(xmipp.MDL_CTF_CS, 2., idctf);
md1.setValue(xmipp.MDL_CTF_Q0, 0.07, idctf);
md1.setValue(xmipp.MDL_CTF_K, 1., idctf);
md1.write(baseFnCtf)
##writeSetOfParticles(self.partSet, setPartMd)
#apply ctf
args = " -i %s"%setPartMd
args += " -o %s"%self.proj.getTmpPath(setPartCtfName)
args += " -f ctf %s"%baseFnCtf
args += " --sampling %f"%samplingRate
runXmippProgram("xmipp_transform_filter", args)
args = " -i %s"%setPartMd
args += " -o %s"%self.proj.getTmpPath(setPartCtfPosName)
args += " -f ctfpos %s"%baseFnCtf
args += " --sampling %f"%samplingRate
runXmippProgram("xmipp_transform_filter", args)
def importData(self, baseFn, objLabel, protType, importFrom):
prot = self.newProtocol(protType,
objLabel=objLabel,
filesPath=baseFn,
maskPath=baseFn,
sqliteFile=baseFn,
haveDataBeenPhaseFlipped=False,
magnification=10000,
samplingRate=samplingRate,
importFrom=importFrom
)
self.launchProtocol(prot)
return prot
def test_pattern(self):
#1) create fake protocol so I have a place to save data
#prot = self.launchFakeProtocol()
#output stack
self.setPartName = self.proj.getTmpPath(setPartName)
self.setPartSqliteName = self.proj.getTmpPath(setPartSqliteName)
self.setPartSqliteCtfName = self.proj.getTmpPath(setPartSqliteCtfName)
self.setPartSqliteCTfPosName = self.proj.getTmpPath(setPartSqliteCTfPosName)
self.kksqlite = self.proj.getTmpPath("kk.sqlite")
self.setPartMd = self.proj.getTmpPath(setPartNameMd)
self.volBaseFn = self.proj.getTmpPath(volName)
self.maskName = self.proj.getTmpPath(maskName)
#2) create projections and sets of particles
self.createProjection(proj1, 1, self.setPartName)
self.createProjection(proj2, 2, self.setPartName)
self.createProjection(proj3, 3, self.setPartName)
self.createSetOfParticles(self.setPartSqliteCTfPosName, self.proj.getTmpPath(setPartCtfPosName), True)
self.createSetOfParticles(self.setPartSqliteCtfName, self.proj.getTmpPath(setPartCtfName), True)
self.createSetOfParticles(self.setPartSqliteName, self.setPartName, False)
#create auxiliary setofparticles
self.createSetOfParticles(self.kksqlite, self.setPartName, True)
#4) apply CTF
self.applyCTF(self.setPartMd)
#5) create volume
self.createVol(vol1)
#6) create mask
self.createMask(self.maskName)
#import three projection datasets, volume and mask
protPlainProj = self.importData(self.setPartSqliteName, "plain projection",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
protCTFProj = self.importData(self.setPartSqliteCtfName,"ctf projection",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
protCTFposProj = self.importData(self.setPartSqliteCTfPosName, "pos ctf projection",
ProtImportParticles,
ProtImportParticles.IMPORT_FROM_SCIPION)
_protImportVol = self.importData(os.path.abspath(self.proj.getTmpPath(volName)), "3D reference",
ProtImportVolumes,
ProtImportParticles.IMPORT_FROM_FILES)
_protImportMask = self.importData(self.proj.getTmpPath(maskName), "3D mask",
ProtImportMask,
ProtImportParticles.IMPORT_FROM_FILES)
mask = VolumeMask()
mask.setFileName(self.proj.getTmpPath(maskName))
mask.setSamplingRate(samplingRate)
#launch substract protocol <<<<<<<<<<<<<<<<<<<<<<<<<<
protSubtract = self.newProtocol(XmippProtSubtractProjection)
protSubtract.inputParticles.set(protPlainProj.outputParticles)
protSubtract.inputVolume.set(_protImportVol.outputVolume)
protSubtract.refMask.set(_protImportMask.outputMask)
protSubtract.projType.set(XmippProtSubtractProjection.CORRECT_NONE)
self.launchProtocol(protSubtract)
protSubtractCTF = self.newProtocol(XmippProtSubtractProjection)
protSubtractCTF.inputParticles.set(protCTFProj.outputParticles)
protSubtractCTF.inputVolume.set(_protImportVol.outputVolume)
protSubtractCTF.refMask.set(_protImportMask.outputMask)
protSubtractCTF.projType.set(XmippProtSubtractProjection.CORRECT_FULL_CTF)
self.launchProtocol(protSubtractCTF)
protSubtractCTFpos = self.newProtocol(XmippProtSubtractProjection)
protSubtractCTFpos.inputParticles.set(protCTFposProj.outputParticles)
protSubtractCTFpos.inputVolume.set(_protImportVol.outputVolume)
protSubtractCTFpos.refMask.set(_protImportMask.outputMask)
protSubtractCTFpos.projType.set(XmippProtSubtractProjection.CORRECT_PHASE_FLIP)
self.launchProtocol(protSubtractCTFpos)
protSubtractCTFRelion = self.newProtocol(ProtRelionSubtract)
protSubtractCTFRelion.inputParticles.set(protCTFProj.outputParticles)
protSubtractCTFRelion.inputVolume.set(_protImportVol.outputVolume)
self.launchProtocol(protSubtractCTFRelion)
self.assertIsNotNone(protSubtract.outputParticles, "There was a problem with subtract projection")
self.assertTrue(True)
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i % 3)
img.setClassId(i % 5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(
ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception(
'Import of images: %s, failed. outputParticles is None.' %
inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [
protSplitSet.outputParticles01, protSplitSet.outputParticles02
]
outputSet = SetOfParticles(
filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
give the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print "\n"
print "*" * 80
print "* Launching test: ", fileKey
print "*" * 80
is2D = alignType == ALIGN_2D
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')
if PRINT_FILES:
print "BINARY DATA: ", stackFn
print "SET1: ", partFn1
print " MD: ", mdFn
print "SET2: ", partFn2
print "OUTPUT: ", outputFn
print "GOLD: ", goldFn
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
partSet.setAcquisition(
Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [numpy.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print "Parset", partFn1
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.writeSetOfParticles(partSet,
mdFn,
"/tmp",
alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print "-" * 5
print img.getFileName(), img.getIndex()
print 'm1:\n', m1, relion.geometryFromMatrix(m1, False)
print 'm2:\n', m2, relion.geometryFromMatrix(m2, False)
# self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
def testOrderBy(self):
""" create set of particles and orderby a given attribute
"""
# This function was written by Roberto. It does things
# differently, so let's keep it for reference.
#create set of particles
inFileNameMetadata = self.proj.getTmpPath('particlesOrderBy.sqlite')
inFileNameData = self.proj.getTmpPath('particlesOrderBy.stk')
imgSet = SetOfParticles(filename=inFileNameMetadata)
imgSet.setSamplingRate(1.5)
acq = Acquisition()
acq.setAmplitudeContrast(0.1)
acq.setMagnification(10000)
acq.setVoltage(200)
acq.setSphericalAberration(2.0)
imgSet.setAcquisition(acq)
img = Particle()
for i in range(1, 10):
img.setLocation(i, inFileNameData)
img.setMicId(i%3)
img.setClassId(i%5)
imgSet.append(img)
img.cleanObjId()
imgSet.write()
#now import the dataset
prot1 = self.newProtocol(ProtImportParticles,
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata,
magnification=10000,
samplingRate=1.5
)
prot1.setObjLabel('from sqlite (test-sets)')
self.launchProtocol(prot1)
if prot1.outputParticles is None:
raise Exception('Import of images: %s, failed. outputParticles is None.' % inFileNameMetadata)
protSplitSet = self.newProtocol(ProtSplitSet,
inputSet=prot1.outputParticles,
numberOfSets=2,
randomize=True)
self.launchProtocol(protSplitSet)
inputSets = [protSplitSet.outputParticles01,protSplitSet.outputParticles02]
outputSet = SetOfParticles(filename=self.proj.getTmpPath('gold.sqlite'))
for itemSet in inputSets:
for obj in itemSet:
outputSet.append(obj)
for item1, item2 in izip(imgSet, outputSet):
if not item1.equalAttributes(item2):
print "Items differ:"
prettyDict(item1.getObjDict())
prettyDict(item2.getObjDict())
self.assertTrue(item1.equalAttributes(item2), )
def launchTest(self, fileKey, mList, alignType=None, **kwargs):
""" Helper function to launch similar alignment tests
give the EMX transformation matrix.
Params:
fileKey: the file where to grab the input stack images.
mList: the matrix list of transformations
(should be the same length of the stack of images)
"""
print ("\n")
print ("*" * 80)
print ("* Launching test: ", fileKey)
print ("*" * 80)
is2D = alignType == ALIGN_2D
stackFn = self.dataset.getFile(fileKey)
partFn1 = self.getOutputPath(fileKey + "_particles1.sqlite")
mdFn = self.getOutputPath(fileKey + "_particles.star")
partFn2 = self.getOutputPath(fileKey + "_particles2.sqlite")
if self.IS_ALIGNMENT:
outputFn = self.getOutputPath(fileKey + "_output.mrcs")
outputFnRelion = self.getOutputPath(fileKey + "_output")
goldFn = self.dataset.getFile(fileKey + '_Gold_output_relion.mrcs')
else:
outputFn = self.getOutputPath(fileKey + "_output.vol")
goldFn = self.dataset.getFile(fileKey + '_Gold_output.vol')
if PRINT_FILES:
print("BINARY DATA: ", stackFn)
print("SET1: ", partFn1)
print(" MD: ", mdFn)
print("SET2: ", partFn2)
print("OUTPUT: ", outputFn)
print("GOLD: ", goldFn)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
partSet = SetOfParticles(filename=partFn1)
else:
partSet = SetOfVolumes(filename=partFn1)
partSet.setAlignment(alignType)
partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
# Populate the SetOfParticles with images
# taken from images.mrc file
# and setting the previous alignment parameters
aList = [numpy.array(m) for m in mList]
for i, a in enumerate(aList):
p = Particle()
p.setLocation(i + 1, stackFn)
p.setTransform(Transform(a))
partSet.append(p)
# Write out the .sqlite file and check that are correctly aligned
print ("Parset", partFn1)
partSet.printAll()
partSet.write()
# Convert to a Xmipp metadata and also check that the images are
# aligned correctly
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.writeSetOfParticles(partSet, mdFn,"/tmp", alignType=alignType)
partSet2 = SetOfParticles(filename=partFn2)
else:
relion.writeSetOfVolumes(partSet, mdFn, alignType=alignType)
partSet2 = SetOfVolumes(filename=partFn2)
# Let's create now another SetOfImages reading back the written
# Xmipp metadata and check one more time.
partSet2.copyInfo(partSet)
if alignType == ALIGN_2D or alignType == ALIGN_PROJ:
relion.readSetOfParticles(mdFn, partSet2, alignType=alignType)
else:
relion.readSetOfVolumes(mdFn, partSet2, alignType=alignType)
partSet2.write()
if PRINT_MATRIX:
for i, img in enumerate(partSet2):
m1 = aList[i]
m2 = img.getTransform().getMatrix()
print ("-" * 5)
print (img.getFileName(), img.getIndex())
print ('m1:\n', m1, relion.geometryFromMatrix(m1, False))
print ('m2:\n', m2, relion.geometryFromMatrix(m2, False))
# self.assertTrue(numpy.allclose(m1, m2, rtol=1e-2))
# Launch apply transformation and check result images
runRelionProgram(self.CMD % locals())
if SHOW_IMAGES:
runRelionProgram('scipion show %(outputFn)s' % locals())
if os.path.exists(goldFn):
self.assertTrue(
ImageHandler().compareData(goldFn, outputFn, tolerance=0.001),
"Different data files:\n>%s\n<%s" % (goldFn, outputFn))
