def createSetOfParticles(self, setPartSqliteName, partFn,
doCtf=False):
# create a set of particles
self.partSet = SetOfParticles(filename=setPartSqliteName)
self.partSet.setAlignment(ALIGN_PROJ)
self.partSet.setAcquisition(Acquisition(voltage=300,
sphericalAberration=2,
amplitudeContrast=0.1,
magnification=60000))
self.partSet.setSamplingRate(samplingRate)
self.partSet.setHasCTF(True)
aList = [np.array(m) for m in mList]
#defocus=15000 + 5000* random.random()
for i, a in enumerate(aList):
p = Particle()
if doCtf:
defocusU = defocusList[i]#+500.
defocusV = defocusList[i]
ctf = CTFModel(defocusU=defocusU,
defocusV=defocusV,
defocusAngle=defocusAngle[i])
ctf.standardize()
p.setCTF(ctf)
p.setLocation(i + 1, partFn)
p.setTransform(Transform(a))
self.partSet.append(p)
self.partSet.write()
def importCTF(self, mic, fileName):
ctf = CTFModel()
ctf.setMicrograph(mic)
readCtfModel(ctf, fileName, ctf4=ctffindOutputVersion(fileName) == 4)
for suffix in ["_psd.mrc", ".mrc"]:
if os.path.exists(pwutils.removeExt(fileName) + suffix):
ctf.setPsdFile(pwutils.removeExt(fileName) + suffix)
return ctf
def _getCTFModel(self, defocusU, defocusV, defocusAngle, resol, psdFile):
ctf = CTFModel()
ctf.setStandardDefocus(defocusU, defocusV, defocusAngle)
ctf.setResolution(resol)
ctf.setPsdFile(psdFile)
return ctf
def importCTF(self, mic, fileName):
defocusU, defocusV, defocusAngle = parseCtffindOutput(fileName)
ctf = CTFModel()
ctf.copyObjId(mic)
ctf.setStandardDefocus(defocusU, defocusV, defocusAngle)
ctf.setMicrograph(mic)
ctf.setPsdFile(removeExt(fileName) + "_psd.mrc")
return ctf
def importCTF(self, mic, fileName):
ctf = CTFModel()
ctf.setMicrograph(mic)
readCtfModel(ctf, fileName, ctf4=ctffindOutputVersion(fileName) == 4)
# Try to find the given PSD file associated with the cttfind log file
# we handle special cases of .ctf extension and _ctffindX prefix for Relion runs
fnBase = pwutils.removeExt(fileName)
for suffix in ["_psd.mrc", ".mrc", ".ctf"]:
psdPrefixes = [fnBase, fnBase.replace("_ctffind3", ""), fnBase.replace("_ctffind4", "")]
for prefix in psdPrefixes:
psdFile = prefix + suffix
if os.path.exists(psdFile):
if psdFile.endswith(".ctf"):
psdFile += ":mrc"
ctf.setPsdFile(psdFile)
return ctf
def _micrographsFromEmx(protocol, emxData, emxFile, outputDir, acquisition,
samplingRate, copyOrLink):
""" Create the output SetOfMicrographs given an EMXData object.
If there is information of the CTF, also the SetOfCTF will
be registered as output of the protocol.
"""
emxMic = emxData.getFirstObject(emxlib.MICROGRAPH)
if emxMic is not None:
micSet = protocol._createSetOfMicrographs()
mic = Micrograph()
mic.setAcquisition(acquisition)
if _hasCtfLabels(emxMic):
mic.setCTF(CTFModel())
ctfSet = protocol._createSetOfCTF()
else:
ctfSet = None
_micrographFromEmx(emxMic, mic)
acq = mic.getAcquisition().clone()
micSet.setAcquisition(acq)
micSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
if not samplingRate:
samplingRate = mic.getSamplingRate()
micSet.setSamplingRate(samplingRate)
micDir = dirname(emxFile)
for emxMic in emxData.iterClasses(emxlib.MICROGRAPH):
_micrographFromEmx(emxMic, mic)
_, fn = mic.getLocation()
micFn = join(micDir, fn)
if copyOrLink is not None:
micBase = basename(micFn)
newFn = join(outputDir, micBase)
copyOrLink(micFn, newFn)
mic.setLocation(newFn)
else:
mic.setLocation(micFn)
_fillMicName(mic, fn)
micSet.append(mic)
emxMic._micId = mic.getObjId()
mic.cleanObjId()
if ctfSet is not None:
ctf = mic.getCTF().clone()
ctf.setMicrograph(mic)
#TODO I do not think next line is needed
#ctf.setMicFile(newFn)
ctfSet.append(ctf)
if emxMic is not None:
protocol._defineOutputs(outputMicrographs=micSet)
if ctfSet is not None:
protocol._defineOutputs(outputCTF=ctfSet)
ctfSet.setMicrographs(micSet)
protocol._defineCtfRelation(micSet, ctfSet)
def importCTF(self, mic, fileName):
ctf = CTFModel()
ctf.setMicrograph(mic)
readCtfModel(ctf, fileName, ctf4=False)
# Try to find the given PSD file associated with the cttfind log file
# we handle special cases of .ctf extension and _ctffindX prefix for Relion runs
fnBase = pwutils.removeExt(fileName)
for suffix in ['_psd.mrc', '.mrc', '.ctf']:
psdPrefixes = [fnBase,
fnBase.replace('_ctffind3', ''),
fnBase.replace('_gctf', '')]
for prefix in psdPrefixes:
psdFile = prefix + suffix
if os.path.exists(psdFile):
if psdFile.endswith('.ctf'):
psdFile += ':mrc'
ctf.setPsdFile(psdFile)
return ctf
def aaatest_particlesToStar(self):
""" Write a SetOfParticles to Relion star input file. """
imgSet = SetOfParticles(
filename=self.getOutputPath("particles.sqlite"))
n = 10
fn = self.particles
ctfs = [
CTFModel(defocusU=10000, defocusV=15000, defocusAngle=15),
CTFModel(defocusU=20000, defocusV=25000, defocusAngle=25)
]
acquisition = Acquisition(magnification=60000,
voltage=300,
sphericalAberration=2.,
amplitudeContrast=0.07)
imgSet.setAcquisition(acquisition)
coord = Coordinate()
coord.setMicId(1)
for i in range(n):
p = Particle()
p.setLocation(i + 1, fn)
ctf = ctfs[i % 2]
p.setCTF(ctf)
p.setAcquisition(acquisition)
p._xmipp_zScore = Float(i)
coord.setX(i * 10)
coord.setY(i * 10)
p.setCoordinate(coord)
imgSet.append(p)
fnStar = self.getOutputPath('particles.star')
fnStk = self.getOutputPath('particles.stk')
print ">>> Writing to file: %s" % fnStar
relion.writeSetOfParticles(imgSet, fnStar, fnStk)
mdAll = md.MetaData(fnStar)
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_X))
self.assertTrue(mdAll.containsLabel(md.RLN_IMAGE_COORD_Y))
self.assertFalse(mdAll.containsLabel(md.RLN_SELECT_PARTICLES_ZSCORE))
def importCtfStep(self, micsId, pattern):
""" Copy movies matching the filename pattern
Register other parameters.
"""
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
ctfFiles = self._getFilePaths(pattern)
ctfDict = {}
for fn in ctfFiles:
# Try to match the micrograph id from filename
# this is set by the user by using #### format in the pattern
match = self._idRegex.match(fn)
if match is None:
raise Exception("File '%s' doesn't match the pattern '%s'" %
(fn, self.pattern.get()))
ctfId = int(match.group(1))
ctfDict[ctfId] = fn
from pyworkflow.em.packages.grigoriefflab.convert import parseCtffindOutput
for mic in inputMics:
if mic.getObjId() in ctfDict:
defocusU, defocusV, defocusAngle = parseCtffindOutput(
ctfDict[mic.getObjId()])
else:
self.warning("CTF for micrograph id %d was not found." %
mic.getObjId())
defocusU, defocusV, defocusAngle = -999, -1, -999
# save the values of defocus for each micrograph in a list
ctf = CTFModel()
ctf.copyObjId(mic)
ctf.setStandardDefocus(defocusU, defocusV, defocusAngle)
ctf.setMicrograph(mic)
ctfSet.append(ctf)
self._defineOutputs(outputCTF=ctfSet)
self._defineCtfRelation(inputMics, ctfSet)
def importCtfStep(self, micsId, pattern):
""" Copy movies matching the filename pattern
Register other parameters.
"""
inputMics = self.inputMicrographs.get()
ctfSet = self._createSetOfCTF()
ctfSet.setMicrographs(inputMics)
ctfFiles = self._getFilePaths(pattern)
ctfDict = {}
for fn in ctfFiles:
# Try to match the micrograph id from filename
# this is set by the user by using #### format in the pattern
match = self._idRegex.match(fn)
if match is None:
raise Exception("File '%s' doesn't match the pattern '%s'" % (fn, self.pattern.get()))
ctfId = int(match.group(1))
ctfDict[ctfId] = fn
from pyworkflow.em.packages.grigoriefflab.convert import parseCtffindOutput
for mic in inputMics:
if mic.getObjId() in ctfDict:
defocusU, defocusV, defocusAngle = parseCtffindOutput(ctfDict[mic.getObjId()])
else:
self.warning("CTF for micrograph id %d was not found." % mic.getObjId())
defocusU, defocusV, defocusAngle = -999, -1, -999
# save the values of defocus for each micrograph in a list
ctf = CTFModel()
ctf.copyObjId(mic)
ctf.setStandardDefocus(defocusU, defocusV, defocusAngle)
ctf.setMicrograph(mic)
ctfSet.append(ctf)
self._defineOutputs(outputCTF=ctfSet)
self._defineCtfRelation(inputMics, ctfSet)
def importCTF(self, mic, fileName):
ctf = CTFModel()
ctf.setMicrograph(mic)
readCtfModel(ctf, fileName, ctf4=False)
# Try to find the given PSD file associated with the cttfind log file
# we handle special cases of .ctf extension and _ctffindX prefix for Relion runs
fnBase = pwutils.removeExt(fileName)
for suffix in ['_psd.mrc', '.mrc', '.ctf']:
psdPrefixes = [
fnBase,
fnBase.replace('_ctffind3', ''),
fnBase.replace('_gctf', '')
]
for prefix in psdPrefixes:
psdFile = prefix + suffix
if os.path.exists(psdFile):
if psdFile.endswith('.ctf'):
psdFile += ':mrc'
ctf.setPsdFile(psdFile)
return ctf
def _getCTFModel(self, defocusU, defocusV, defocusAngle, psdFile):
ctf = CTFModel()
ctf.setStandardDefocus(defocusU, defocusV, defocusAngle)
ctf.setPsdFile(psdFile)
return ctf
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def _particlesFromEmx(protocol,
emxData,
emxFile,
outputDir,
acquisition,
samplingRate,
copyOrLink,
alignType=ALIGN_NONE):
""" Create the output SetOfCoordinates or SetOfParticles given an EMXData object.
Add CTF information to the particles if present.
"""
emxParticle = emxData.getFirstObject(emxlib.PARTICLE)
partDir = dirname(emxFile)
micSet = getattr(protocol, 'outputMicrographs', None)
if emxParticle is not None:
# Check if there are particles or coordinates
fn = emxParticle.get(emxlib.FILENAME)
if exists(join(
partDir,
fn)): # if the particles has binary data, means particles case
partSet = protocol._createSetOfParticles()
partSet.setAcquisition(
acquisition) # this adquisition is per project
#we need one per particle
part = Particle()
if _hasCtfLabels(emxParticle) or _hasCtfLabels(
emxParticle.getMicrograph()):
part.setCTF(CTFModel())
partSet.setHasCTF(True)
if emxParticle.has('centerCoord__X'):
part.setCoordinate(Coordinate())
#if emxParticle.has('transformationMatrix__t11'):
# _particleFromEmx(emxParticle, part)
# #partSet.setAlignment3D()
# partSet.setAlignment(alignType)
#else:
# partSet.setAlignment(alignType)
partSet.setAlignment(alignType)
if not samplingRate:
samplingRate = part.getSamplingRate()
partSet.setSamplingRate(samplingRate)
partSet.setIsPhaseFlipped(protocol.haveDataBeenPhaseFlipped.get())
particles = True
else: # if not binary data, the coordinate case
if micSet is None:
raise Exception(
'Could not import Coordinates from EMX, micrographs not imported.'
)
partSet = protocol._createSetOfCoordinates(micSet)
part = Coordinate()
particles = False
copiedFiles = {} # copied or linked
for emxParticle in emxData.iterClasses(emxlib.PARTICLE):
if particles:
_particleFromEmx(emxParticle, part)
i, fn = part.getLocation()
partFn = join(partDir, fn)
newFn = join(outputDir, basename(partFn))
newLoc = (i, newFn)
if not partFn in copiedFiles:
copyOrLink(partFn, newFn)
copiedFiles[partFn] = newFn
part.setLocation(newLoc)
if partSet.hasAlignment():
transform = Transform()
_transformFromEmx(emxParticle, part, transform, alignType)
part.setTransform(transform)
else:
_coordinateFromEmx(emxParticle, part)
partSet.append(part)
part.cleanObjId()
if particles:
protocol._defineOutputs(outputParticles=partSet)
else:
protocol._defineOutputs(outputCoordinates=partSet)
if micSet is not None:
protocol._defineSourceRelation(protocol.outputMicrographs, partSet)
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter +=1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True
)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'),
"join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
counter += 1
def importCTF(self, mic, fileName):
ctf = CTFModel()
ctf.setMicrograph(mic)
readCTFModel(ctf, fileName)
return ctf