def testCtfdiscrepancyWorkflow(self):
""" Import 3 EMX files with micrographs and defocus and compare them
"""
emxFn1 = self.dataset.getFile('emxMicrographCtf1')
emxFn2 = self.dataset.getFile('emxMicrographCtf2')
emxFn3 = self.dataset.getFile('emxMicrographCtf3')
protEmxImport1 = self.newProtocol(
ProtImportMicrographs,
importFrom=ProtImportMicrographs.IMPORT_FROM_EMX,
samplingRate=1,
emxFile=emxFn1)
protEmxImport2 = self.newProtocol(
ProtImportMicrographs,
importFrom=ProtImportMicrographs.IMPORT_FROM_EMX,
samplingRate=1,
emxFile=emxFn2)
protEmxImport3 = self.newProtocol(
ProtImportMicrographs,
importFrom=ProtImportMicrographs.IMPORT_FROM_EMX,
samplingRate=1,
emxFile=emxFn3)
pl = PointerList([
Pointer(value=protEmxImport1, extended='outputCTF'),
Pointer(value=protEmxImport2, extended='outputCTF'),
Pointer(value=protEmxImport3, extended='outputCTF')
])
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTFs.set(pl)
self.proj.saveProtocol(protEmxImport1)
self.proj.saveProtocol(protEmxImport2)
self.proj.saveProtocol(protEmxImport3)
self.proj.saveProtocol(protCtfDiscrepancy)
self.launchProtocol(protEmxImport1)
self.launchProtocol(protEmxImport2)
self.launchProtocol(protEmxImport3)
self.launchProtocol(protCtfDiscrepancy)
ctfsGold = SetOfCTF(filename=self.dataset.getFile('ctfsGold'))
ctfSetFn, ctfSetPairFn = protCtfDiscrepancy._getAnalyzeFiles()
ctfComputed = SetOfCTF(filename=ctfSetPairFn)
for ctf1, ctf2 in izip(ctfComputed, ctfsGold):
ctf1.getMicrograph().setFileName(
os.path.basename(ctf1.getMicrograph().getFileName()))
ctf2.getMicrograph().setFileName(
os.path.basename(ctf2.getMicrograph().getFileName()))
self.assertTrue(ctf1.equalAttributes(ctf2))
def _createMicsSubSetFromCTF(self, inputCTFs):
""" Create a subset of Micrographs and CTFs when analyzing the CTFs. """
outputMics = self._createSetOfMicrographs()
outputCtfs = self._createSetOfCTF()
setOfMics = inputCTFs.getMicrographs()
if setOfMics is None:
raise Exception('Could not create SetOfMicrographs subset from '
'this SetOfCTF, the micrographs were not set.')
outputMics.copyInfo(setOfMics)
modifiedSet = SetOfCTF(filename=self._dbName, prefix=self._dbPrefix)
count = 0
for ctf in modifiedSet:
if ctf.isEnabled():
mic = ctf.getMicrograph()
outputMics.append(mic)
outputCtfs.append(ctf)
count += 1
# Register outputs
outputCtfs.setMicrographs(outputMics)
self._defineOutputs(outputMicrographs=outputMics, outputCTF=outputCtfs)
self._defineTransformRelation(setOfMics, outputMics)
self._defineCtfRelation(outputMics, outputCtfs)
msg = 'From input %s of size %s created output '% (inputCTFs.getClassName(),
inputCTFs.getSize())
msg += 'SetOfMicrographs and SetOfCTF of size %d' % count
self.summaryVar.set(msg)
return outputMics, outputCtfs
def getMicCTF(self, mic):
""" Retrieve the CTF associated to this micrograph. """
# The first time this function is called, the set of micrographs
# will be loaded from the given sqlite file
if not hasattr(self, 'ctfSet'):
self.ctfSet = SetOfCTF(filename=self._sqliteFile)
return self.ctfSet[mic.getObjId()]
def checkOutputs(prot):
t0 = time.time()
while not (prot.isFinished() or prot.isFailed()):
# Time out 6 minutes, just in case
tdelta = time.time() - t0
if tdelta > 6 * 60:
break
time.sleep(10)
prot = self._updateProtocol(prot)
# Check if the protocol is still launched
if prot.isLaunched():
break
elif prot.isScheduled():
continue
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(
prot.outputCTF, "Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def checkOutputs(prot):
while not (prot.isFinished() or prot.isFailed()):
time.sleep(10)
prot = self.updateProtocol(prot)
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(
prot.outputCTF, "Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def _createSubSetOfCTF(self, inputCtf):
""" Create a subset of CTF and Micrographs analyzing the CTFs. """
setOfCtf = self._createSetOfCTF("_subset")
modifiedSet = SetOfCTF(filename=self._dbName, prefix=self._dbPrefix)
for ctf in modifiedSet:
if ctf.isEnabled():
setOfCtf.append(ctf)
# Register outputs
self._defineOutput(self.outputClassName.get(), setOfCtf)
self._defineTransformRelation(inputCtf, setOfCtf)
return setOfCtf
def checkOutputs(prot):
while not (prot.isFinished() or prot.isFailed()):
time.sleep(10)
prot = self._updateProtocol(prot)
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(prot.outputCTF,
"Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def _insertRecalculateSteps(self):
recalDeps = []
# For each psd insert the steps to process it
self.recalculateSet = SetOfCTF(filename=self.sqliteFile.get(),
objDoStore=False)
for ctf in self.recalculateSet:
line = ctf.getObjComment()
if ctf.isEnabled() and line:
# CTF Re-estimation
copyId = self._insertFunctionStep('copyMicDirectoryStep',
ctf.getObjId())
# Make estimation steps independent between them
stepId = self._insertFunctionStep('_restimateCTF',
ctf.getObjId(),
prerequisites=[copyId])
recalDeps.append(stepId)
return recalDeps
def _insertRecalculateSteps(self):
recalDeps = []
# For each psd insert the steps to process it
self.recalculateSet = SetOfCTF(filename=self.sqliteFile.get(),
objDoStore=False)
inputMics = self.getInputMicrographs()
for ctf in self.recalculateSet:
line = ctf.getObjComment()
if ctf.isEnabled() and line:
# CTF Re-estimation
# Make estimation steps independent between them
objId = ctf.getObjId()
stepId = self._insertFunctionStep('reEstimateCtfStep',
objId,
prerequisites=[])
recalDeps.append(stepId)
self.micDict[objId] = inputMics[objId].clone()
return recalDeps
def _createMicsSubSetFromCTF(self, inputCTFs):
""" Create a subset of Micrographs analyzing the CTFs. """
outputMics = self._createSetOfMicrographs()
setOfMics = inputCTFs.getMicrographs()
if setOfMics is None:
raise Exception('Could not create SetOfMicrographs subset from'
'this SetOfCTF, the micrographs were not set.')
outputMics.copyInfo(setOfMics)
modifiedSet = SetOfCTF(filename=self._dbName, prefix=self._dbPrefix)
for ctf in modifiedSet:
if ctf.isEnabled():
mic = ctf.getMicrograph()
outputMics.append(mic)
self._defineOutputs(outputMicrographs=outputMics)
self._defineTransformRelation(setOfMics, outputMics)
return outputMics
class TestCtfStreaming(BaseTest):
@classmethod
def setUpClass(cls):
setupTestProject(cls)
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
def checkOutputs(prot):
while not (prot.isFinished() or prot.isFailed()):
time.sleep(10)
prot = self.updateProtocol(prot)
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(
prot.outputCTF, "Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
kwargs = {
'xDim': 4096,
'yDim': 4096,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 15,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS
} # SetOfMicrographs
# create mic in streaming mode
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
# wait until a micrograph has been created
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(10)
protStream = self._updateProtocol(protStream)
if counter > 10:
break
counter += 1
# run ctffind4
# then introduce monitor, checking all the time ctf and saving to
# database
protCTF = ProtCTFFind(useCftfind4=True)
#time.sleep(10)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.findPhaseShift.set(True)
protCTF.slowSearch.set(False)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=True)
checkOutputs(protCTF)
kwargs = {'ctfDownFactor': 2, 'numberOfThreads': 3}
protCTF2 = self.newProtocol(XmippProtCTFMicrographs, **kwargs)
protCTF2.inputMicrographs.set(protStream.outputMicrographs)
self.proj.launchProtocol(protCTF2)
checkOutputs(protCTF2)
# run gctf
# check if box has nvidia cuda libs.
try:
nvmlInit() # fails if not GPU attached
protCTF3 = ProtGctf()
protCTF3.inputMicrographs.set(protStream.outputMicrographs)
protCTF3.ctfDownFactor.set(2)
self.proj.launchProtocol(protCTF3, wait=False)
checkOutputs(protCTF3)
except NVMLError, err:
print("Cannot find GPU."
"I assume that no GPU is connected to this machine")
# run xmipp ctf. Since this is the slower method wait until finish
# before running asserts
kwargs = {'numberOfThreads': MICS + 1}
protCTF2 = self.newProtocol(XmippProtCTFMicrographs, **kwargs)
protCTF2.inputMicrographs.set(protStream.outputMicrographs)
protCTF2.ctfDownFactor.set(2)
self.proj.launchProtocol(protCTF2, wait=True)
ctfSet = SetOfCTF(filename=protCTF._getPath(CTF_SQLITE))
baseFn = protCTF._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
self.assertSetSize(ctfSet, MICS, "Ctffind4 output size does not match")
for ctf in ctfSet:
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
def testCtfdiscrepancyWorkflow(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
ctfSet2 = SetOfCTF(filename=fnCTF2)
# create one fake micrographs image
projSize = 32
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 1000 + 10 * i
defocusV = 1000 + i
defocusAngle = i * 10
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
defocusU = 1000 + 20 * i
defocusV = 1000 + i
defocusAngle = i * 20
psdFile = "psd_2%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet2.append(ctf)
ctfSet1.write()
ctfSet2.write()
micSet.write()
# import micrograph set
args = {
'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF2 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF2)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
protCTF2.setObjLabel('import ctfs from scipion_2 ')
self.launchProtocol(protCTF1)
self.launchProtocol(protCTF2)
# launch CTF discrepancy protocol
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
protCtfDiscrepancy.setObjLabel('ctf discrepancy')
self.launchProtocol(protCtfDiscrepancy)
ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 1010)
def _checkNewInput(self):
if self.calculateConsensus:
# Check if there are new ctf to process from the input set
ctfsFile1 = self.inputCTF.get().getFileName()
ctfsFile2 = self.inputCTF2.get().getFileName()
self.lastCheck = getattr(self, 'lastCheck', datetime.now())
mTime = max(datetime.fromtimestamp(os.path.getmtime(ctfsFile1)),
datetime.fromtimestamp(os.path.getmtime(ctfsFile2)))
# If the input movies.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'SetOfCtf1'):
return None
ctfsSet1 = SetOfCTF(filename=ctfsFile1)
ctfsSet2 = SetOfCTF(filename=ctfsFile2)
ctfsSet1.loadAllProperties()
ctfsSet2.loadAllProperties()
if len(self.allCtf1) > 0:
newCtf1 = [ctf.clone() for ctf in
ctfsSet1.iterItems(orderBy='creation',
where='creation>"' + str(
self.checkCtf1) + '"')]
else:
newCtf1 = [ctf.clone() for ctf in ctfsSet1]
self.allCtf1 = self.allCtf1 + newCtf1
if len(newCtf1) > 0:
for ctf in ctfsSet1.iterItems(orderBy='creation',
direction='DESC'):
self.checkCtf1 = ctf.getObjCreation()
break
if len(self.allCtf2) > 0:
newCtf2 = [ctf.clone() for ctf in
ctfsSet2.iterItems(orderBy='creation',
where='creation>"' + str(
self.checkCtf2) + '"')]
else:
newCtf2 = [ctf.clone() for ctf in ctfsSet2]
self.allCtf2 = self.allCtf2 + newCtf2
if len(newCtf2) > 0:
for ctf in ctfsSet2.iterItems(orderBy='creation',
direction='DESC'):
self.checkCtf2 = ctf.getObjCreation()
break
self.lastCheck = datetime.now()
self.isStreamClosed = ctfsSet1.isStreamClosed() and \
ctfsSet2.isStreamClosed()
ctfsSet1.close()
ctfsSet2.close()
outputStep = self._getFirstJoinStep()
if len(set(self.allCtf1)) > len(set(self.processedDict)) and \
len(set(self.allCtf2)) > len(set(self.processedDict)):
fDeps = self._insertNewCtfsSteps(ctfsSet1, ctfsSet2,
self.insertedDict)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
else:
ctfFile = self.inputCTF.get().getFileName()
now = datetime.now()
self.lastCheck = getattr(self, 'lastCheck', now)
mTime = datetime.fromtimestamp(os.path.getmtime(ctfFile))
self.debug('Last check: %s, modification: %s'
% (pwutils.prettyTime(self.lastCheck),
pwutils.prettyTime(mTime)))
# Open input ctfs.sqlite and close it as soon as possible
ctfSet = self._loadInputCtfSet()
self.isStreamClosed = ctfSet.isStreamClosed()
self.allCtf1 = [m.clone() for m in ctfSet]
ctfSet.close()
# If the input ctfs.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime and hasattr(self, 'allCtf1'):
return None
self.lastCheck = now
newCtf = any(ctf.getObjId() not in
self.insertedDict for ctf in self.allCtf1)
outputStep = self._getFirstJoinStep()
if newCtf:
fDeps = self._insertNewSelectionSteps(self.insertedDict,
self.allCtf1)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {'xDim': 1024,
'yDim': 1024,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# create input micrographs
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(2)
protStream = self._updateProtocol(protStream)
if counter > 100:
break
counter += 1
# then introduce monitor, checking all the time ctf and
# saving to database
protCTF = ProtCTFFind(useCftfind4=True)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=False)
counter = 1
while not protCTF.hasAttribute('outputCTF'):
time.sleep(2)
protCTF = self._updateProtocol(protCTF)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 28000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 7
}
protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel.inputCTF.set(protCTF.outputCTF)
self.proj.launchProtocol(protCTFSel, wait=False)
counter = 1
while not protCTFSel.hasAttribute('outputCTF'):
time.sleep(2)
protCTFSel = self._updateProtocol(protCTFSel)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 3.7,
}
protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
self.proj.launchProtocol(protCTFSel2)
counter = 1
while not (protCTFSel2.hasAttribute('outputCTF') and
protCTFSel2.hasAttribute('outputMicrographs')):
time.sleep(2)
protCTFSel2 = self._updateProtocol(protCTFSel2)
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the first CTF
# selection protocol must be equal to the number of mics in the
# CTF estimation protocol
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
counter = 1
while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
time.sleep(2)
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the second CTF
# selection protocol must be equal to the number of ctfs in the first
# CTF selection protocol
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
counter = 1
while not (ctfSet1.getSize() == (micSetDiscarded2.getSize() +
micSet2.getSize())):
time.sleep(2)
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
for ctf in ctfSet:
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
astigm = defocusU - defocusV
resol = ctf.getResolution() # TODO
if defocusU < 1000 \
or defocusU > 28000 \
or defocusV < 1000 \
or defocusV > 28000 \
or astigm > 1000 or resol > 3.7:
self.assertTrue(False, "A CTF without the correct parameters"
" is included in the output set")
def testCtfdiscrepancyWorkflow(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
ctfSet2 = SetOfCTF(filename=fnCTF2)
# create one fake micrographs image
projSize = 32
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 1000+10*i
defocusV = 1000+i
defocusAngle = i*10
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU,
defocusV,
defocusAngle,
psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
defocusU = 1000+20*i
defocusV = 1000+i
defocusAngle = i*20
psdFile = "psd_2%04d" % i
ctf = self._getCTFModel(defocusU,
defocusV,
defocusAngle,
psdFile)
ctf.setMicrograph(mic)
ctfSet2.append(ctf)
ctfSet1.write()
ctfSet2.write()
micSet.write()
# import micrograph set
args = {'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF2 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF2)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
protCTF2.setObjLabel('import ctfs from scipion_2 ')
self.launchProtocol(protCTF1)
self.launchProtocol(protCTF2)
# launch CTF discrepancy protocol
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
protCtfDiscrepancy.setObjLabel('ctf discrepancy')
self.launchProtocol(protCtfDiscrepancy)
ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 1010)
def _checkNewCTFs(self, micSet):
""" Check for already computed CTF and update the output set. """
newCTFs = []
ctfDict = {}
ctfSet = SetOfCTF(filename=self._getPath('ctfs.sqlite'))
ctfSet.setMicrographs(self.inputMicrographs.get())
for ctf in ctfSet:
ctfDict[ctf.getObjId()] = True
if ctfDict: # it means there are previous ctfs computed
ctfSet.loadAllProperties()
if ctfSet.getSize():
ctfSet.enableAppend()
else:
ctfSet.setStreamState(ctfSet.STREAM_OPEN)
for micFn, micDir, mic in self._iterMicrographs(micSet):
if (exists(self._getMicrographDone(micDir))
and not mic.getObjId() in ctfDict):
ctf = self._createCtfModel(mic)
ctfSet.append(ctf)
newCTFs.append(mic.getObjId())
return ctfSet, newCTFs
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {
'xDim': 1024,
'yDim': 1024,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# create input micrographs
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(2)
protStream = self._updateProtocol(protStream)
if counter > 100:
break
counter += 1
# then introduce monitor, checking all the time ctf and
# saving to database
protCTF = ProtCTFFind(useCftfind4=True)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=False)
counter = 1
while not protCTF.hasAttribute('outputCTF'):
time.sleep(2)
protCTF = self._updateProtocol(protCTF)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 28000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 7
}
protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel.inputCTF.set(protCTF.outputCTF)
self.proj.launchProtocol(protCTFSel, wait=False)
counter = 1
while not protCTFSel.hasAttribute('outputCTF'):
time.sleep(2)
protCTFSel = self._updateProtocol(protCTFSel)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 3.7,
}
protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
self.proj.launchProtocol(protCTFSel2)
counter = 1
while not (protCTFSel2.hasAttribute('outputCTF')
and protCTFSel2.hasAttribute('outputMicrographs')):
time.sleep(2)
protCTFSel2 = self._updateProtocol(protCTFSel2)
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the first CTF
# selection protocol must be equal to the number of mics in the
# CTF estimation protocol
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(filename=protCTFSel._getPath(MIC_SQLITE))
counter = 1
while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
time.sleep(2)
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the second CTF
# selection protocol must be equal to the number of ctfs in the first
# CTF selection protocol
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
counter = 1
while not (ctfSet1.getSize()
== (micSetDiscarded2.getSize() + micSet2.getSize())):
time.sleep(2)
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
for ctf in ctfSet:
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
astigm = defocusU - defocusV
resol = ctf.getResolution() # TODO
if defocusU < 1000 \
or defocusU > 28000 \
or defocusV < 1000 \
or defocusV > 28000 \
or astigm > 1000 or resol > 3.7:
self.assertTrue(
False, "A CTF without the correct parameters"
" is included in the output set")