def _checkNewCTFs(self, micSet):
""" Check for already computed CTF and update the output set. """
newCTFs = []
ctfDict = {}
ctfSet = SetOfCTF(filename=self._getPath('ctfs.sqlite'))
ctfSet.setMicrographs(self.inputMicrographs.get())
for ctf in ctfSet:
ctfDict[ctf.getObjId()] = True
if ctfDict: # it means there are previous ctfs computed
ctfSet.loadAllProperties()
if ctfSet.getSize():
ctfSet.enableAppend()
else:
ctfSet.setStreamState(ctfSet.STREAM_OPEN)
for micFn, micDir, mic in self._iterMicrographs(micSet):
if (exists(self._getMicrographDone(micDir))
and not mic.getObjId() in ctfDict):
ctf = self._createCtfModel(mic)
ctfSet.append(ctf)
newCTFs.append(mic.getObjId())
return ctfSet, newCTFs
def checkOutputs(prot):
t0 = time.time()
while not (prot.isFinished() or prot.isFailed()):
# Time out 6 minutes, just in case
tdelta = time.time() - t0
if tdelta > 6 * 60:
break
time.sleep(10)
prot = self._updateProtocol(prot)
# Check if the protocol is still launched
if prot.isLaunched():
break
elif prot.isScheduled():
continue
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(
prot.outputCTF, "Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def checkOutputs(prot):
while not (prot.isFinished() or prot.isFailed()):
time.sleep(10)
prot = self.updateProtocol(prot)
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(
prot.outputCTF, "Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def checkOutputs(prot):
while not (prot.isFinished() or prot.isFailed()):
time.sleep(10)
prot = self._updateProtocol(prot)
if prot.hasAttribute("outputCTF"):
ctfSet = SetOfCTF(filename=prot._getPath(CTF_SQLITE))
baseFn = prot._getPath(CTF_SQLITE)
self.assertTrue(os.path.isfile(baseFn))
counter = 0
if ctfSet.getSize() > counter:
counter += 1
for ctf in ctfSet:
self.assertNotEqual(ctf._resolution.get(), None)
self.assertNotEqual(ctf._fitQuality.get(), None)
self.assertNotEqual(ctf.isEnabled(), None)
self.assertNotEqual(ctf._defocusU.get(), None)
self.assertNotEqual(ctf._defocusV.get(), None)
self.assertNotEqual(ctf._defocusRatio.get(), None)
if ctf.getPhaseShift():
self.assertNotEqual(ctf.getPhaseShift(), None)
self.assertIsNotNone(prot.outputCTF,
"Error: outputCTF is not produced "
"in %s." % prot.getClassName())
self.assertEqual(prot.outputCTF.getSize(), MICS)
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {'xDim': 1024,
'yDim': 1024,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# create input micrographs
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(2)
protStream = self._updateProtocol(protStream)
if counter > 100:
break
counter += 1
# then introduce monitor, checking all the time ctf and
# saving to database
protCTF = ProtCTFFind(useCftfind4=True)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=False)
counter = 1
while not protCTF.hasAttribute('outputCTF'):
time.sleep(2)
protCTF = self._updateProtocol(protCTF)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 28000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 7
}
protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel.inputCTF.set(protCTF.outputCTF)
self.proj.launchProtocol(protCTFSel, wait=False)
counter = 1
while not protCTFSel.hasAttribute('outputCTF'):
time.sleep(2)
protCTFSel = self._updateProtocol(protCTFSel)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 3.7,
}
protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
self.proj.launchProtocol(protCTFSel2)
counter = 1
while not (protCTFSel2.hasAttribute('outputCTF') and
protCTFSel2.hasAttribute('outputMicrographs')):
time.sleep(2)
protCTFSel2 = self._updateProtocol(protCTFSel2)
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the first CTF
# selection protocol must be equal to the number of mics in the
# CTF estimation protocol
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
counter = 1
while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
time.sleep(2)
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the second CTF
# selection protocol must be equal to the number of ctfs in the first
# CTF selection protocol
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
counter = 1
while not (ctfSet1.getSize() == (micSetDiscarded2.getSize() +
micSet2.getSize())):
time.sleep(2)
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
for ctf in ctfSet:
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
astigm = defocusU - defocusV
resol = ctf.getResolution() # TODO
if defocusU < 1000 \
or defocusU > 28000 \
or defocusV < 1000 \
or defocusV > 28000 \
or astigm > 1000 or resol > 3.7:
self.assertTrue(False, "A CTF without the correct parameters"
" is included in the output set")
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {
'xDim': 1024,
'yDim': 1024,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# create input micrographs
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(2)
protStream = self._updateProtocol(protStream)
if counter > 100:
break
counter += 1
# then introduce monitor, checking all the time ctf and
# saving to database
protCTF = ProtCTFFind(useCftfind4=True)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=False)
counter = 1
while not protCTF.hasAttribute('outputCTF'):
time.sleep(2)
protCTF = self._updateProtocol(protCTF)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 28000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 7
}
protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel.inputCTF.set(protCTF.outputCTF)
self.proj.launchProtocol(protCTFSel, wait=False)
counter = 1
while not protCTFSel.hasAttribute('outputCTF'):
time.sleep(2)
protCTFSel = self._updateProtocol(protCTFSel)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 3.7,
}
protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
self.proj.launchProtocol(protCTFSel2)
counter = 1
while not (protCTFSel2.hasAttribute('outputCTF')
and protCTFSel2.hasAttribute('outputMicrographs')):
time.sleep(2)
protCTFSel2 = self._updateProtocol(protCTFSel2)
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the first CTF
# selection protocol must be equal to the number of mics in the
# CTF estimation protocol
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(filename=protCTFSel._getPath(MIC_SQLITE))
counter = 1
while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
time.sleep(2)
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the second CTF
# selection protocol must be equal to the number of ctfs in the first
# CTF selection protocol
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
counter = 1
while not (ctfSet1.getSize()
== (micSetDiscarded2.getSize() + micSet2.getSize())):
time.sleep(2)
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
for ctf in ctfSet:
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
astigm = defocusU - defocusV
resol = ctf.getResolution() # TODO
if defocusU < 1000 \
or defocusU > 28000 \
or defocusV < 1000 \
or defocusV > 28000 \
or astigm > 1000 or resol > 3.7:
self.assertTrue(
False, "A CTF without the correct parameters"
" is included in the output set")