def _checkProcessedData(self):
if self.setof == SET_OF_MOVIES:
objSet = SetOfMovies(filename=self._getPath('movies.sqlite'))
elif self.setof == SET_OF_MICROGRAPHS:
objSet = \
SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
objSet = \
SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_PARTICLES:
objSet = SetOfParticles(filename=self._getPath('particles.sqlite'))
else:
raise Exception('Unknown data type')
newObjSet, newObj = self._checkNewItems(objSet)
if self.setof == SET_OF_MOVIES:
newObjSet.setFramesRange(self.inputMovies.get().getFramesRange())
# check if end ....
if self.setof != SET_OF_PARTICLES:
self.nDims = self.nDim.get()
endObjs = newObjSet.getSize() == self.nDims
if newObj:
if endObjs:
newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
self._updateOutput(newObjSet)
newObjSet.close()
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
""" Create a subset of Micrographs Tilt Pair. """
output = MicrographsTiltPair(
filename=self._getPath('micrographs_pairs.sqlite'))
modifiedSet = MicrographsTiltPair(filename=self._dbName,
prefix=self._dbPrefix)
inputU = micrographsTiltPair.getUntilted()
inputT = micrographsTiltPair.getTilted()
outputU = SetOfMicrographs(
filename=self._getPath('mics_untilted.sqlite'))
outputT = SetOfMicrographs(
filename=self._getPath('mics_tilted.sqlite'))
outputU.copyInfo(inputU)
outputT.copyInfo(inputT)
for micPair, u, t in izip(modifiedSet, inputU, inputT):
if micPair.isEnabled():
output.append(micPair)
outputU.append(u)
outputT.append(t)
output.setUntilted(outputU)
output.setTilted(outputT)
# Register outputs
outputDict = {'outputMicrographsTiltPair': output}
self._defineOutputs(**outputDict)
self._defineTransformRelation(micrographsTiltPair, output)
return output
def test_fromScipion(self):
""" Import an EMX file with micrographs and defocus
"""
micsSqlite = self.dsXmipp.getFile('micrographs/micrographs.sqlite')
print "Importing from sqlite: ", micsSqlite
micSet = SetOfMicrographs(filename=micsSqlite)
# Gold values
#_samplingRate -> 1.237
#_acquisition._voltage -> 300.0
#_acquisition._magnification -> 50000.0
for k in ['_samplingRate','_acquisition._voltage','_acquisition._magnification']:
print k, "->", micSet.getProperty(k)
prot = self.newProtocol(ProtImportMicrographs,
objLabel='from scipion',
importFrom=ProtImportMicrographs.IMPORT_FROM_SCIPION,
sqliteFile=micsSqlite,
samplingRate=float(micSet.getProperty('_samplingRate')),
voltage=float(micSet.getProperty('_acquisition._voltage')),
magnification=int(float(micSet.getProperty('_acquisition._magnification')))
)
self.launchProtocol(prot)
micSet = getattr(prot, 'outputMicrographs', None)
self.assertIsNotNone(micSet)
self.assertAlmostEqual(1.237, micSet.getSamplingRate())
acq = micSet.getAcquisition()
self.assertAlmostEqual(300., acq.getVoltage())
self.assertAlmostEqual(50000., acq.getMagnification())
def loadInputs(self):
micsFn = self.getInputMicrographs().getFileName()
micsSet = SetOfMicrographs(filename=micsFn)
micsSet.loadAllProperties()
availableMics = []
for mic in micsSet:
availableMics.append(mic.getObjId())
micsSetClosed = micsSet.isStreamClosed()
micsSet.close()
partsFn = self.getInputParticles().getFileName()
partsSet = SetOfParticles(filename=partsFn)
partsSet.loadAllProperties()
newParts = []
newMics = []
for item in partsSet:
micKey = item.getCoordinate().getMicId()
if micKey not in self.micsDone and micKey in availableMics:
newParts.append(item.getObjId())
if not micKey in self.micsDone:
newMics.append(micKey)
self.micsDone += newMics
self.inputSize = partsSet.getSize()
partSetClosed = partsSet.isStreamClosed()
partsSet.close()
return newParts, micsSetClosed and partSetClosed
def _createSubSetFromMicrographsTiltPair(self, micrographsTiltPair):
""" Create a subset of Micrographs Tilt Pair. """
output = MicrographsTiltPair(
filename=self._getPath('micrographs_pairs.sqlite'))
print "self._dbName=%s" % self._dbName
modifiedSet = MicrographsTiltPair(filename=self._dbName,
prefix=self._dbPrefix)
inputU = micrographsTiltPair.getUntilted()
inputT = micrographsTiltPair.getTilted()
outputU = SetOfMicrographs(
filename=self._getPath('mics_untilted.sqlite'))
outputT = SetOfParticles(filename=self._getPath('mics_tilted.sqlite'))
outputU.copyInfo(inputU)
outputT.copyInfo(inputT)
for micPairI in modifiedSet:
untilted = micPairI.getUntilted()
tilted = micPairI.getTilted()
if micPairI.isEnabled():
micPairO = TiltPair()
micPairO.setUntilted(untilted)
micPairO.setTilted(tilted)
output.append(micPairO)
outputU.append(untilted)
outputT.append(tilted)
output.setUntilted(outputU)
output.setTilted(outputT)
# Register outputs
outputDict = {'outputMicrographsTiltPair': output}
self._defineOutputs(**outputDict)
self._defineTransformRelation(micrographsTiltPair, output)
return output
def writeCtfStarStep(self):
inputCTF = self.inputCTF.get()
if self.micrographSource == 0: # same as CTF estimation
ctfMicSet = inputCTF.getMicrographs()
else:
ctfMicSet = self.inputMicrographs.get()
micSet = SetOfMicrographs(filename=':memory:')
psd = inputCTF.getFirstItem().getPsdFile()
hasPsd = psd and os.path.exists(psd)
if hasPsd:
psdPath = self._getPath('PSD')
pwutils.makePath(psdPath)
print "Writing PSD files to %s" % psdPath
for ctf in inputCTF:
# Get the corresponding micrograph
mic = ctfMicSet[ctf.getObjId()]
if mic is None:
print(
"Skipping CTF id: %s, it is missing from input "
"micrographs. " % ctf.getObjId())
continue
micFn = mic.getFileName()
if not os.path.exists(micFn):
print "Skipping micrograph %s, it does not exists. " % micFn
continue
mic2 = mic.clone()
mic2.setCTF(ctf)
if hasPsd:
psdFile = ctf.getPsdFile()
newPsdFile = join(psdPath, '%s_psd.mrc' % mic.getMicName())
if not os.path.exists(psdFile):
print "PSD file %s does not exits" % psdFile
print "Skipping micrograph %s" % micFn
continue
pwutils.copyFile(psdFile, newPsdFile)
ctf.setPsdFile(newPsdFile)
micSet.append(mic2)
starFile = self._getPath(self.CTF_STAR_FILE % self.getObjId())
print "Writing set: %s" % inputCTF
print " to: %s" % starFile
writeSetOfMicrographs(micSet,
starFile,
preprocessImageRow=self.preprocessMicrograph)
# Let's create a link from the project roots to facilitate the import
# of the star file into a Relion project
pwutils.createLink(starFile, os.path.basename(starFile))
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the micrographs in
# the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
# create tmp set with all mics from coords set
coordMics = SetOfMicrographs(filename=':memory:')
coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())
for micU, micT in izip(
self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micU.cleanObjId()
micT.cleanObjId()
coordMics.append(micU)
coordMics.append(micT)
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeExt(mic.getMicName())] = micPos
# now match micDict and inputMics
if any(
pwutils.removeExt(mic.getMicName()) in micDict
for mic in self.inputMics):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception(
'Could not match input micrographs and coordinates '
'by micName.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' %
(micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _stepsCheck(self):
setOfMicFn = self._getPath('micrographs.sqlite')
micSet = SetOfMicrographs(filename=setOfMicFn)
micSet, newMic = self._checkNewMics(micSet)
#check if end ....
endMics = micSet.getSize() == self.nDim.get()
if newMic:
if endMics:
micSet.setStreamState(micSet.STREAM_CLOSED)
self._updateOutput(micSet)
micSet.close()
def _stepsCheck(self):
if self.setof == SET_OF_MOVIES:
objSet = SetOfMovies(filename=self._getPath('movies.sqlite'))
elif self.setof == SET_OF_MICROGRAPHS:
objSet = \
SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
elif self.setof == SET_OF_RANDOM_MICROGRAPHS:
objSet = \
SetOfMicrographs(filename=self._getPath('micrographs.sqlite'))
else:
raise Exception('Unknown data type')
newObjSet, newObj = self._checkNewItems(objSet)
# check if end ....
endObjs = newObjSet.getSize() == self.nDim.get()
if newObj:
if endObjs:
newObjSet.setStreamState(newObjSet.STREAM_CLOSED)
self._updateOutput(newObjSet)
newObjSet.close()
def checkMicSet(self, micSet, goldFn):
""" Compare micrographs of micSet with the
ones in the goldFn. Maybe except the full path.
"""
goldSet = SetOfMicrographs(filename = goldFn)
for mic1, mic2 in izip(goldSet, micSet):
# Remove the absolute path in the micrographs to
# really check that the attributes should be equal
mic1.setFileName(os.path.basename(mic1.getFileName()))
mic2.setFileName(os.path.basename(mic2.getFileName()))
self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
def _checkNewInput(self):
# Check if there are new micrographs to process from the input set
micsFile = self.inputMicrographs.get().getFileName()
micsSet = SetOfMicrographs(filename=micsFile)
micsSet.loadAllProperties()
self.SetOfMicrographs = [m.clone() for m in micsSet]
self.streamClosed = micsSet.isStreamClosed()
micsSet.close()
newMics = any(m.getObjId() not in self.insertedDict
for m in self.inputMics)
outputStep = self._getFirstJoinStep()
if newMics:
fDeps = self._insertNewMicsSteps(self.insertedDict, self.inputMics)
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def importMicrographs(self):
""" Import a SetOfMicrographs from a given sqlite file. """
inputSet = SetOfMicrographs(filename=self._sqliteFile)
self._findImagesPath(inputSet)
micSet = self.protocol._createSetOfMicrographs()
micSet.setObjComment('Micrographs imported from sqlite file:\n%s' %
self._sqliteFile)
# Update both samplingRate and acquisition with parameters
# selected in the protocol form
self.protocol.setSamplingRate(micSet)
micSet.setIsPhaseFlipped(self.protocol.haveDataBeenPhaseFlipped.get())
self.protocol.fillAcquisition(micSet.getAcquisition())
# Read the micrographs from the 'self._sqliteFile' metadata
# but fixing the filenames with new ones (linked or copy to extraDir)
micSet.copyItems(inputSet, updateItemCallback=self._updateParticle)
self.protocol._defineOutputs(outputMicrographs=micSet)
def _setupBasicProperties(self):
# Get sampling rate and inputMics according to micsSource type
self.inputCoords = self.getCoords()
self.uMics, self.tMics = self.getInputMicrographs()
self.samplingInput = self.inputCoords.getUntilted().getMicrographs(
).getSamplingRate()
self.samplingMics = self.uMics.getSamplingRate()
self.samplingFactor = float(self.samplingMics / self.samplingInput)
# create tmp set with all mic pairs
self.inputMics = SetOfMicrographs(filename=':memory:')
self.inputMics.copyInfo(self.uMics)
self.inputMics.setStore(False)
for micU, micT in izip(self.uMics, self.tMics):
micU.cleanObjId()
micT.cleanObjId()
self.inputMics.append(micU)
self.inputMics.append(micT)
def test_micrographImport(self):
""" Import an EMX file with micrographs and defocus
"""
emxFn = self.dataset.getFile('emxMicrographCtf1')
protEmxImport = self.newProtocol(
ProtImportMicrographs,
objLabel='emx - import mics',
importFrom=ProtImportMicrographs.IMPORT_FROM_EMX,
emxFile=emxFn,
magnification=10000,
samplingRate=2.46,
)
self.launchProtocol(protEmxImport)
micFn = self.dataset.getFile('emxMicrographCtf1Gold')
mics = SetOfMicrographs(filename=micFn)
for mic1, mic2 in izip(mics, protEmxImport.outputMicrographs):
# Remove the absolute path in the micrographs to
# really check that the attributes should be equal
mic1.setFileName(os.path.basename(mic1.getFileName()))
mic2.setFileName(os.path.basename(mic2.getFileName()))
self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
def test_particleImportDefocus(self):
""" Import an EMX file with a stack of particles
that has defocus
"""
emxFn = self.dataset.getFile('defocusParticleT2')
protEmxImport = self.newProtocol(
ProtImportParticles,
objLabel='emx - import ctf',
importFrom=ProtImportParticles.IMPORT_FROM_EMX,
emxFile=emxFn,
alignType=3,
magnification=10000,
samplingRate=2.8)
self.launchProtocol(protEmxImport)
micFn = self.dataset.getFile('micrographsGoldT2')
mics = SetOfMicrographs(filename=micFn)
for mic1, mic2 in izip(mics, protEmxImport.outputMicrographs):
# Remove the absolute path in the micrographs to
# really check that the attributes should be equal
mic1.setFileName(os.path.basename(mic1.getFileName()))
mic2.setFileName(os.path.basename(mic2.getFileName()))
self.assertTrue(mic1.equalAttributes(mic2, verbose=True))
def _stepsCheck(self):
# For now the streaming is not allowed for recalculate CTF
if self.recalculate:
return
# Check if there are new micrographs to process
micFn = self.inputMicrographs.get().getFileName()
micSet = SetOfMicrographs(filename=micFn)
micSet.loadAllProperties()
streamClosed = micSet.isStreamClosed()
outputStep = self._getFirstJoinStep()
self._checkNewMicrographs(micSet, outputStep)
ctfSet, newCTFs = self._checkNewCTFs(micSet)
if ctfSet is None:
return
endCTFs = streamClosed and micSet.getSize() == ctfSet.getSize()
if newCTFs:
# Check if it is the first time we are registering CTF to
# create the CTF_RELATION only once
firstTime = not self.hasAttribute('outputCTF')
ctfSet.setMicrographs(self.inputMics)
self._computeDefocusRange(ctfSet)
streamMode = ctfSet.STREAM_CLOSED if endCTFs else ctfSet.STREAM_OPEN
self._updateOutputSet('outputCTF', ctfSet, streamMode)
if firstTime: # define relation just once
self._defineCtfRelation(self.inputMics, ctfSet)
else:
ctfSet.close()
if outputStep and outputStep.isWaiting() and endCTFs:
outputStep.setStatus(STATUS_NEW)
micSet.close()
def test_pattern(self):
""" Import several Particles from a given pattern.
"""
kwargs = {
'xDim': 1024,
'yDim': 1024,
'nDim': MICS,
'samplingRate': 1.25,
'creationInterval': 5,
'delay': 0,
'setof': SET_OF_RANDOM_MICROGRAPHS # SetOfMicrographs
}
# create input micrographs
protStream = self.newProtocol(ProtCreateStreamData, **kwargs)
protStream.setObjLabel('create Stream Mic')
self.proj.launchProtocol(protStream, wait=False)
counter = 1
while not protStream.hasAttribute('outputMicrographs'):
time.sleep(2)
protStream = self._updateProtocol(protStream)
if counter > 100:
break
counter += 1
# then introduce monitor, checking all the time ctf and
# saving to database
protCTF = ProtCTFFind(useCftfind4=True)
protCTF.inputMicrographs.set(protStream.outputMicrographs)
protCTF.ctfDownFactor.set(2)
protCTF.highRes.set(0.4)
protCTF.lowRes.set(0.05)
protCTF.numberOfThreads.set(4)
self.proj.launchProtocol(protCTF, wait=False)
counter = 1
while not protCTF.hasAttribute('outputCTF'):
time.sleep(2)
protCTF = self._updateProtocol(protCTF)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 28000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 7
}
protCTFSel = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel.inputCTF.set(protCTF.outputCTF)
self.proj.launchProtocol(protCTFSel, wait=False)
counter = 1
while not protCTFSel.hasAttribute('outputCTF'):
time.sleep(2)
protCTFSel = self._updateProtocol(protCTFSel)
if counter > 100:
break
counter += 1
kwargs = {
'maxDefocus': 40000,
'minDefocus': 1000,
'astigmatism': 1000,
'resolution': 3.7,
}
protCTFSel2 = self.newProtocol(XmippProtCTFSelection, **kwargs)
protCTFSel2.inputCTF.set(protCTFSel.outputCTF)
self.proj.launchProtocol(protCTFSel2)
counter = 1
while not (protCTFSel2.hasAttribute('outputCTF')
and protCTFSel2.hasAttribute('outputMicrographs')):
time.sleep(2)
protCTFSel2 = self._updateProtocol(protCTFSel2)
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the first CTF
# selection protocol must be equal to the number of mics in the
# CTF estimation protocol
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(filename=protCTFSel._getPath(MIC_SQLITE))
counter = 1
while not ((micSetDiscarded1.getSize() + micSet1.getSize()) == 10):
time.sleep(2)
micSetDiscarded1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_DISCARDED_SQLITE))
micSet1 = SetOfMicrographs(
filename=protCTFSel._getPath(MIC_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
# AJ the number of micrographs discarded and selected in the second CTF
# selection protocol must be equal to the number of ctfs in the first
# CTF selection protocol
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
counter = 1
while not (ctfSet1.getSize()
== (micSetDiscarded2.getSize() + micSet2.getSize())):
time.sleep(2)
micSetDiscarded2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_DISCARDED_SQLITE))
micSet2 = SetOfMicrographs(
filename=protCTFSel2._getPath(MIC_SQLITE))
ctfSet1 = SetOfCTF(filename=protCTFSel._getPath(CTF_SQLITE))
if counter > 100:
self.assertTrue(False)
counter += 1
ctfSet = SetOfCTF(filename=protCTFSel2._getPath(CTF_SQLITE))
for ctf in ctfSet:
defocusU = ctf.getDefocusU()
defocusV = ctf.getDefocusV()
astigm = defocusU - defocusV
resol = ctf.getResolution() # TODO
if defocusU < 1000 \
or defocusU > 28000 \
or defocusV < 1000 \
or defocusV > 28000 \
or astigm > 1000 or resol > 3.7:
self.assertTrue(
False, "A CTF without the correct parameters"
" is included in the output set")
def testCtfdiscrepancyWorkflow(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
ctfSet2 = SetOfCTF(filename=fnCTF2)
# create one fake micrographs image
projSize = 32
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 1000 + 10 * i
defocusV = 1000 + i
defocusAngle = i * 10
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
defocusU = 1000 + 20 * i
defocusV = 1000 + i
defocusAngle = i * 20
psdFile = "psd_2%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet2.append(ctf)
ctfSet1.write()
ctfSet2.write()
micSet.write()
# import micrograph set
args = {
'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF2 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF2)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
protCTF2.setObjLabel('import ctfs from scipion_2 ')
self.launchProtocol(protCTF1)
self.launchProtocol(protCTF2)
# launch CTF discrepancy protocol
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
protCtfDiscrepancy.setObjLabel('ctf discrepancy')
self.launchProtocol(protCtfDiscrepancy)
ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 1010)
