def _preprocessImageRow(self, img, imgRow):
from convert import setupCTF, copyOrLinkFileName
if self._imgPath is not None:
copyOrLinkFileName(imgRow, self._imgPath,
self.protocol._getExtraPath())
setupCTF(imgRow, self.protocol.samplingRate.get())
if self._micIdOrName:
micId = imgRow.getValue('rlnMicrographId', None)
micName = imgRow.getValue('rlnMicrographName', None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath(
'fake_micrograph%6d' % micId)
mic.setFileName(micName)
mic.setMicName(os.path.basename(micName))
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow.setValue('rlnMicrographId', long(mic.getObjId()))
def _preprocessImageRow(self, img, imgRow):
from convert import setupCTF, copyOrLinkFileName
if self._imgPath is not None:
copyOrLinkFileName(imgRow, self._imgPath, self.protocol._getExtraPath())
setupCTF(imgRow, self.protocol.samplingRate.get())
if self._micIdOrName:
micId = imgRow.getValue('rlnMicrographId', None)
micName = imgRow.getValue('rlnMicrographName', None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath('fake_micrograph%6d' % micId)
mic.setFileName(micName)
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow.setValue('rlnMicrographId', long(mic.getObjId()))
def _checkNewMics(self, micSet):
""" Check for already computed CTF and update the output set. """
micDict = {}
newMic = False
for mic in micSet:
micDict[mic.getFileName()] = True
if micDict:
if micSet.getSize():
micSet.enableAppend()
micSet.loadAllProperties()
else:
micSet.setStreamState(micSet.STREAM_OPEN)
acquisition = Acquisition()
acquisition.setMagnification(self._magnification)
acquisition.setVoltage(self._voltage)
acquisition.setSphericalAberration(self._sphericalAberration)
acquisition.setAmplitudeContrast(self._amplitudeContrast)
micSet.setAcquisition(acquisition)
micSet.setSamplingRate(self.samplingRate.get())
mic = Micrograph()
counter = 0
for k, v in self.xmippMic.iteritems():
counter += 1
if (k not in micDict):
mic.setFileName(k)
mic.setMicName(basename(k))
mic.setObjId(counter)
micSet.append(mic)
newMic = True
return micSet, newMic #why a dictionary, a boolean may be enought
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs):
""" Write a SetOfMicrograph as expected in EMX format
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
micSet: micrographs set associated with the particles
**kwargs: writeImages: if set to False, only coordinates are exported.
imagesStack: if passed all images will be output into a single
stack file.
imagesPrefix: used when not imagesStack is passed. A different
stack will be created per micrograph.
"""
writeImages = kwargs.get('writeImages', True)
imagesPrefix = kwargs.get('imagesPrefix', None)
micDict = {}
# Use singleMic for count all particles to be written to a single stack
imagesStack = kwargs.get('imagesStack', None)
singleMic = Micrograph()
singleMic.setFileName(imagesStack)
singleMic.counter = pwobj.Integer(0)
def _getMicKey(particle):
coord = particle.getCoordinate()
if coord is None or coord.getMicName() is None:
return '%05d' % particle.getMicId()
else:
return pwutils.removeExt(coord.getMicName())
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix,
'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
ih = ImageHandler()
partAlign = partSet.getAlignment()
for particle in partSet:
if writeImages:
newLoc = _getLocation(particle)
ih.convert(particle, newLoc)
localFn = basename(newLoc[1])
particle.setLocation(newLoc[0], localFn)
emxObj = _particleToEmx(emxData, particle, micSet, partAlign)
else:
emxObj = _coordinateToEmx(emxData, particle, micSet)
emxData.addObject(emxObj)
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs):
""" Write a SetOfMicrograph as expected in EMX format
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
micSet: micrographs set associated with the particles
**kwargs: writeImages: if set to False, only coordinates are exported.
imagesStack: if passed all images will be output into a single
stack file.
imagesPrefix: used when not imagesStack is passed. A different
stack will be created per micrograph.
"""
writeImages = kwargs.get('writeImages', True)
imagesPrefix = kwargs.get('imagesPrefix', None)
micDict = {}
# Use singleMic for count all particles to be written to a single stack
imagesStack = kwargs.get('imagesStack', None)
singleMic = Micrograph()
singleMic.setFileName(imagesStack)
singleMic.counter = pwobj.Integer(0)
def _getMicKey(particle):
coord = particle.getCoordinate()
if coord is None or coord.getMicName() is None:
return '%05d' % particle.getMicId()
else:
return pwutils.removeExt(coord.getMicName())
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
ih = ImageHandler()
partAlign = partSet.getAlignment()
for particle in partSet:
if writeImages:
newLoc = _getLocation(particle)
ih.convert(particle, newLoc)
localFn = basename(newLoc[1])
particle.setLocation(newLoc[0], localFn)
emxObj = _particleToEmx(emxData, particle, micSet, partAlign)
else:
emxObj = _coordinateToEmx(emxData, particle, micSet)
emxData.addObject(emxObj)
def _preprocessParticleRow(self, img, imgRow):
if self._imgPath:
# Create a link or copy files to extraPath
# and update the Row properly
index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE))
imgBase = basename(fn)
imgDst = self.protocol._getExtraPath(imgBase)
if not exists(imgDst):
self.copyOrLink(join(self._imgPath, fn), imgDst)
imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst))
if self._micIdOrName:
micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None)
micName = imgRow.getValue(md.MDL_MICROGRAPH, None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath(
'fake_micrograph%6d' % micId)
mic.setFileName(micName)
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId()))
# JMRT: This means that the metadata contains MDL_CTF_MODEL
# and the files path were found from some root
# In Xmipp 3.1 the ctfparam metadata in particles
# was replaced with directly seeting the CTF values
# so we need to fill those in the particle row
if self._ctfPath:
ctfModel = imgRow.getValue(md.MDL_CTF_MODEL)
if ctfModel in self._ctfDict:
ctfRow = self._ctfDict[ctfModel]
else:
ctfRow = md.Row()
ctfRow.readFromFile(join(self._ctfPath, ctfModel))
self._ctfDict[ctfModel] = ctfRow
imgRow.copyFromRow(ctfRow)
def _preprocessParticleRow(self, img, imgRow):
if self._imgPath:
# Create a link or copy files to extraPath
# and update the Row properly
index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE))
imgBase = basename(fn)
imgDst = self.protocol._getExtraPath(imgBase)
if not exists(imgDst):
self.copyOrLink(join(self._imgPath, fn), imgDst)
imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst))
if self._micIdOrName:
micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None)
micName = imgRow.getValue(md.MDL_MICROGRAPH, None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath('fake_micrograph%6d'
% micId)
mic.setFileName(micName)
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId()))
# JMRT: This means that the metadata contains MDL_CTF_MODEL
# and the files path were found from some root
# In Xmipp 3.1 the ctfparam metadata in particles
# was replaced with directly seeting the CTF values
# so we need to fill those in the particle row
if self._ctfPath:
ctfModel = imgRow.getValue(md.MDL_CTF_MODEL)
if ctfModel in self._ctfDict:
ctfRow = self._ctfDict[ctfModel]
else:
ctfRow = md.Row()
ctfRow.readFromFile(join(self._ctfPath, ctfModel))
self._ctfDict[ctfModel] = ctfRow
imgRow.copyFromRow(ctfRow)
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix,
'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
def _preprocessParticleRow(self, img, imgRow):
if self._imgPath:
# Create a link or copy files to extraPath
# and update the Row properly
index, fn = xmippToLocation(imgRow.getValue(md.MDL_IMAGE))
imgBase = basename(fn)
imgDst = self.protocol._getExtraPath(imgBase)
if not exists(imgDst):
self.copyOrLink(join(self._imgPath, fn), imgDst)
imgRow.setValue(md.MDL_IMAGE, locationToXmipp(index, imgDst))
if self._micIdOrName:
micId = imgRow.getValue(md.MDL_MICROGRAPH_ID, None)
micName = imgRow.getValue(md.MDL_MICROGRAPH, None)
# Check which is the key to identify micrographs (id or name)
if micId is not None:
micKey = micId
else:
micKey = micName
mic = self.micDict.get(micKey, None)
# First time I found this micrograph (either by id or name)
if mic is None:
mic = Micrograph()
mic.setObjId(micId)
if micName is None:
micName = self.protocol._getExtraPath('fake_micrograph%6d' % micId)
mic.setFileName(micName)
self.micSet.append(mic)
# Update dict with new Micrograph
self.micDict[micKey] = mic
# Update the row to set a MDL_MICROGRAPH_ID
imgRow.setValue(md.MDL_MICROGRAPH_ID, long(mic.getObjId()))
def testCtfdiscrepancyWorkflow(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
ctfSet2 = SetOfCTF(filename=fnCTF2)
# create one fake micrographs image
projSize = 32
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 1000+10*i
defocusV = 1000+i
defocusAngle = i*10
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU,
defocusV,
defocusAngle,
psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
defocusU = 1000+20*i
defocusV = 1000+i
defocusAngle = i*20
psdFile = "psd_2%04d" % i
ctf = self._getCTFModel(defocusU,
defocusV,
defocusAngle,
psdFile)
ctf.setMicrograph(mic)
ctfSet2.append(ctf)
ctfSet1.write()
ctfSet2.write()
micSet.write()
# import micrograph set
args = {'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF2 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF2)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
protCTF2.setObjLabel('import ctfs from scipion_2 ')
self.launchProtocol(protCTF1)
self.launchProtocol(protCTF2)
# launch CTF discrepancy protocol
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
protCtfDiscrepancy.setObjLabel('ctf discrepancy')
self.launchProtocol(protCtfDiscrepancy)
ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 1010)
def testCtfdiscrepancyWorkflow(self):
# create one micrograph set
fnMicSet = self.proj.getTmpPath("mics.sqlite")
fnMic = self.proj.getTmpPath("mic.mrc")
mic = Micrograph()
mic.setFileName(fnMic)
micSet = SetOfMicrographs(filename=fnMicSet)
# create two CTFsets
fnCTF1 = self.proj.getTmpPath("ctf1.sqlite")
fnCTF2 = self.proj.getTmpPath("ctf2.sqlite")
ctfSet1 = SetOfCTF(filename=fnCTF1)
ctfSet2 = SetOfCTF(filename=fnCTF2)
# create one fake micrographs image
projSize = 32
img = xmipp.Image()
img.setDataType(xmipp.DT_FLOAT)
img.resize(projSize, projSize)
img.write(fnMic)
# fill the sets
for i in range(1, 4):
mic = Micrograph()
mic.setFileName(fnMic)
micSet.append(mic)
defocusU = 1000 + 10 * i
defocusV = 1000 + i
defocusAngle = i * 10
psdFile = "psd_1%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet1.append(ctf)
defocusU = 1000 + 20 * i
defocusV = 1000 + i
defocusAngle = i * 20
psdFile = "psd_2%04d" % i
ctf = self._getCTFModel(defocusU, defocusV, defocusAngle, psdFile)
ctf.setMicrograph(mic)
ctfSet2.append(ctf)
ctfSet1.write()
ctfSet2.write()
micSet.write()
# import micrograph set
args = {
'importFrom': ProtImportMicrographs.IMPORT_FROM_SCIPION,
'sqliteFile': fnMicSet,
'amplitudConstrast': 0.1,
'sphericalAberration': 2.,
'voltage': 100,
'samplingRate': 2.1
}
protMicImport = self.newProtocol(ProtImportMicrographs, **args)
protMicImport.setObjLabel('import micrographs from sqlite ')
self.launchProtocol(protMicImport)
# import ctfsets
protCTF1 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF1)
protCTF2 = \
self.newProtocol(ProtImportCTF,
importFrom=ProtImportCTF.IMPORT_FROM_SCIPION,
filesPath=fnCTF2)
protCTF1.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF2.inputMicrographs.set(protMicImport.outputMicrographs)
protCTF1.setObjLabel('import ctfs from scipion_1 ')
protCTF2.setObjLabel('import ctfs from scipion_2 ')
self.launchProtocol(protCTF1)
self.launchProtocol(protCTF2)
# launch CTF discrepancy protocol
protCtfDiscrepancy = self.newProtocol(XmippProtCTFDiscrepancy)
protCtfDiscrepancy.inputCTF1.set(protCTF1.outputCTF)
protCtfDiscrepancy.inputCTF2.set(protCTF2.outputCTF)
protCtfDiscrepancy.setObjLabel('ctf discrepancy')
self.launchProtocol(protCtfDiscrepancy)
ctf0 = protCtfDiscrepancy.outputCTF.getFirstItem()
resolution = int(ctf0.getResolution())
defocusU = int(ctf0.getDefocusU())
self.assertEqual(resolution, 2)
self.assertEqual(defocusU, 1010)