def test_readDM4(self):
""" Check we can read dm4 files (using EMAN)
"""
micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')
ih = ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 1)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % (micFn, outFn)
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
# Check dimensions are still the same:
self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)
# Clean up tmp files
pwutils.cleanPath(outFn)
def volumeToNGL(volPath):
""" We need an mrc map to be loaded into the ngl viewer"""
from pyworkflow.em.convert import ImageHandler
# return getResource('reference.mrc')
# Clean volPath in case it comes with :mrc, ...
nglVolume = volPath.split(":")[0]
# If it ends in mrc
if nglVolume.endswith(".mrc"):
return nglVolume
else:
# Add mrc
nglVolume += ".mrc"
# If the ngl volume already exists
if os.path.exists(nglVolume):
return nglVolume
else:
# Convert input volumes
ih = ImageHandler()
ih.convert(volPath, nglVolume)
return nglVolume
def convertInputStep(self, particlesId, volId):
""" Write the input images as a Xmipp metadata file.
particlesId: is only need to detect changes in
input particles and cause restart from here.
"""
inputParticles = self.inputParticles.get()
inputVolume = self.inputVolume.get()
writeSetOfParticles(inputParticles, self._getExpParticlesFn())
img = ImageHandler()
img.convert(inputVolume, self._getInputVolFn())
if self._useSeveralClasses():
# Scale particles
Xdim = inputParticles.getXDim()
Ts = inputParticles.getSamplingRate()
newTs = self.targetResolution.get() * 0.4
newTs = max(Ts, newTs)
newXdim = Xdim * Ts / newTs
self.runJob("xmipp_image_resize",
"-i %s -o %s --save_metadata_stack %s --fourier %d" %
(self._getExpParticlesFn(),
self._getTmpPath('scaled_particles.stk'),
self._getTmpPath('scaled_particles.xmd'),
newXdim))
# Scale volume
Xdim = inputVolume.getXDim()
if Xdim != newXdim:
self.runJob("xmipp_image_resize", "-i %s --dim %d"
% (self._getInputVolFn(), newXdim), numberOfMpi=1)
def convertInputStep(self, particlesId, volId):
""" Write the input images as a Xmipp metadata file.
particlesId: is only need to detect changes in
input particles and cause restart from here.
"""
inputParticles = self.inputParticles.get()
inputVolume = self.inputVolume.get()
writeSetOfParticles(inputParticles, self._getExpParticlesFn())
img = ImageHandler()
img.convert(inputVolume, self._getInputVolFn())
if self._useSeveralClasses():
# Scale particles
Xdim = inputParticles.getXDim()
Ts = inputParticles.getSamplingRate()
newTs = self.targetResolution.get() * 0.4
newTs = max(Ts, newTs)
newXdim = long(Xdim * Ts / newTs)
self.writeInfoField(self._getExtraPath(), "sampling",
xmippLib.MDL_SAMPLINGRATE, newTs)
self.writeInfoField(self._getExtraPath(), "size", xmippLib.MDL_XSIZE,
newXdim)
self.runJob("xmipp_image_resize",
"-i %s -o %s --save_metadata_stack %s --fourier %d" %
(self._getExpParticlesFn(),
self._getTmpPath('scaled_particles.stk'),
self._getTmpPath('scaled_particles.xmd'),
newXdim))
# Scale volume
Xdim = inputVolume.getXDim()
if Xdim != newXdim:
self.runJob("xmipp_image_resize", "-i %s --dim %d"
% (self._getInputVolFn(), newXdim), numberOfMpi=1)
def test_readDM4(self):
""" Check we can read dm4 files (using EMAN)
"""
micFn = self.dsFormat.getFile('SuperRef_c3-adp-se-xyz-0228_001.dm4')
ih = ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 1)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % (micFn, outFn)
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
# Check dimensions are still the same:
self.assertEqual(ih.getDimensions(outFn), EXPECTED_SIZE)
# Clean up tmp files
pwutils.cleanPath(outFn)
def prepareMask(self,maskObject,fnMask,TsMaskOut,XdimOut):
img=ImageHandler()
img.convert(maskObject, fnMask)
self.runJob('xmipp_image_resize',"-i %s --factor %f"%(fnMask,maskObject.getSamplingRate()/TsMaskOut),numberOfMpi=1)
maskXdim, _, _, _ =img.getDimensions((1,fnMask))
if XdimOut!=maskXdim:
self.runJob('xmipp_transform_window',"-i %s --size %d"%(fnMask,XdimOut),numberOfMpi=1)
self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs):
""" Write a SetOfMicrograph as expected in EMX format
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
micSet: micrographs set associated with the particles
**kwargs: writeImages: if set to False, only coordinates are exported.
imagesStack: if passed all images will be output into a single
stack file.
imagesPrefix: used when not imagesStack is passed. A different
stack will be created per micrograph.
"""
writeImages = kwargs.get('writeImages', True)
imagesPrefix = kwargs.get('imagesPrefix', None)
micDict = {}
# Use singleMic for count all particles to be written to a single stack
imagesStack = kwargs.get('imagesStack', None)
singleMic = Micrograph()
singleMic.setFileName(imagesStack)
singleMic.counter = pwobj.Integer(0)
def _getMicKey(particle):
coord = particle.getCoordinate()
if coord is None or coord.getMicName() is None:
return '%05d' % particle.getMicId()
else:
return pwutils.removeExt(coord.getMicName())
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix,
'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
ih = ImageHandler()
partAlign = partSet.getAlignment()
for particle in partSet:
if writeImages:
newLoc = _getLocation(particle)
ih.convert(particle, newLoc)
localFn = basename(newLoc[1])
particle.setLocation(newLoc[0], localFn)
emxObj = _particleToEmx(emxData, particle, micSet, partAlign)
else:
emxObj = _coordinateToEmx(emxData, particle, micSet)
emxData.addObject(emxObj)
def projectInitialVolume(self):
fnOutputInitVolume=self._getTmpPath("initialVolume.vol")
img = ImageHandler()
img.convert(self.initialVolume.get(), fnOutputInitVolume)
self.runJob("xmipp_image_resize","-i %s --dim %d %d"%(fnOutputInitVolume,self.Xdim2,self.Xdim2))
fnGallery=self._getTmpPath('gallery_InitialVolume.stk')
fnOutputReducedClass = self._getExtraPath("reducedClasses.xmd")
self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\
%(fnOutputInitVolume,fnGallery,self.angularSampling.get(),self.symmetryGroup.get(),fnOutputReducedClass))
def _runBeforePreWhitening(self):
prot = self.form.protocol
# Convert input volumes
ih = ImageHandler()
ih.convert(prot.inputVolume.get(), join(self.workingDir, 'volume1.map'))
if prot.useSplitVolume:
ih.convert(prot.splitVolume.get(), join(self.workingDir, 'volume2.map'))
self.results = prot.runResmap(self.workingDir, wizardMode=True)
def _particlesToEmx(emxData, partSet, micSet=None, **kwargs):
""" Write a SetOfMicrograph as expected in EMX format
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
micSet: micrographs set associated with the particles
**kwargs: writeImages: if set to False, only coordinates are exported.
imagesStack: if passed all images will be output into a single
stack file.
imagesPrefix: used when not imagesStack is passed. A different
stack will be created per micrograph.
"""
writeImages = kwargs.get('writeImages', True)
imagesPrefix = kwargs.get('imagesPrefix', None)
micDict = {}
# Use singleMic for count all particles to be written to a single stack
imagesStack = kwargs.get('imagesStack', None)
singleMic = Micrograph()
singleMic.setFileName(imagesStack)
singleMic.counter = pwobj.Integer(0)
def _getMicKey(particle):
coord = particle.getCoordinate()
if coord is None or coord.getMicName() is None:
return '%05d' % particle.getMicId()
else:
return pwutils.removeExt(coord.getMicName())
def _getLocation(particle):
if imagesStack is not None:
mic = singleMic
else:
micKey = _getMicKey(particle)
if micKey not in micDict:
mic = Micrograph()
mic.setFileName(join(imagesPrefix, 'particles_%s.mrc' % micKey))
mic.counter = pwobj.Integer(0)
micDict[micKey] = mic
else:
mic = micDict[micKey]
# Count one more particle assigned to this micrograph
mic.counter.increment()
return (mic.counter.get(), mic.getFileName())
ih = ImageHandler()
partAlign = partSet.getAlignment()
for particle in partSet:
if writeImages:
newLoc = _getLocation(particle)
ih.convert(particle, newLoc)
localFn = basename(newLoc[1])
particle.setLocation(newLoc[0], localFn)
emxObj = _particleToEmx(emxData, particle, micSet, partAlign)
else:
emxObj = _coordinateToEmx(emxData, particle, micSet)
emxData.addObject(emxObj)
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def exportVolumeStep(self):
#create directory if needed
dirName = self.filesPath.get()
try:
os.makedirs(dirName)
except OSError:
if not os.path.isdir(dirName):
raise
ih = ImageHandler()
ih.convert(self.exportVolume.get().getLocation(),
os.path.join(dirName, self.VOLUMENAME))
def projectInitialVolume(self):
fnOutputInitVolume = self._getTmpPath("initialVolume.vol")
img = ImageHandler()
img.convert(self.initialVolume.get(), fnOutputInitVolume)
self.runJob(
"xmipp_image_resize",
"-i %s --dim %d %d" % (fnOutputInitVolume, self.Xdim2, self.Xdim2))
fnGallery = self._getTmpPath('gallery_InitialVolume.stk')
fnOutputReducedClass = self._getExtraPath("reducedClasses.xmd")
self.runJob("xmipp_angular_project_library", "-i %s -o %s --sampling_rate %f --sym %s --method fourier 1 0.25 bspline --compute_neighbors --angular_distance -1 --experimental_images %s"\
%(fnOutputInitVolume,fnGallery,self.angularSampling.get(),self.symmetryGroup.get(),fnOutputReducedClass))
def convertInputStep(self, particlesId):
""" Write the input images as a Xmipp metadata file.
particlesId: is only need to detect changes in
input particles and cause restart from here.
"""
writeSetOfParticles(self.inputParticles.get(),
self._getPath('input_particles.xmd'))
if self.doWiener.get():
params = ' -i %s' % self._getPath('input_particles.xmd')
params += ' -o %s' % self._getExtraPath(
'corrected_ctf_particles.stk')
params += ' --save_metadata_stack %s' % self._getExtraPath(
'corrected_ctf_particles.xmd')
params += ' --pad %s' % self.padding_factor.get()
params += ' --wc %s' % self.wiener_constant.get()
params += ' --sampling_rate %s' % self.inputParticles.get(
).getSamplingRate()
if self.inputParticles.get().isPhaseFlipped():
params += ' --phase_flipped '
if self.correctEnvelope:
params += ' --correct_envelope '
nproc = self.numberOfMpi.get()
nT = self.numberOfThreads.get()
self.runJob('xmipp_ctf_correct_wiener2d', params)
newTs, newXdim = self._getModifiedSizeAndSampling()
if self.doWiener.get():
params = ' -i %s' % self._getExtraPath(
'corrected_ctf_particles.xmd')
else:
params = ' -i %s' % self._getPath('input_particles.xmd')
params += ' -o %s' % self._getExtraPath('scaled_particles.stk')
params += ' --save_metadata_stack %s' % self._getExtraPath(
'scaled_particles.xmd')
params += ' --dim %d' % newXdim
self.runJob('xmipp_image_resize', params)
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
img.convert(self.inputVolumes.get(), self._getExtraPath("volume.vol"))
Xdim = self.inputVolumes.get().getDim()[0]
if Xdim != newXdim:
self.runJob("xmipp_image_resize","-i %s --dim %d"%\
(self._getExtraPath("volume.vol"),
newXdim), numberOfMpi=1)
def _beforePreWhitening(protocol, dir):
from pyworkflow.em.convert import ImageHandler
# Convert input volumes
ih = ImageHandler()
inputVolume = protocol.inputVolume.get()
path = join(dir, 'volume1.map')
print path
ih.convert(inputVolume, path)
if protocol.useSplitVolume:
ih.convert(protocol.splitVolume.get(), join(dir, 'volume2.map'))
return protocol.runResmap(dir, wizardMode=True)
def _beforePreWhitening(protocol, dir):
from pyworkflow.em.convert import ImageHandler
# Convert input volumes
ih = ImageHandler()
inputVolume = protocol.inputVolume.get()
path = join(dir, 'volume1.map')
print path
ih.convert(inputVolume, path)
if protocol.useSplitVolume:
ih.convert(protocol.splitVolume.get(), join(dir, 'volume2.map'))
return protocol.runResmap(dir, wizardMode=True)
def convertInputStep(self, particlesId):
""" Write the input images as a Xmipp metadata file.
particlesId: is only need to detect changes in
input particles and cause restart from here.
"""
writeSetOfParticles(self.inputParticles.get(),
self._getPath('input_particles.xmd'))
if self.doWiener.get():
params = ' -i %s' % self._getPath('input_particles.xmd')
params += ' -o %s' % self._getExtraPath('corrected_ctf_particles.stk')
params += ' --save_metadata_stack %s' % self._getExtraPath('corrected_ctf_particles.xmd')
params += ' --pad %s' % self.padding_factor.get()
params += ' --wc %s' % self.wiener_constant.get()
params += ' --sampling_rate %s' % self.inputParticles.get().getSamplingRate()
if self.inputParticles.get().isPhaseFlipped():
params += ' --phase_flipped '
if self.correctEnvelope:
params += ' --correct_envelope '
nproc = self.numberOfMpi.get()
nT=self.numberOfThreads.get()
self.runJob('xmipp_ctf_correct_wiener2d',
params)
newTs, newXdim = self._getModifiedSizeAndSampling()
if self.doWiener.get():
params = ' -i %s' % self._getExtraPath('corrected_ctf_particles.xmd')
else :
params = ' -i %s' % self._getPath('input_particles.xmd')
params += ' -o %s' % self._getExtraPath('scaled_particles.stk')
params += ' --save_metadata_stack %s' % self._getExtraPath('scaled_particles.xmd')
params += ' --fourier %d' % newXdim
self.runJob('xmipp_image_resize',params)
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
img.convert(self.inputVolumes.get(), self._getExtraPath("volume.vol"))
Xdim = self.inputVolumes.get().getDim()[0]
if Xdim!=newXdim:
self.runJob("xmipp_image_resize","-i %s --dim %d"%\
(self._getExtraPath("volume.vol"),
newXdim), numberOfMpi=1)
def convertStep(self, imgsFn):
from convert import writeSetOfClasses2D, writeSetOfParticles
imgSet = self.inputSet.get()
if isinstance(imgSet, SetOfClasses2D):
writeSetOfClasses2D(imgSet, self.imgsFn, writeParticles=True)
else:
writeSetOfParticles(imgSet, self.imgsFn)
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
fnVol = self._getTmpPath("volume.vol")
img.convert(self.inputVolume.get(), fnVol)
xdim=self.inputVolume.get().getDim()[0]
if xdim!=self._getDimensions():
self.runJob("xmipp_image_resize","-i %s --dim %d"%(fnVol,self._getDimensions()))
def convertStep(self, imgsFn):
from ..convert import writeSetOfClasses2D, writeSetOfParticles
imgSet = self.inputSet.get()
if isinstance(imgSet, SetOfClasses2D):
writeSetOfClasses2D(imgSet, self.imgsFn, writeParticles=True)
else:
writeSetOfParticles(imgSet, self.imgsFn)
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
fnVol = self._getTmpPath("volume.vol")
img.convert(self.inputVolume.get(), fnVol)
xdim=self.inputVolume.get().getDim()[0]
if xdim!=self._getDimensions():
self.runJob("xmipp_image_resize","-i %s --dim %d"%(fnVol,self._getDimensions()),numberOfMpi=1)
def prepareMask(self, maskObject, fnMask, TsMaskOut, XdimOut):
img = ImageHandler()
img.convert(maskObject, fnMask)
self.runJob('xmipp_image_resize',
"-i %s --factor %f" %
(fnMask, maskObject.getSamplingRate() / TsMaskOut),
numberOfMpi=1)
maskXdim, _, _, _ = img.getDimensions((1, fnMask))
if XdimOut != maskXdim:
self.runJob('xmipp_transform_window',
"-i %s --size %d" % (fnMask, XdimOut),
numberOfMpi=1)
self.runJob('xmipp_transform_threshold',
"-i %s --select below 0.5 --substitute binarize" % fnMask,
numberOfMpi=1)
def convertStep(self, imgsFn):
from xmipp3.convert import writeSetOfParticles
imgSet = self.inputSet.get()
writeSetOfParticles(imgSet, self.imgsFn)
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
fnVol = self._getTmpPath("volume.vol")
img.convert(self.inputVolume.get(), fnVol)
xdim = self.inputVolume.get().getDim()[0]
imgXdim = imgSet.getDim()[0]
if xdim != imgXdim:
self.runJob("xmipp_image_resize",
"-i %s --dim %d" % (fnVol, imgXdim),
numberOfMpi=1)
def convertStep(self):
inputParts = self.inputParticles.get()
writeSetOfParticles(inputParts, self.imgsInputFn)
#Resizing inputs
ih = ImageHandler()
ih.convert(self.inputReference.get(), self.fnRefVol)
XdimRef = self.inputReference.get().getDim()[0]
ih.convert(self.inputVolume.get(), self.fnInputVol)
XdimInput = self.inputVolume.get().getDim()[0]
if XdimRef != XdimInput:
self.runJob("xmipp_image_resize",
"-i %s --dim %d" % (self.fnRefVol, XdimInput),
numberOfMpi=1)
def prime2DStep(self):
partFile = self._getExtraPath("particles.mrc")
SamplingRate = self.inputParticles.get().getSamplingRate()
kV = self.inputParticles.get().getAcquisition().getVoltage()
partitions = 1
partName = os.path.basename(partFile)
partName = os.path.splitext(partName)[0]
tmpDir = self._getTmpPath(partName)
makePath(tmpDir)
paramsOri = 'prg=print_project_field oritype=ptcl2D > oritab.txt'
paramsImp = 'prg=import_particles cs=2.7 ctf=no fraca=0.1 kv=%f smpd=%f stk=%s' % (
kV, SamplingRate, os.path.abspath(partFile))
paramsC2D = ' prg=cluster2D msk=%d ncls=%d nparts=%d nthr=%d' % (
self.mask.get(), self.clusters.get(), partitions,
self.numberOfThreads.get())
if self.maxIter.get() > 0:
paramsC2D = paramsC2D + (' maxits=%d' % self.maxIter.get())
self.runJob(simple.Plugin.sim_exec(),
'prg=new_project projname=temp',
cwd=os.path.abspath(tmpDir),
env=simple.Plugin.getEnviron())
self.runJob(simple.Plugin.sim_exec(),
paramsImp,
cwd=os.path.abspath(tmpDir) + '/temp',
env=simple.Plugin.getEnviron())
self.runJob(simple.Plugin.distr_exec(),
paramsC2D,
cwd=os.path.abspath(tmpDir) + '/temp',
env=simple.Plugin.getEnviron())
self.runJob(simple.Plugin.sim_exec(),
paramsOri,
cwd=os.path.abspath(tmpDir) + '/temp',
env=simple.Plugin.getEnviron())
#Move output files to ExtraPath and rename them properly
lastIter = self.getLastIteration(tmpDir)
os.remove(os.path.abspath(self._getExtraPath("particles.mrc")))
mvRoot1 = os.path.join(tmpDir + '/temp/2_cluster2D',
"cavgs_iter%03d.mrc" % lastIter)
mvRoot2 = os.path.join(tmpDir + '/temp', "oritab.txt")
# moveFile(mvRoot1, self._getExtraPath(partName + "_cavgs_final.mrc"))
ih = ImageHandler()
ih.convert(mvRoot1, self._getExtraPath(partName + "_cavgs_final.mrcs"))
moveFile(mvRoot2, self._getExtraPath(partName + "_oritab.txt"))
cleanPath(tmpDir)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Armstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def _insertAllSteps(self):
self._params = {}
# diameter must be passed in Angstrongs and therefore should be converted
self._params['diam'] = self.diameter.get() * self.getInputMicrographs().getSamplingRate()
# self._params['num-slices'] = self.numberSizes.get()
# self._params['size-range'] = self.sizeRange.get()
self._params['apix'] = self.inputMicrographs.get().getSamplingRate()
self._params['thresh'] = self.threshold.get()
# self._params['max-thresh'] = self.maxThreshold.get()
# self._params['max-area'] = self.maxArea.get()
# self._params['max-peaks'] = self.maxPeaks.get()
args = ""
for par, val in self._params.iteritems():
args += " --%s=%s" % (par, str(val))
if self.invert:
args += " --invert"
args += " " + self.extraParams.get('')
deps = [] # Store all steps ids, final step createOutput depends on all of them
ih = ImageHandler()
for mic in self.inputMicrographs.get():
# Create micrograph folder
micName = mic.getFileName()
micDir = self._getTmpPath(removeBaseExt(micName))
makePath(micDir)
# If needed convert micrograph to mrc format, otherwise link it
if getExt(micName) != ".mrc":
fnMicBase = replaceBaseExt(micName, 'mrc')
inputMic = join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = join(micDir, basename(micName))
createLink(micName, inputMic)
# Insert step to execute program
stepId = self._insertFunctionStep('executeDogpickerStep', inputMic, args)
deps.append(stepId)
self._insertFinalSteps(deps)
def generateSplittedVolumes(self):
inputParticles = self.directionalClasses.get()
Xdim = inputParticles.getDimensions()[0]
fnMask = ""
if self.mask.hasValue():
fnMask = self._getExtraPath("mask.vol")
img=ImageHandler()
img.convert(self.mask.get(), fnMask)
self.runJob('xmipp_image_resize',"-i %s --dim %d"%(fnMask,Xdim),numberOfMpi=1)
self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
args="-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f"%\
(self._getExtraPath("directionalClasses.xmd"),self._getExtraPath("split"),self.Nrec.get(),self.Nsamples.get(),
self.symmetryGroup.get(), self.alpha.get())
if fnMask!="":
args+=" --mask binary_file %s"%fnMask
self.runJob("xmipp_classify_first_split",args)
def generateSplittedVolumes(self):
inputParticles = self.directionalClasses.get()
Xdim = inputParticles.getDimensions()[0]
fnMask = ""
if self.mask.hasValue():
fnMask = self._getExtraPath("mask.vol")
img=ImageHandler()
img.convert(self.mask.get(), fnMask)
self.runJob('xmipp_image_resize',"-i %s --dim %d"%(fnMask,Xdim),numberOfMpi=1)
self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
args="-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f"%\
(self._getExtraPath("directionalClasses.xmd"),self._getExtraPath("split"),self.Nrec.get(),self.Nsamples.get(),
self.symmetryGroup.get(), self.alpha.get())
if fnMask!="":
args+=" --mask binary_file %s"%fnMask
self.runJob("xmipp_classify_first_split",args)
def splitVolumeStep(self):
newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE)
newXdim = self.readInfoField(self._getExtraPath(),"size",xmippLib.MDL_XSIZE)
fnMask = ""
if self.mask.hasValue():
fnMask = self._getExtraPath("mask.vol")
img=ImageHandler()
img.convert(self.mask.get(), fnMask)
self.runJob('xmipp_image_resize',"-i %s --dim %d"%(fnMask,newXdim),numberOfMpi=1)
self.runJob('xmipp_transform_threshold',"-i %s --select below 0.5 --substitute binarize"%fnMask,numberOfMpi=1)
args="-i %s --oroot %s --Nrec %d --Nsamples %d --sym %s --alpha %f"%\
(self._getDirectionalClassesFn(),self._getExtraPath("split"),self.Nrec.get(),self.Nsamples.get(),
self.symmetryGroup.get(), self.alpha.get())
if fnMask!="":
args+=" --mask binary_file %s"%fnMask
self.runJob("xmipp_classify_first_split",args,numberOfMpi=1)
def writeSetOfImages(imgSet, stackFn, selFn):
""" This function will write a SetOfMicrographs as a Spider stack and selfile.
Params:
imgSet: the SetOfMicrograph instance.
stackFn: the filename where to write the stack.
selFn: the filename of the Spider selection file.
"""
ih = ImageHandler()
doc = SpiderDocFile(selFn, 'w+')
for i, img in enumerate(imgSet):
ih.convert(img, (i+1, stackFn))
doc.writeValues(i+1)
doc.close()
convertEndian(stackFn, imgSet.getSize())
def prepareReferences(self,fnDir,TsCurrent,targetResolution):
print "Preparing references to sampling rate=",TsCurrent
fnMask=''
newXdim=self.readInfoField(fnDir,"size",xmipp.MDL_XSIZE)
if self.nextMask.hasValue():
fnMask=join(fnDir,"mask.vol")
self.prepareMask(self.nextMask.get(), fnMask, TsCurrent, newXdim)
fnReferenceVol=join(fnDir,"volumeRef.vol")
img=ImageHandler()
img.convert(self.inputVolume.get(), fnReferenceVol)
Xdim=self.inputVolume.get().getDim()[0]
if Xdim!=newXdim:
self.runJob("xmipp_image_resize","-i %s --fourier %d"%(fnReferenceVol,newXdim),numberOfMpi=1)
if fnMask!='':
self.runJob('xmipp_image_operate','-i %s --mult %s'%(fnReferenceVol,fnMask),numberOfMpi=1)
def test_convertMicrographs(self):
""" Convert micrograhs to different formats.
EMAN2 required for .img
"""
micFn = self.dataset.getFile('micrographs/BPV_1386.mrc')
outSuffix = pwutils.replaceBaseExt(micFn, 'img')
ih = ImageHandler()
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % (micFn, outFn)
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
pwutils.cleanPath(outFn)
pwutils.cleanPath(outFn.replace('.img', '.hed'))
def test_convertMicrographs(self):
""" Convert micrograhs to different formats.
EMAN2 required for .img
"""
micFn = self.dataset.getFile('micrographs/BPV_1386.mrc')
outSuffix = pwutils.replaceBaseExt(micFn, 'img')
ih = ImageHandler()
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % (micFn, outFn)
ih.convert(micFn, outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
pwutils.cleanPath(outFn)
pwutils.cleanPath(outFn.replace('.img', '.hed'))
def _micrographsToEmx(emxData, micSet, emxDir, ctfSet=None, writeData=True):
""" Write a SetOfMicrograph as expected in EMX format (xml file)
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
"""
ih = ImageHandler()
for mic in micSet:
if writeData:
loc = mic.getLocation()
fnMicBase = pwutils.replaceBaseExt(loc[1], 'mrc')
newLoc = join(emxDir, fnMicBase)
ih.convert(loc, newLoc)
mic.setLocation(NO_INDEX, fnMicBase)
if ctfSet:
mic.setCTF(ctfSet[mic.getObjId()])
emxMic = _micrographToEmx(mic)
emxData.addObject(emxMic)
def _micrographsToEmx(emxData, micSet, emxDir, ctfSet=None, writeData=True):
""" Write a SetOfMicrograph as expected in EMX format (xml file)
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
"""
ih = ImageHandler()
for mic in micSet:
if writeData:
loc = mic.getLocation()
fnMicBase = pwutils.replaceBaseExt(loc[1], 'mrc')
newLoc = join(emxDir, fnMicBase)
ih.convert(loc, newLoc)
mic.setLocation(NO_INDEX, fnMicBase)
if ctfSet:
mic.setCTF(ctfSet[mic.getObjId()])
emxMic = _micrographToEmx(mic)
emxData.addObject(emxMic)
def _particlesToEmx(emxData, partSet, stackFn=None, micSet=None):
""" Write a SetOfMicrograph as expected in EMX format
Params:
micSet: input set of micrographs
filename: the EMX file where to store the micrographs information.
"""
ih = ImageHandler()
partAlign = partSet.getAlignment()
for i, particle in enumerate(partSet):
if stackFn:
print ("stackFn-----------------------------------------")
loc = particle.getLocation()
newLoc = (i+1, stackFn)
ih.convert(loc, newLoc)
newFn = basename(stackFn)
particle.setLocation(i+1, newFn)
emxObj = _particleToEmx(emxData, particle, micSet, partAlign)
else:
emxObj = _coordinateToEmx(emxData, particle, micSet)
emxData.addObject(emxObj)
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
inputPath = os.path.join("Tmp", "bsoft_filter_input.spi")
outputPath = os.path.join("Tmp", "bsoft_filter_output.spi")
cleanPath(inputPath, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), inputPath)
self.protocolParent.runFilter(inputPath, outputPath)
# Get output image and update filtered image
img = ih._img
img.read(outputPath)
self.rightImage = img
self.updateFilteredImage()
def prepareReferences(self, fnDir, TsCurrent, targetResolution):
print "Preparing references to sampling rate=", TsCurrent
fnMask = ''
newXdim = self.readInfoField(fnDir, "size", xmipp.MDL_XSIZE)
if self.nextMask.hasValue():
fnMask = join(fnDir, "mask.vol")
self.prepareMask(self.nextMask.get(), fnMask, TsCurrent, newXdim)
fnReferenceVol = join(fnDir, "volumeRef.vol")
img = ImageHandler()
img.convert(self.inputVolume.get(), fnReferenceVol)
Xdim = self.inputVolume.get().getDim()[0]
if Xdim != newXdim:
self.runJob("xmipp_image_resize",
"-i %s --fourier %d" % (fnReferenceVol, newXdim),
numberOfMpi=1)
if fnMask != '':
self.runJob('xmipp_image_operate',
'-i %s --mult %s' % (fnReferenceVol, fnMask),
numberOfMpi=1)
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
inputPath = os.path.join("Tmp", "bsoft_filter_input.spi")
outputPath = os.path.join("Tmp", "bsoft_filter_output.spi")
cleanPath(inputPath, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), inputPath)
self.protocolParent.runFilter(inputPath, outputPath)
# Get output image and update filtered image
img = ih._img
img.read(outputPath)
self.rightImage = img
self.updateFilteredImage()
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
outputName = os.path.join("Tmp", "filtered_particle")
outputPath = outputName + ".spi"
cleanPath(outputPath)
outputLoc = (1, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), outputLoc)
outputLocSpiStr = locationToSpider(1, outputName)
pars = {}
pars["filterType"] = self.protocolParent.filterType.get()
pars["filterMode"] = self.protocolParent.filterMode.get()
pars["usePadding"] = self.protocolParent.usePadding.get()
pars["op"] = "FQ"
if self.protocolParent.filterType <= FILTER_SPACE_REAL:
pars['filterRadius'] = self.getRadius()
else:
pars['lowFreq'] = self.getLowFreq()
pars['highFreq'] = self.getHighFreq()
if self.protocolParent.filterType == FILTER_FERMI:
pars['temperature'] = self.getTemperature()
filter_spider(outputLocSpiStr, outputLocSpiStr, **pars)
# Get output image and update filtered image
img = ImageHandler()._img
locXmippStr = locationToXmipp(1, outputPath)
img.read(locXmippStr)
self.rightImage = img
self.updateFilteredImage()
def _computeRightPreview(self):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
from pyworkflow.em.packages.xmipp3 import locationToXmipp
# Copy image to filter to Tmp project folder
outputName = os.path.join("Tmp", "filtered_particle")
outputPath = outputName + ".spi"
cleanPath(outputPath)
outputLoc = (1, outputPath)
ih = ImageHandler()
ih.convert(self.lastObj.getLocation(), outputLoc)
outputLocSpiStr = locationToSpider(1, outputName)
pars = {}
pars["filterType"] = self.protocolParent.filterType.get()
pars["filterMode"] = self.protocolParent.filterMode.get()
pars["usePadding"] = self.protocolParent.usePadding.get()
pars["op"] = "FQ"
if self.protocolParent.filterType <= FILTER_FERMI:
pars['filterRadius'] = self.getRadius()
else:
pars['lowFreq'] = self.getLowFreq()
pars['highFreq'] = self.getHighFreq()
if self.protocolParent.filterType == FILTER_FERMI:
pars['temperature'] = self.getTemperature()
filter_spider(outputLocSpiStr, outputLocSpiStr, **pars)
# Get output image and update filtered image
img = ImageHandler()._img
locXmippStr = locationToXmipp(1, outputPath)
img.read(locXmippStr)
self.rightImage = img
self.updateFilteredImage()
def _pickMicrograph(self, mic, radius):
micFn = mic.getFileName()
micDir = self._getMicDir(micFn)
fnMicBase = pwutils.replaceBaseExt(micFn, 'mrc')
fnMicCfg = pwutils.replaceBaseExt(micFn, 'cfg')
fnMicFull = os.path.join(micDir, fnMicBase)
fnPosBase = self._getMicPosFn(micFn)
# Convert micrographs to mrc (uint8) as required by ETHAN program
ih = ImageHandler()
ih.convert(micFn, fnMicFull, md.DT_UCHAR)
# Create a configuration file to be used by ETHAN with the parameters
# selected by the user
self.writeConfigFile(os.path.join(micDir, fnMicCfg))
# Run ethan program with the required arguments
program = self.getProgram()
args = "%s %s %s %s" % (radius, fnMicBase, fnPosBase, fnMicCfg)
self.runJob(program, args, cwd=micDir)
# Clean temporary micrograph
pwutils.cleanPath(fnMicFull)
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def test_readCompressedTIF(self):
""" Check we can read tif files
"""
micFn = self.dsFormat.getFile('c3-adp-se-xyz-0228_200.tif')
ih = ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 38)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % ((1, micFn), outFn)
ih.convert((1, micFn), outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
self.assertEqual(ih.getDimensions(outFn), (7676, 7420, 1, 1))
# Clean up tmp files
pwutils.cleanPath(outFn)
def _pickMicrograph(self, mic, args):
# Prepare mic folder and convert if needed
micName = mic.getFileName()
micDir = self._getTmpPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
ih = ImageHandler()
# If needed convert micrograph to mrc format, otherwise link it
if pwutils.getExt(micName) != ".mrc":
fnMicBase = pwutils.replaceBaseExt(micName, 'mrc')
inputMic = os.path.join(micDir, fnMicBase)
ih.convert(mic.getLocation(), inputMic)
else:
inputMic = os.path.join(micDir, os.path.basename(micName))
pwutils.createLink(micName, inputMic)
# Program to execute and it arguments
program = "ApDogPicker.py"
outputFile = self._getExtraPath(pwutils.replaceBaseExt(
inputMic, "txt"))
args += " --image=%s --outfile=%s" % (inputMic, outputFile)
self.runJob(program, args)
def test_readCompressedTIF(self):
""" Check we can read tif files
"""
micFn = self.dsFormat.getFile('c3-adp-se-xyz-0228_200.tif')
ih = ImageHandler()
# Check that we can read the dimensions of the dm4 file:
EXPECTED_SIZE = (7676, 7420, 1, 38)
self.assertEqual(ih.getDimensions(micFn), EXPECTED_SIZE)
# We could even convert to an mrc file:
outSuffix = pwutils.replaceBaseExt(micFn, 'mrc')
outFn = join('/tmp', outSuffix)
print "Converting: \n%s -> %s" % ((1, micFn), outFn)
ih.convert((1, micFn), outFn)
self.assertTrue(os.path.exists(outFn))
self.assertTrue(pwutils.getFileSize(outFn) > 0)
self.assertEqual(ih.getDimensions(outFn), (7676, 7420, 1, 1))
# Clean up tmp files
pwutils.cleanPath(outFn)
def _runBeforePreWhitening(self):
prot = self.form.protocol
# Convert input volumes
ih = ImageHandler()
if prot.useSplitVolume:
ih.convert(prot.volumeHalf1.get(),
join(self.workingDir, 'volume1.map'))
ih.convert(prot.volumeHalf2.get(),
join(self.workingDir, 'volume2.map'))
else:
ih.convert(prot.inputVolume.get(),
join(self.workingDir, 'volume1.map'))
self.results = prot.runResmap(self.workingDir, wizardMode=True)
class CustomMaskDialog(ImagePreviewDialog):
def _beforePreview(self):
imgLocation = self.protocolParent.inputImage.get().getLocation()
self.dim = ImageHandler().getDimensions(imgLocation)[0]
self.lastObj = None
self.rightPreviewLabel = "Final mask"
self.message = "Generating mask..."
self.ih = ImageHandler()
self.rightImage = self.ih.createImage()
def _createPreview(self, frame):
""" Should be implemented by subclasses to
create the items preview.
"""
self._previews = []
for i, label in enumerate(MASKRESULT_LABELS):
self.previewLabel = label
previewFrame = tk.Frame(frame)
ImagePreviewDialog._createPreview(self, previewFrame)
self._previews.append(self.preview) # store all previews created
previewFrame.grid(row=i/4, column=i%4)
def _itemSelected(self, obj):
self.lastObj = obj
dialog.FlashMessage(self, self.message, func=self._computeRightPreview)
def _createVarWidgets(self, parent, varName, varLabel, row, col):
var = tk.StringVar()
self._vars[varName] = var
var.set(self.protocolParent.getAttributeValue(varName))
varLabel = tk.Label(parent, text=varLabel)
varLabel.grid(row=row, column=col*2, padx=5, pady=5)
varEntry = tk.Entry(parent, width=10, textvariable=var)
varEntry.grid(row=row, column=col*2+1, padx=5, pady=5)
def _createControls(self, frame):
self._vars = {}
inputFrame = tk.Frame(frame)
inputFrame.grid(row=0, column=0)
for i, varName in enumerate(CUSTOMMASK_VARS):
self._createVarWidgets(inputFrame, varName, CUSTOMMASK_VARS[varName], i%2, i/2)
previewBtn = HotButton(frame, text='Preview', command=self._computeRightPreview)
previewBtn.grid(row=1, column=1, padx=5, pady=5)
def getVarValue(self, varName):
return self._vars[varName].get()
def _computeRightPreview(self, e=None):
""" This function should compute the right preview
using the self.lastObj that was selected
"""
prot = self.protocolParent # short notation
tmp = prot.getProject().getTmpPath()
ext = prot.getExt()
# Convert input image to spider
imgPrefix = 'inputImage'
imgName = '%s.%s' % (imgPrefix, ext)
imgFn = os.path.join(tmp, imgName)
self.ih.convert(self.lastObj, (1, imgFn))
runCustomMaskScript(self.getVarValue('filterRadius1'),
self.getVarValue('sdFactor'),
self.getVarValue('filterRadius2'),
self.getVarValue('maskThreshold'),
workingDir=tmp, ext=ext,
inputImage=imgPrefix+'@1')
for i, preview in enumerate(self._previews):
if i == 0:
self.rightImage.read(imgFn)
else:
self.rightImage.read('%d@%s/stkmask.%s' % (i, tmp, ext))
preview.updateData(self.rightImage.getData())
class ProtRelion2Autopick(ProtParticlePickingAuto, ProtRelionBase):
"""
This Relion protocol uses the 'relion_autopick' program to pick particles
from micrographs, either using templates or gaussian blobs.
The picking with this protocol is divided in three steps:
1) Run with 'Optimize' option for several (less than 30) micrographs.
2) Execute the wizard to refine the picking parameters.
3) Run with 'Pick all' option to pick particles from all micrographs.
The first steps will use internally the option '--write-fom-maps' to write
to disk the FOM maps. The expensive part of this calculation is to calculate
a probability-based figure-of-merit (related to the cross-correlation
coefficient between each rotated reference and all positions in the
micrographs. That's why it is only done in an small subset of the
micrographs, where one should use representative micrographs for the entire
data set, e.g. a high and a low-defocus one, and/or with thin or thick ice.
Step 2 uses a much cheaper peak-detection algorithm that uses the threshold
and minimum distance parameters.
"""
_label = 'auto-picking'
@classmethod
def isDisabled(cls):
return not isVersion2()
# -------------------------- DEFINE param functions ------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMicrographs', params.PointerParam,
pointerClass='SetOfMicrographs',
label='Input micrographs', important=True,
help='Select the input micrographs. '
'If using the *Optimize* mode, just a subset of '
'micrographs are used to compute the FOM maps. '
'If in *Compute* mode, all micrographs will be '
'auto-picked.')
form.addParam('ctfRelations', params.RelationParam,
relationName=RELATION_CTF,
attributeName='getInputMicrographs',
label='CTF estimation',
help='Choose some CTF estimation related to the '
'input micrographs.')
form.addParam('runType', params.EnumParam, default=RUN_OPTIMIZE,
choices=['Optimize params', 'Pick all micrographs'],
display=params.EnumParam.DISPLAY_LIST,
label='Run type: ',
help='Usually, first you should use the *Optimize* mode '
'to compute the FOM maps for a few micrographs and '
'use them to tune the picking parameters using the '
'wizard. After that you can run the job in *Compute*'
' mode and auto-pick all the micrographs. ')
group = form.addGroup('Micrographs for optimization',
condition='runType==%d' % RUN_OPTIMIZE)
group.addParam('micrographsSelection', params.EnumParam,
default=MICS_AUTO,
choices=['automatic selection', 'input subset'],
display=params.EnumParam.DISPLAY_HLIST,
label='Choose micrographs by',
help='If you choose "automatic selection", you only '
'need to provide the number of microgrphs to use '
'and that number will be selected to cover the '
'defocus range. ')
group.addParam('micrographsNumber', params.IntParam, default='10',
condition='micrographsSelection==%d' % MICS_AUTO,
label='Micrographs for optimization:',
help='Select the number of micrographs that you want'
'to be used for the parameters optimization. ')
group.addParam('micrographsSubset', params.PointerParam,
condition='micrographsSelection==%d' % MICS_SUBSET,
pointerClass='SetOfMicrographs',
label='Subset of micrographs',
help='Choose as input a subset of micrographs that '
'you have previously selected. '
'(Probably covering the defocus range).')
# From Relion 2.+, it can be picked with gaussian blobs, so we
# need to add these parameters
refCondition = 'referencesType==%s' % REF_AVERAGES
group = form.addGroup('References')
group.addParam('referencesType', params.EnumParam,
choices=['References', 'Gaussian blobs'],
default=REF_AVERAGES,
display=params.EnumParam.DISPLAY_HLIST,
label='References type',
help='You may select "Gaussian blobs" to be used as '
'references. The preferred way to autopick is '
'by providing 2D references images that were '
'by 2D classification. \n'
'The Gaussian blob references may be useful to '
'kickstart a new data set.')
group.addParam('gaussianPeak', params.FloatParam, default=0.1,
condition='referencesType==%s' % REF_BLOBS,
label='Gaussian peak value',
help='The peak value of the Gaussian blob. '
'Weaker data will need lower values.')
group.addParam('inputReferences', params.PointerParam,
pointerClass='SetOfAverages',
condition=refCondition,
label='Input references', important=True,
help='Input references (SetOfAverages) for auto-pick. \n\n'
'Note that the absolute greyscale needs to be correct, \n'
'so only use images with proper normalization.')
group.addParam('particleDiameter', params.IntParam, default=-1,
label='Mask diameter (A)',
help='Diameter of the circular mask that will be applied '
'around the templates in Angstroms. When set to a '
'negative value, this value is estimated '
'automatically from the templates themselves.')
form.addSection('References')
form.addParam('lowpassFilterRefs', params.IntParam, default=20,
condition=refCondition,
label='Lowpass filter references (A)',
help='Lowpass filter that will be applied to the '
'references before template matching. \n'
'Do NOT use very high-resolution templates to '
'search your micrographs. \n'
'The signal will be too weak at high resolution '
'anyway, and you may find Einstein from noise...')
form.addParam('highpassFilterMics', params.IntParam, default=-1,
label='Highpass filter (A)',
help='Highpass filter that will be applied to the '
'micrographs. This may be useful to get rid of '
'background ramps due to uneven ice distributions. '
'Give a negative value to skip the highpass '
'filter. Useful values are often in the range '
'of 200-400 Angstroms.')
form.addParam('angularSampling', params.IntParam, default=5,
label='Angular sampling (deg)',
help='Angular sampling in degrees for exhaustive searches '
'of the in-plane rotations for all references.')
form.addParam('refsHaveInvertedContrast', params.BooleanParam,
default=True,
label='References have inverted contrast?',
help='Set to Yes to indicate that the reference have '
'inverted contrast with respect to the particles '
'in the micrographs.')
form.addParam('refsCtfCorrected', params.BooleanParam, default=True,
condition=refCondition,
label='Are References CTF corrected?',
help='Set to Yes if the references were created with '
'CTF-correction inside RELION.\n'
'If set to Yes, the input micrographs should contain '
'the CTF information.')
form.addParam('ignoreCTFUntilFirstPeak', params.BooleanParam,
condition=refCondition,
default=False, expertLevel=params.LEVEL_ADVANCED,
label='Ignore CTFs until first peak?',
help='Set this to Yes, only if this option was also used '
'to generate the references.')
form.addSection('Autopicking')
group = form.addGroup('Autopick')
group.addParam('pickingThreshold', params.FloatParam, default=0.25,
label='Picking threshold:',
help='Use lower thresholds to pick more particles '
'(and more junk probably)')
group.addParam('interParticleDistance', params.IntParam, default=-1,
label='Minimum inter-particle distance (A):',
help='Particles closer together than this distance \n'
'will be consider to be a single cluster. \n'
'From each cluster, only one particle will be '
'picked.')
group.addParam('maxStddevNoise', params.FloatParam, default=1.1,
label='Maximum stddev noise:',
help='This is useful to prevent picking in carbon areas, '
'or areas with big contamination features. Peaks in '
'areas where the background standard deviation in '
'the normalized micrographs is higher than this '
'value will be ignored. Useful values are probably '
'in the range 1.0 to 1.2. Set to -1 to switch off '
'the feature to eliminate peaks due to high '
'background standard deviations.')
group = form.addGroup('Computing')
group.addParam('shrinkFactor', params.FloatParam, default=0,
validators=[params.Range(0, 1, "value should be "
"between 0 and 1. ")],
label='Shrink factor',
help='This is useful to speed up the calculations, '
'and to make them less memory-intensive. The '
'micrographs will be downscaled (shrunk) to '
'calculate the cross-correlations, and peak '
'searching will be done in the downscaled FOM '
'maps. When set to 0, the micrographs will de '
'downscaled to the lowpass filter of the '
'references, a value between 0 and 1 will '
'downscale the micrographs by that factor. '
'Note that the results will not be exactly '
'the same when you shrink micrographs!')
group.addParam('doGpu', params.BooleanParam, default=True,
label='Use GPU acceleration?',
help='If set to Yes, the job will try to use GPU '
'acceleration.')
group.addParam('gpusToUse', params.StringParam, default='',
label='Which GPUs to use:', condition='doGpu',
help='This argument is not necessary. If left empty, '
'the job itself will try to allocate available GPU '
'resources. You can override the default '
'allocation by providing a list of which GPUs '
'(0,1,2,3, etc) to use. MPI-processes are '
'separated by ":", threads by ",". '
'For example: "0,0:1,1:0,0:1,1"')
form.addParam('extraParams', params.StringParam, default='',
label='Additional arguments:',
help='In this box command-line arguments may be provided '
'that are not generated by the GUI. This may be '
'useful for testing developmental options and/or '
'expert use of the program. \n'
'The command "relion_autopick" will print a list '
'of possible options.')
form.addSection('Helix')
form.addParam('fomLabel', params.LabelParam,
important=True,
label='Helix processing is not implemented still.')
self._defineStreamingParams(form)
form.addParallelSection(threads=0, mpi=4)
# -------------------------- INSERT steps functions -----------------------
def _insertAllSteps(self):
self.inputStreaming = self.getInputMicrographs().isStreamOpen()
if self.inputStreaming and not self.isRunOptimize():
# If the input is in streaming, follow the base class policy
# about inserting new steps and discovery new input/output
ProtParticlePickingAuto._insertAllSteps(self)
self.createOutputStep = self._doNothing
else:
# If not in streaming, then we will just insert a single step to
# pick all micrographs at once since it is much faster
self._insertInitialSteps()
self._insertFunctionStep('_pickMicrographsFromStar',
self._getPath('input_micrographs.star'),
*self._getPickArgs())
self._insertFunctionStep('createOutputStep')
# Disable streaming functions:
self._insertFinalSteps = self._doNothing
self._stepsCheck = self._doNothing
def _insertInitialSteps(self):
# Convert the input micrographs and references to
# the required Relion star files
inputRefs = self.getInputReferences()
refId = inputRefs.strId() if self.useInputReferences() else 'Gaussian'
convertId = self._insertFunctionStep('convertInputStep',
self.getInputMicrographs().strId(),
refId, self.runType.get())
return [convertId]
def _doNothing(self, *args):
pass # used to avoid some streaming functions
def _loadInputList(self):
""" This function is re-implemented in this protocol, because it have
a SetOfCTF as input, so for streaming, we only want to report those
micrographs for which the CTF is ready.
"""
micDict, micClose = self._loadMics(self.getInputMicrographs())
ctfDict, ctfClosed = self._loadCTFs(self.ctfRelations.get())
# Remove the micrographs that have not CTF
# and set the CTF property for those who have it
for micKey, mic in micDict.iteritems():
if micKey in ctfDict:
mic.setCTF(ctfDict[micKey])
else:
del micDict[micKey]
# Return the updated micDict and the closed status
return micDict, micClose and ctfClosed
# -------------------------- STEPS functions ------------------------------
def convertInputStep(self, micsId, refsId, runType):
# runType is passed as parameter to force a re-execute of this step
# if there is a change in the type
self._ih = ImageHandler() # used to convert micrographs
# Match ctf information against the micrographs
self.ctfDict = {}
if self.ctfRelations.get() is not None:
for ctf in self.ctfRelations.get():
self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone()
micStar = self._getPath('input_micrographs.star')
writeSetOfMicrographs(self.getMicrographList(), micStar,
alignType=ALIGN_NONE,
preprocessImageRow=self._preprocessMicrographRow)
if self.useInputReferences():
writeReferences(self.getInputReferences(),
self._getPath('input_references'), useBasename=True)
# FIXME: (JMRT-20180523) The following code does not seems to work
# here it has been worked around by changing the name of the wizard
# output but this seems to reflect a deeper problem of deleting
# already existing output objects in a protocol. Maybe when updating
# from run.db to project.sqlite?
# Clean up if previously created the outputMicrographs and Coordinates
# in the wizard - optimization run
# if self.hasAttribute('outputMicrographs'):
# self._deleteChild('outputMicrographs', self.outputMicrographs)
# if self.hasAttribute('outputCoordinates'):
# self._deleteChild('outputCoordinates', self.outputCoordinates)
# self._store()
def getAutopickParams(self):
# Return the autopicking parameters except for the interative ones:
# - threshold
# - minDistance
# - maxStd
params = ' --pickname autopick'
params += ' --odir ""'
params += ' --particle_diameter %d' % self.particleDiameter
params += ' --angpix %0.3f' % self.getInputMicrographs().getSamplingRate()
params += ' --shrink %0.3f' % self.shrinkFactor
if self.doGpu:
params += ' --gpu "%s"' % self.gpusToUse
# Now in Relion2.0 autopick can use gassian blobs
if self.useInputReferences():
params += ' --ref input_references.star'
ps = self.getInputReferences().getSamplingRate()
params += ' --angpix_ref %0.3f' % ps
else: # Gaussian blobs
params += ' --ref gauss --gauss_max %0.3f' % self.gaussianPeak
if self.refsHaveInvertedContrast:
params += ' --invert'
if self.refsCtfCorrected:
params += ' --ctf'
params += ' --ang %d' % self.angularSampling
# Negative values for filters means no-filter
if self.lowpassFilterRefs > 0:
params += ' --lowpass %d' % self.lowpassFilterRefs
if self.highpassFilterMics > 0:
params += ' --highpass %d' % self.highpassFilterMics
# Add extra params is any
params += ' %s' % self.extraParams
return params
def _getPickArgs(self):
basicArgs = self.getAutopickParams()
threshold = self.pickingThreshold.get()
interDist = self.interParticleDistance.get()
fomParam = ' --write_fom_maps' if self.isRunOptimize() else ''
return [basicArgs, threshold, interDist, fomParam]
def _pickMicrographsFromStar(self, micStarFile, params,
threshold, minDistance, fom):
""" Launch the 'relion_autopick' for micrographs in the inputStarFile.
If the input set of complete, the star file will contain all the
micrographs. If working in streaming, it will be only one micrograph.
"""
params += ' --i %s' % relpath(micStarFile, self.getWorkingDir())
params += ' --threshold %0.3f ' % threshold
params += ' --min_distance %0.3f %s' % (minDistance, fom)
program = self._getProgram('relion_autopick')
self.runJob(program, params, cwd=self.getWorkingDir())
def _pickMicrograph(self, mic, params, threshold, minDistance, fom):
""" This method should be invoked only when working in streaming mode.
"""
micRow = md.Row()
self._preprocessMicrographRow(mic, micRow)
micrographToRow(mic, micRow)
self._postprocessMicrographRow(mic, micRow)
self._pickMicrographsFromStar(self._getMicStarFile(mic), params,
threshold, minDistance, fom)
def _pickMicrographList(self, micList, params, threshold, minDistance, fom):
micStar = self._getPath('input_micrographs_%s-%s.star' %
(micList[0].strId(), micList[-1].strId()))
writeSetOfMicrographs(micList, micStar,
alignType=ALIGN_NONE,
preprocessImageRow=self._preprocessMicrographRow)
self._pickMicrographsFromStar(micStar, params, threshold, minDistance,
fom)
def _createSetOfCoordinates(self, micSet, suffix=''):
""" Override this method to set the box size. """
coordSet = ProtParticlePickingAuto._createSetOfCoordinates(self, micSet,
suffix=suffix)
coordSet.setBoxSize(self.getBoxSize())
return coordSet
def readCoordsFromMics(self, workingDir, micList, coordSet):
""" Parse back the output star files and populate the SetOfCoordinates.
"""
template = self._getExtraPath("%s_autopick.star")
starFiles = [template % pwutils.removeBaseExt(mic.getFileName())
for mic in micList]
readSetOfCoordinates(coordSet, starFiles, micList)
# -------------------------- STEPS functions -------------------------------
def autopickStep(self, micStarFile, params, threshold,
minDistance, maxStddevNoise, fom):
""" This method is used from the wizard to optimize the parameters. """
self._pickMicrographsFromStar(micStarFile, params, threshold,
minDistance, maxStddevNoise, fom)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(),
coordSet)
# -------------------------- INFO functions --------------------------------
def _validate(self):
errors = []
self.validatePackageVersion('RELION_HOME', errors)
if self.useInputReferences():
if self.particleDiameter > self.getInputDimA():
errors.append('Particle diameter (%d) can not be greater than '
'size (%d)' % (self.particleDiameter,
self.getInputDimA()))
if self.getInputReferences().isOddX():
errors.append("Relion only works with even values for the "
"average dimensions!")
else:
if self.particleDiameter <= 0:
errors.append('When using Gaussian blobs, you need to specify '
'the particles diameter manually. ')
if self.ctfRelations.get() is None and self.refsCtfCorrected:
errors.append("References CTF corrected parameter must be set to "
"False or set ctf relations.")
errors.extend(self._validateMicSelection())
return errors
def _validateMicSelection(self):
""" Validate the cases when selecting a subset of micrographs
to optimize.
"""
inputMics = self.getInputMicrographs()
inputCTFs = self.ctfRelations.get()
if self.isRunOptimize():
if self.micrographsSelection == MICS_AUTO:
n = self.micrographsNumber.get()
if n < 3 or n > min(30, inputMics.getSize()):
return ['Number of micrographs should be between 3 and '
'min(30, input_size)']
else:
micSubset = self.micrographsSubset.get()
if micSubset is None:
return ['Select the subset of micrographs']
def missing(mic):
micId = mic.getObjId()
return inputMics[micId] is None or inputCTFs[micId] is None
if any(missing(mic) for mic in micSubset):
return ['Some selected micrograph IDs are missing from the '
'input micrographs or CTFs.']
return []
def _warnings(self):
if not self.isRunOptimize():
if not hasattr(self, 'wizardExecuted'):
return ['It seems that you have not executed the wizard to '
'optimize the picking parameters. \n'
'Do you want to launch the whole picking anyway?']
return []
def _summary(self):
summary = []
return summary
def _methods(self):
methodsMsgs = []
if self.getOutputsSize() > 0:
output = self.getCoords()
methodsMsgs.append("%s: User picked %d particles with a particle "
"size of %d px."
% (self.getObjectTag(output), output.getSize(),
output.getBoxSize()))
else:
methodsMsgs.append(Message.TEXT_NO_OUTPUT_CO)
return methodsMsgs
# -------------------------- UTILS functions -------------------------------
def useInputReferences(self):
return self.referencesType == REF_AVERAGES
def isRunOptimize(self):
return self.runType == RUN_OPTIMIZE
def getInputDimA(self):
""" Return the dimension of input references in A. """
inputRefs = self.getInputReferences()
if inputRefs is None:
return None
else:
return inputRefs.getXDim() * inputRefs.getSamplingRate()
def getBoxSize(self):
""" Return a reasonable box-size in pixels. """
inputRefs = self.getInputReferences()
inputMics = self.getInputMicrographs()
micsSampling = inputMics.getSamplingRate()
if inputRefs is None:
boxSize = int(self.particleDiameter.get() * 1.25 / micsSampling)
else:
# Scale boxsize if the pixel size of the references is not the same
# of the micrographs
scale = inputRefs.getSamplingRate() / micsSampling
boxSize = int(inputRefs.getXDim() * scale)
if boxSize % 2 == 1:
boxSize += 1 # Use even box size for relion
return boxSize
def getInputReferences(self):
return self.inputReferences.get()
def getInputMicrographsPointer(self):
return self.inputMicrographs
def getInputMicrographs(self):
return self.getInputMicrographsPointer().get()
def getMicrographList(self):
""" Return the list of micrographs (either a subset or the full set)
that will be used for optimizing the parameters or the picking.
"""
# Use all micrographs only when going for the full picking
inputMics = self.getInputMicrographs()
if not self.isRunOptimize():
return inputMics
if self.micrographsSelection == MICS_AUTO:
mics = getSubsetByDefocus(self.ctfRelations.get(), inputMics,
self.micrographsNumber.get())
else: # Subset selection
mics = [mic.clone() for mic in self.micrographsSubset.get()]
return mics
def getCoordsDir(self):
return self._getTmpPath('xmipp_coordinates')
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def writeXmippOutputCoords(self):
return self._writeXmippCoords(self.outputCoordinates)
def writeXmippCoords(self):
""" Write the SetOfCoordinates as expected by Xmipp
to be displayed with its GUI.
"""
micSet = self.getInputMicrographs()
coordSet = self._createSetOfCoordinates(micSet)
coordSet.setBoxSize(self.getBoxSize())
starFiles = [self._getExtraPath(pwutils.removeBaseExt(mic.getFileName())
+ '_autopick.star') for mic in micSet]
readSetOfCoordinates(coordSet, starFiles)
return self._writeXmippCoords(coordSet)
def _preprocessMicrographRow(self, img, imgRow):
# Temporarly convert the few micrographs to tmp and make sure
# they are in 'mrc' format
# Get basename and replace extension by 'mrc'
newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc')
newPath = self._getExtraPath(newName)
# If the micrographs are in 'mrc' format just create a link
# if not, convert to 'mrc'
if img.getFileName().endswith('mrc'):
pwutils.createLink(img.getFileName(), newPath)
else:
self._ih.convert(img, newPath)
# The command will be launched from the working dir
# so, let's make the micrograph path relative to that
img.setFileName(os.path.join('extra', newName))
# JMRT: The following is not needed since it is set when loading CTF
# if self.ctfRelations.get() is not None:
# img.setCTF(self.ctfDict[img.getMicName()])
def _postprocessMicrographRow(self, img, imgRow):
imgRow.writeToFile(self._getMicStarFile(img))
def _getMicStarFile(self, mic):
micBase = pwutils.replaceBaseExt(mic.getFileName(), 'star')
return self._getExtraPath(micBase)
def _processMovie(self, movie):
movId = movie.getObjId()
x, y, n = movie.getDim()
iniFrame, lastFrame, _ = movie.getFramesRange()
frame0, frameN = self._getRange(movie)
boxSize = self.boxSize.get()
if movie.hasAlignment() and self.applyAlignment:
shiftX, shiftY = movie.getAlignment().getShifts() # lists.
else:
shiftX = [0] * (lastFrame - iniFrame + 1)
shiftY = shiftX
stkIndex = 0
movieStk = self._getMovieName(movie, '.stk')
movieMdFile = self._getMovieName(movie, '.xmd')
movieMd = md.MetaData()
frameMd = md.MetaData()
frameMdImages = md.MetaData()
frameRow = md.Row()
if self._hasCoordinates(movie):
imgh = ImageHandler()
for frame in range(frame0, frameN + 1):
indx = frame - iniFrame
frameName = self._getFnRelated('frameMic', movId, frame)
frameMdFile = self._getFnRelated('frameMdFile', movId, frame)
coordinatesName = self._getFnRelated('frameCoords', movId,
frame)
frameImages = self._getFnRelated('frameImages', movId, frame)
frameStk = self._getFnRelated('frameStk', movId, frame)
self._writeXmippPosFile(movie, coordinatesName, shiftX[indx],
shiftY[indx])
self.info("Writing frame: %s" % frameName)
# TODO: there is no need to write the frame and then operate
# the input of the first operation should be the movie
movieName = imgh.fixXmippVolumeFileName(movie)
imgh.convert((frame, movieName), frameName)
if self.doRemoveDust:
self.info("Removing Dust")
self._runNoDust(frameName)
self.info("Extracting particles")
args = '-i %(frameName)s --pos %(coordinatesName)s ' \
'-o %(frameImages)s --Xdim %(boxSize)d' % locals()
if self.doInvert:
args += " --invert"
if self.doBorders:
args += " --fillBorders"
args += " --downsampling %f " % self.getBoxScale()
self.runJob('xmipp_micrograph_scissor', args)
cleanPath(frameName)
self.info("Combining particles into one stack.")
frameMdImages.read(frameMdFile)
frameMd.read('particles@%s' % coordinatesName)
frameMd.merge(frameMdImages)
for objId in frameMd:
stkIndex += 1
frameRow.readFromMd(frameMd, objId)
location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
newLocation = (stkIndex, movieStk)
imgh.convert(location, newLocation)
# Fix the name to be accessible from the Project directory
# so we know that the movie stack file will be moved
# to final particles folder
newImageName = '%d@%s' % newLocation
frameRow.setValue(md.MDL_IMAGE, newImageName)
frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movId))
frameRow.setValue(md.MDL_MICROGRAPH, str(movId))
frameRow.setValue(md.MDL_FRAME_ID, long(frame))
frameRow.setValue(md.MDL_PARTICLE_ID,
frameRow.getValue(md.MDL_ITEM_ID))
frameRow.writeToMd(movieMd, movieMd.addObject())
movieMd.addItemId()
movieMd.write(movieMdFile)
cleanPath(frameStk)
if self.doNormalize:
numberOfFrames = frameN - frame0 + 1
self._runNormalize(movieStk, numberOfFrames)
class ProtRelion2Autopick(ProtParticlePickingAuto, ProtRelionBase):
"""
This Relion protocol uses the 'relion_autopick' program to pick particles
from micrographs, either using templates or gaussian blobs.
The picking with this protocol is divided in three steps:
1) Run with 'Optimize' option for several (less than 30) micrographs.
2) Execute the wizard to refine the picking parameters.
3) Run with 'Pick all' option to pick particles from all micrographs.
The first steps will use internally the option '--write-fom-maps' to write
to disk the FOM maps. The expensive part of this calculation is to calculate
a probability-based figure-of-merit (related to the cross-correlation
coefficient between each rotated reference and all positions in the
micrographs. That's why it is only done in an small subset of the
micrographs, where one should use representative micrographs for the entire
data set, e.g. a high and a low-defocus one, and/or with thin or thick ice.
Step 2 uses a much cheaper peak-detection algorithm that uses the threshold
and minimum distance parameters.
"""
_label = 'auto-picking'
@classmethod
def isDisabled(cls):
return not (isVersion2() or isVersion3())
# -------------------------- DEFINE param functions ------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMicrographs',
params.PointerParam,
pointerClass='SetOfMicrographs',
label='Input micrographs',
important=True,
help='Select the input micrographs. '
'If using the *Optimize* mode, just a subset of '
'micrographs are used to compute the FOM maps. '
'If in *Compute* mode, all micrographs will be '
'auto-picked.')
form.addParam('ctfRelations',
params.RelationParam,
relationName=RELATION_CTF,
attributeName='getInputMicrographs',
label='CTF estimation',
help='Choose some CTF estimation related to the '
'input micrographs.')
form.addParam('runType',
params.EnumParam,
default=RUN_OPTIMIZE,
choices=['Optimize params', 'Pick all micrographs'],
display=params.EnumParam.DISPLAY_LIST,
label='Run type: ',
help='Usually, first you should use the *Optimize* mode '
'to compute the FOM maps for a few micrographs and '
'use them to tune the picking parameters using the '
'wizard. After that you can run the job in *Compute*'
' mode and auto-pick all the micrographs. ')
group = form.addGroup('Micrographs for optimization',
condition='runType==%d' % RUN_OPTIMIZE)
group.addParam('micrographsSelection',
params.EnumParam,
default=MICS_AUTO,
choices=['automatic selection', 'input subset'],
display=params.EnumParam.DISPLAY_HLIST,
label='Choose micrographs by',
help='If you choose "automatic selection", you only '
'need to provide the number of microgrphs to use '
'and that number will be selected to cover the '
'defocus range. ')
group.addParam('micrographsNumber',
params.IntParam,
default='10',
condition='micrographsSelection==%d' % MICS_AUTO,
label='Micrographs for optimization:',
help='Select the number of micrographs that you want'
'to be used for the parameters optimization. ')
group.addParam('micrographsSubset',
params.PointerParam,
condition='micrographsSelection==%d' % MICS_SUBSET,
pointerClass='SetOfMicrographs',
label='Subset of micrographs',
help='Choose as input a subset of micrographs that '
'you have previously selected. '
'(Probably covering the defocus range).')
# From Relion 2.+, it can be picked with gaussian blobs, so we
# need to add these parameters
refCondition = 'referencesType==%s' % REF_AVERAGES
group = form.addGroup('References')
group.addParam('referencesType',
params.EnumParam,
choices=['References', 'Gaussian blobs'],
default=REF_AVERAGES,
display=params.EnumParam.DISPLAY_HLIST,
label='References type',
help='You may select "Gaussian blobs" to be used as '
'references. The preferred way to autopick is '
'by providing 2D references images that were '
'obtained by 2D classification. \n'
'The Gaussian blob references may be useful to '
'kickstart a new data set.')
group.addParam('gaussianPeak',
params.FloatParam,
default=0.1,
condition='referencesType==%s' % REF_BLOBS,
label='Gaussian peak value',
help='The peak value of the Gaussian blob. '
'Weaker data will need lower values.')
pointerClassStr = 'SetOfAverages'
# In Relion 3 it is also possible to pass a volume as reference for
# autopicking
if isVersion3():
pointerClassStr += ",Volume"
group.addParam(
'inputReferences',
params.PointerParam,
pointerClass='SetOfAverages',
condition=refCondition,
label='Input references',
important=True,
help='Input references (SetOfAverages) for auto-pick. \n\n'
'Note that the absolute greyscale needs to be correct, \n'
'so only use images with proper normalization. '
'From Relion 3.0 it is also possible to provide a '
'3D volume which projections will be used as '
'references.')
group.addParam(
'particleDiameter',
params.IntParam,
default=-1,
label='Mask diameter (A)',
help='Diameter of the circular mask that will be applied '
'around the templates in Angstroms. When set to a '
'negative value, this value is estimated '
'automatically from the templates themselves.')
form.addSection('References')
form.addParam('lowpassFilterRefs',
params.IntParam,
default=20,
condition=refCondition,
label='Lowpass filter references (A)',
help='Lowpass filter that will be applied to the '
'references before template matching. \n'
'Do NOT use very high-resolution templates to '
'search your micrographs. \n'
'The signal will be too weak at high resolution '
'anyway, and you may find Einstein from noise...')
form.addParam('highpassFilterMics',
params.IntParam,
default=-1,
label='Highpass filter (A)',
help='Highpass filter that will be applied to the '
'micrographs. This may be useful to get rid of '
'background ramps due to uneven ice distributions. '
'Give a negative value to skip the highpass '
'filter. Useful values are often in the range '
'of 200-400 Angstroms.')
form.addParam(
'angularSampling',
params.IntParam,
default=5,
label='Angular sampling (deg)',
help='Angular sampling in degrees for exhaustive searches '
'of the in-plane rotations for all references.')
form.addParam('refsHaveInvertedContrast',
params.BooleanParam,
default=True,
label='References have inverted contrast?',
help='Set to Yes to indicate that the reference have '
'inverted contrast with respect to the particles '
'in the micrographs.')
form.addParam('refsCtfCorrected',
params.BooleanParam,
default=True,
condition=refCondition,
label='Are References CTF corrected?',
help='Set to Yes if the references were created with '
'CTF-correction inside RELION.\n'
'If set to Yes, the input micrographs should contain '
'the CTF information.')
form.addParam(
'ignoreCTFUntilFirstPeak',
params.BooleanParam,
condition=refCondition,
default=False,
expertLevel=params.LEVEL_ADVANCED,
label='Ignore CTFs until first peak?',
help='Set this to Yes, only if this option was also used '
'to generate the references.')
form.addSection('Autopicking')
group = form.addGroup('Autopick')
group.addParam('pickingThreshold',
params.FloatParam,
default=0.25,
label='Picking threshold:',
help='Use lower thresholds to pick more particles '
'(and more junk probably)')
group.addParam('interParticleDistance',
params.IntParam,
default=-1,
label='Minimum inter-particle distance (A):',
help='Particles closer together than this distance \n'
'will be consider to be a single cluster. \n'
'From each cluster, only one particle will be '
'picked.')
group.addParam(
'maxStddevNoise',
params.FloatParam,
default=1.1,
label='Maximum stddev noise:',
help='This is useful to prevent picking in carbon areas, '
'or areas with big contamination features. Peaks in '
'areas where the background standard deviation in '
'the normalized micrographs is higher than this '
'value will be ignored. Useful values are probably '
'in the range 1.0 to 1.2. Set to -1 to switch off '
'the feature to eliminate peaks due to high '
'background standard deviations.')
group = form.addGroup('Computing')
group.addParam('shrinkFactor',
params.FloatParam,
default=0,
validators=[
params.Range(0, 1, "value should be "
"between 0 and 1. ")
],
label='Shrink factor',
help='This is useful to speed up the calculations, '
'and to make them less memory-intensive. The '
'micrographs will be downscaled (shrunk) to '
'calculate the cross-correlations, and peak '
'searching will be done in the downscaled FOM '
'maps. When set to 0, the micrographs will de '
'downscaled to the lowpass filter of the '
'references, a value between 0 and 1 will '
'downscale the micrographs by that factor. '
'Note that the results will not be exactly '
'the same when you shrink micrographs!')
group.addParam('doGpu',
params.BooleanParam,
default=True,
label='Use GPU acceleration?',
help='If set to Yes, the job will try to use GPU '
'acceleration.')
group.addParam('gpusToUse',
params.StringParam,
default='',
label='Which GPUs to use:',
condition='doGpu',
help='This argument is not necessary. If left empty, '
'the job itself will try to allocate available GPU '
'resources. You can override the default '
'allocation by providing a list of which GPUs '
'(0,1,2,3, etc) to use. MPI-processes are '
'separated by ":", threads by ",". '
'For example: "0,0:1,1:0,0:1,1"')
form.addParam(
'extraParams',
params.StringParam,
default='',
label='Additional arguments:',
help='In this box command-line arguments may be provided '
'that are not generated by the GUI. This may be '
'useful for testing developmental options and/or '
'expert use of the program. \n'
'The command "relion_autopick" will print a list '
'of possible options.')
form.addSection('Helix')
form.addParam('fomLabel',
params.LabelParam,
important=True,
label='Helix processing is not implemented still.')
self._defineStreamingParams(form)
form.addParallelSection(threads=0, mpi=4)
# -------------------------- INSERT steps functions -----------------------
def _insertAllSteps(self):
self.inputStreaming = self.getInputMicrographs().isStreamOpen()
if self.inputStreaming and not self.isRunOptimize():
# If the input is in streaming, follow the base class policy
# about inserting new steps and discovery new input/output
ProtParticlePickingAuto._insertAllSteps(self)
self.createOutputStep = self._doNothing
else:
# If not in streaming, then we will just insert a single step to
# pick all micrographs at once since it is much faster
self._insertInitialSteps()
self._insertFunctionStep('_pickMicrographsFromStar',
self._getPath('input_micrographs.star'),
*self._getPickArgs())
self._insertFunctionStep('createOutputStep')
# Disable streaming functions:
self._insertFinalSteps = self._doNothing
self._stepsCheck = self._doNothing
def _insertInitialSteps(self):
# Convert the input micrographs and references to
# the required Relion star files
inputRefs = self.getInputReferences()
refId = inputRefs.strId() if self.useInputReferences() else 'Gaussian'
convertId = self._insertFunctionStep(
'convertInputStep',
self.getInputMicrographs().strId(), refId, self.runType.get())
return [convertId]
def _doNothing(self, *args):
pass # used to avoid some streaming functions
def _loadInputList(self):
""" This function is re-implemented in this protocol, because it have
a SetOfCTF as input, so for streaming, we only want to report those
micrographs for which the CTF is ready.
"""
micDict, micClose = self._loadMics(self.getInputMicrographs())
ctfDict, ctfClosed = self._loadCTFs(self.ctfRelations.get())
# Remove the micrographs that have not CTF
# and set the CTF property for those who have it
for micKey, mic in micDict.iteritems():
if micKey in ctfDict:
mic.setCTF(ctfDict[micKey])
else:
del micDict[micKey]
# Return the updated micDict and the closed status
return micDict, micClose and ctfClosed
# -------------------------- STEPS functions ------------------------------
def convertInputStep(self, micsId, refsId, runType):
# runType is passed as parameter to force a re-execute of this step
# if there is a change in the type
self._ih = ImageHandler() # used to convert micrographs
# Match ctf information against the micrographs
self.ctfDict = {}
if self.ctfRelations.get() is not None:
for ctf in self.ctfRelations.get():
self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone()
micStar = self._getPath('input_micrographs.star')
writeSetOfMicrographs(self.getMicrographList(),
micStar,
alignType=ALIGN_NONE,
preprocessImageRow=self._preprocessMicrographRow)
if self.useInputReferences():
writeReferences(self.getInputReferences(),
self._getPath('input_references'),
useBasename=True)
# FIXME: (JMRT-20180523) The following code does not seems to work
# here it has been worked around by changing the name of the wizard
# output but this seems to reflect a deeper problem of deleting
# already existing output objects in a protocol. Maybe when updating
# from run.db to project.sqlite?
# Clean up if previously created the outputMicrographs and Coordinates
# in the wizard - optimization run
# if self.hasAttribute('outputMicrographs'):
# self._deleteChild('outputMicrographs', self.outputMicrographs)
# if self.hasAttribute('outputCoordinates'):
# self._deleteChild('outputCoordinates', self.outputCoordinates)
# self._store()
def getAutopickParams(self):
# Return the autopicking parameters except for the interative ones:
# - threshold
# - minDistance
# - maxStd
params = ' --pickname autopick'
params += ' --odir ""'
params += ' --particle_diameter %d' % self.particleDiameter
params += ' --angpix %0.3f' % self.getInputMicrographs(
).getSamplingRate()
params += ' --shrink %0.3f' % self.shrinkFactor
if self.doGpu:
params += ' --gpu "%s"' % self.gpusToUse
# Now in Relion2.0 autopick can use gassian blobs
if self.useInputReferences():
params += ' --ref input_references.star'
ps = self.getInputReferences().getSamplingRate()
params += ' --angpix_ref %0.3f' % ps
else: # Gaussian blobs
params += ' --ref gauss --gauss_max %0.3f' % self.gaussianPeak
if self.refsHaveInvertedContrast:
params += ' --invert'
if self.refsCtfCorrected:
params += ' --ctf'
params += ' --ang %d' % self.angularSampling
# Negative values for filters means no-filter
if self.lowpassFilterRefs > 0:
params += ' --lowpass %d' % self.lowpassFilterRefs
if self.highpassFilterMics > 0:
params += ' --highpass %d' % self.highpassFilterMics
# Add extra params is any
params += ' %s' % self.extraParams
return params
def _getPickArgs(self):
basicArgs = self.getAutopickParams()
threshold = self.pickingThreshold.get()
interDist = self.interParticleDistance.get()
fomParam = ' --write_fom_maps' if self.isRunOptimize() else ''
return [basicArgs, threshold, interDist, fomParam]
def _pickMicrographsFromStar(self, micStarFile, params, threshold,
minDistance, fom):
""" Launch the 'relion_autopick' for micrographs in the inputStarFile.
If the input set of complete, the star file will contain all the
micrographs. If working in streaming, it will be only one micrograph.
"""
params += ' --i %s' % relpath(micStarFile, self.getWorkingDir())
params += ' --threshold %0.3f ' % threshold
params += ' --min_distance %0.3f %s' % (minDistance, fom)
program = self._getProgram('relion_autopick')
self.runJob(program, params, cwd=self.getWorkingDir())
def _pickMicrograph(self, mic, params, threshold, minDistance, fom):
""" This method should be invoked only when working in streaming mode.
"""
micRow = md.Row()
self._preprocessMicrographRow(mic, micRow)
micrographToRow(mic, micRow)
self._postprocessMicrographRow(mic, micRow)
self._pickMicrographsFromStar(self._getMicStarFile(mic), params,
threshold, minDistance, fom)
def _pickMicrographList(self, micList, params, threshold, minDistance,
fom):
micStar = self._getPath('input_micrographs_%s-%s.star' %
(micList[0].strId(), micList[-1].strId()))
writeSetOfMicrographs(micList,
micStar,
alignType=ALIGN_NONE,
preprocessImageRow=self._preprocessMicrographRow)
self._pickMicrographsFromStar(micStar, params, threshold, minDistance,
fom)
def _createSetOfCoordinates(self, micSet, suffix=''):
""" Override this method to set the box size. """
coordSet = ProtParticlePickingAuto._createSetOfCoordinates(
self, micSet, suffix=suffix)
coordSet.setBoxSize(self.getBoxSize())
return coordSet
def readCoordsFromMics(self, workingDir, micList, coordSet):
""" Parse back the output star files and populate the SetOfCoordinates.
"""
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micList
]
readSetOfCoordinates(coordSet, starFiles, micList)
# -------------------------- STEPS functions -------------------------------
def autopickStep(self, micStarFile, params, threshold, minDistance,
maxStddevNoise, fom):
""" This method is used from the wizard to optimize the parameters. """
self._pickMicrographsFromStar(micStarFile, params, threshold,
minDistance, maxStddevNoise, fom)
def createOutputStep(self):
micSet = self.getInputMicrographs()
outputCoordinatesName = 'outputCoordinates'
outputSuffix = ''
# If in optimization phase, let's create a subset of the micrographs
if self.isRunOptimize():
outputSuffix = '_subset'
outputCoordinatesName = 'outputCoordinatesSubset'
micSubSet = self._createSetOfMicrographs(suffix=outputSuffix)
micSubSet.copyInfo(micSet)
# Use previously written star file for reading the subset of micrographs,
for row in md.iterRows(self._getPath('input_micrographs.star')):
mic = micSet[row.getValue('rlnImageId')]
micSubSet.append(mic)
self._defineOutputs(outputMicrographsSubset=micSubSet)
self._defineTransformRelation(self.getInputMicrographsPointer(),
micSubSet)
micSet = micSubSet
coordSet = self._createSetOfCoordinates(micSet)
template = self._getExtraPath("%s_autopick.star")
starFiles = [
template % pwutils.removeBaseExt(mic.getFileName())
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles, micSet)
self._defineOutputs(**{outputCoordinatesName: coordSet})
self._defineSourceRelation(self.getInputMicrographsPointer(), coordSet)
# -------------------------- INFO functions --------------------------------
def _validate(self):
errors = []
self.validatePackageVersion('RELION_HOME', errors)
if self.useInputReferences():
if self.particleDiameter > self.getInputDimA():
errors.append('Particle diameter (%d) can not be greater than '
'size (%d)' %
(self.particleDiameter, self.getInputDimA()))
if self.getInputReferences().isOddX():
errors.append("Relion only works with even values for the "
"average dimensions!")
else:
if self.particleDiameter <= 0:
errors.append('When using Gaussian blobs, you need to specify '
'the particles diameter manually. ')
if self.ctfRelations.get() is None and self.refsCtfCorrected:
errors.append("References CTF corrected parameter must be set to "
"False or set ctf relations.")
errors.extend(self._validateMicSelection())
return errors
def _validateMicSelection(self):
""" Validate the cases when selecting a subset of micrographs
to optimize.
"""
inputMics = self.getInputMicrographs()
inputCTFs = self.ctfRelations.get()
if self.isRunOptimize():
if self.micrographsSelection == MICS_AUTO:
n = self.micrographsNumber.get()
if n < 3 or n > min(30, inputMics.getSize()):
return [
'Number of micrographs should be between 3 and '
'min(30, input_size)'
]
else:
micSubset = self.micrographsSubset.get()
if micSubset is None:
return ['Select the subset of micrographs']
def missing(mic):
micId = mic.getObjId()
return inputMics[micId] is None or inputCTFs[micId] is None
if any(missing(mic) for mic in micSubset):
return [
'Some selected micrograph IDs are missing from the '
'input micrographs or CTFs.'
]
return []
def _warnings(self):
if not self.isRunOptimize():
if not hasattr(self, 'wizardExecuted'):
return [
'It seems that you have not executed the wizard to '
'optimize the picking parameters. \n'
'Do you want to launch the whole picking anyway?'
]
return []
def _summary(self):
summary = []
return summary
def _methods(self):
methodsMsgs = []
if self.getOutputsSize() > 0:
output = self.getCoords()
methodsMsgs.append("%s: User picked %d particles with a particle "
"size of %d px." %
(self.getObjectTag(output), output.getSize(),
output.getBoxSize()))
else:
methodsMsgs.append(Message.TEXT_NO_OUTPUT_CO)
return methodsMsgs
# -------------------------- UTILS functions -------------------------------
def useInputReferences(self):
return self.referencesType == REF_AVERAGES
def isRunOptimize(self):
return self.runType == RUN_OPTIMIZE
def getInputDimA(self):
""" Return the dimension of input references in A. """
inputRefs = self.getInputReferences()
if inputRefs is None:
return None
else:
return inputRefs.getXDim() * inputRefs.getSamplingRate()
def getBoxSize(self):
""" Return a reasonable box-size in pixels. """
inputRefs = self.getInputReferences()
inputMics = self.getInputMicrographs()
micsSampling = inputMics.getSamplingRate()
if inputRefs is None:
boxSize = int(self.particleDiameter.get() * 1.25 / micsSampling)
else:
# Scale boxsize if the pixel size of the references is not the same
# of the micrographs
scale = inputRefs.getSamplingRate() / micsSampling
boxSize = int(inputRefs.getXDim() * scale)
if boxSize % 2 == 1:
boxSize += 1 # Use even box size for relion
return boxSize
def getInputReferences(self):
return self.inputReferences.get()
def getInputMicrographsPointer(self):
return self.inputMicrographs
def getInputMicrographs(self):
return self.getInputMicrographsPointer().get()
def getMicrographList(self):
""" Return the list of micrographs (either a subset or the full set)
that will be used for optimizing the parameters or the picking.
"""
# Use all micrographs only when going for the full picking
inputMics = self.getInputMicrographs()
if not self.isRunOptimize():
return inputMics
if self.micrographsSelection == MICS_AUTO:
mics = getSubsetByDefocus(self.ctfRelations.get(), inputMics,
self.micrographsNumber.get())
else: # Subset selection
mics = [mic.clone() for mic in self.micrographsSubset.get()]
return mics
def getCoordsDir(self):
return self._getTmpPath('xmipp_coordinates')
def _writeXmippCoords(self, coordSet):
micSet = self.getInputMicrographs()
coordPath = self._getTmpPath('xmipp_coordinates')
pwutils.cleanPath(coordPath)
pwutils.makePath(coordPath)
import pyworkflow.em.packages.xmipp3 as xmipp3
micPath = micSet.getFileName()
xmipp3.writeSetOfCoordinates(coordPath, coordSet, ismanual=False)
return micPath, coordPath
def writeXmippOutputCoords(self):
return self._writeXmippCoords(self.outputCoordinates)
def writeXmippCoords(self):
""" Write the SetOfCoordinates as expected by Xmipp
to be displayed with its GUI.
"""
micSet = self.getInputMicrographs()
coordSet = self._createSetOfCoordinates(micSet)
coordSet.setBoxSize(self.getBoxSize())
starFiles = [
self._getExtraPath(
pwutils.removeBaseExt(mic.getFileName()) + '_autopick.star')
for mic in micSet
]
readSetOfCoordinates(coordSet, starFiles)
return self._writeXmippCoords(coordSet)
def _preprocessMicrographRow(self, img, imgRow):
# Temporarly convert the few micrographs to tmp and make sure
# they are in 'mrc' format
# Get basename and replace extension by 'mrc'
newName = pwutils.replaceBaseExt(img.getFileName(), 'mrc')
newPath = self._getExtraPath(newName)
# If the micrographs are in 'mrc' format just create a link
# if not, convert to 'mrc'
if img.getFileName().endswith('mrc'):
pwutils.createLink(img.getFileName(), newPath)
else:
self._ih.convert(img, newPath)
# The command will be launched from the working dir
# so, let's make the micrograph path relative to that
img.setFileName(os.path.join('extra', newName))
# JMRT: The following is not needed since it is set when loading CTF
# if self.ctfRelations.get() is not None:
# img.setCTF(self.ctfDict[img.getMicName()])
def _postprocessMicrographRow(self, img, imgRow):
imgRow.writeToFile(self._getMicStarFile(img))
def _getMicStarFile(self, mic):
micBase = pwutils.replaceBaseExt(mic.getFileName(), 'star')
return self._getExtraPath(micBase)
def _processMovie(self, movieId, movieName, movieFolder,
shifts): ###pasar shifts
movieName = os.path.join(movieFolder, movieName)
boxSize = self.boxSize.get()
# Read movie dimensions to iterate through each frame
imgh = ImageHandler()
x, y, z, n = imgh.getDimensions(movieName)
first = self.firstFrame.get()
if first <= 1:
first = 1
last = self.lastFrame.get()
if last <= 0 or last >= n:
last = n
numberOfFrames = last - first + 1
if shifts is None:
frames = max(z, n)
shifts = [0] * (2 * frames)
stkIndex = 0
movieStk = self._getMovieName(movieId, '.stk')
movieMdFile = self._getMovieName(movieId, '.xmd')
movieMd = md.MetaData()
frameMd = md.MetaData()
frameMdImages = md.MetaData()
frameRow = md.Row()
for frame in range(first, last + 1):
# Get the frame shifts
index = frame - first
shiftX = shifts[2 * index]
shiftY = shifts[2 * index + 1]
frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame)
frameName = frameRoot + '.mrc'
frameMdFile = frameRoot + '.xmd'
framePosFile = frameRoot + '_coordinates.xmd'
coordinatesName = frameRoot + '_coordinates.xmd'
hasCoordinates = self._writeXmippPosFile(movieId, movieName,
coordinatesName, shiftX,
shiftY)
if hasCoordinates:
self.info("Writing frame: %s" % frameName)
#TODO: there is no need to write the frame and then operate
#the input of the first operation should be the movie
imgh.convert(tuple([frame, movieName]), frameName)
if self.doRemoveDust:
self.info("Removing Dust")
self._runNoDust(frameName)
self.info("Extracting particles")
frameImages = frameRoot + '_images'
args = '-i %(frameName)s --pos %(coordinatesName)s ' \
'-o %(frameRoot)s --Xdim %(boxSize)d' % locals()
if self.doInvert:
args += " --invert"
args += " --downsampling %f " % self.factor
self.runJob('xmipp_micrograph_scissor', args)
cleanPath(frameName)
frameStk = frameRoot + '.stk'
self.info("Combining particles into one stack.")
frameMdImages.read(frameMdFile)
frameMd.read('particles@%s' % framePosFile)
frameMd.merge(frameMdImages)
for objId in frameMd:
stkIndex += 1
frameRow.readFromMd(frameMd, objId)
location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
newLocation = (stkIndex, movieStk)
imgh.convert(location, newLocation)
# Fix the name to be accesible from the Project directory
# so we know that the movie stack file will be moved
# to final particles folder
newImageName = '%d@%s' % newLocation
frameRow.setValue(md.MDL_IMAGE, newImageName)
frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId))
frameRow.setValue(md.MDL_MICROGRAPH, str(movieId))
frameRow.setValue(md.MDL_FRAME_ID, long(frame))
frameRow.setValue(md.MDL_PARTICLE_ID,
frameRow.getValue(md.MDL_ITEM_ID))
frameRow.writeToMd(movieMd, movieMd.addObject())
movieMd.addItemId()
movieMd.write(movieMdFile)
cleanPath(frameStk)
if self.doNormalize:
self._runNormalize(movieStk, numberOfFrames)
def _processMovie(self, movieId, movieName, movieFolder, shifts):###pasar shifts
movieName = os.path.join(movieFolder, movieName)
boxSize = self.boxSize.get()
# Read movie dimensions to iterate through each frame
imgh = ImageHandler()
x, y, z, n = imgh.getDimensions(movieName)
first = self.firstFrame.get()
if first <= 1:
first = 1
last = self.lastFrame.get()
if last <= 0 or last >= n:
last = n
numberOfFrames = last - first + 1
stkIndex = 0
movieStk = self._getMovieName(movieId, '.stk')
movieMdFile = self._getMovieName(movieId, '.xmd')
movieMd = md.MetaData()
frameMd = md.MetaData()
frameMdImages = md.MetaData()
frameRow = md.Row()
for frame in range(first, last+1):
# Get the frame shifts
index = frame - first
shiftX = shifts[2*index]
shiftY = shifts[2*index+1]
frameRoot = os.path.join(movieFolder, 'frame_%02d' % frame)
frameName = frameRoot + '.mrc'
frameMdFile = frameRoot + '.xmd'
framePosFile = frameRoot + '_coordinates.xmd'
coordinatesName = frameRoot + '_coordinates.xmd'
hasCoordinates = self._writeXmippPosFile(movieId, movieName, coordinatesName,
shiftX, shiftY)
if hasCoordinates:
self.info("Writing frame: %s" % frameName)
#TODO: there is no need to write the frame and then operate
#the input of the first operation should be the movie
imgh.convert(tuple([frame, movieName]), frameName)
if self.doRemoveDust:
self.info("Removing Dust")
self._runNoDust(frameName)
self.info("Extracting particles")
frameImages = frameRoot + '_images'
args = '-i %(frameName)s --pos %(coordinatesName)s ' \
'-o %(frameRoot)s --Xdim %(boxSize)d' % locals()
if self.doInvert:
args += " --invert"
args += " --downsampling %f " % self.factor
self.runJob('xmipp_micrograph_scissor', args)
cleanPath(frameName)
frameStk = frameRoot + '.stk'
self.info("Combining particles into one stack.")
frameMdImages.read(frameMdFile)
frameMd.read('particles@%s' % framePosFile)
frameMd.merge(frameMdImages)
for objId in frameMd:
stkIndex += 1
frameRow.readFromMd(frameMd, objId)
location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
newLocation = (stkIndex, movieStk)
imgh.convert(location, newLocation)
# Fix the name to be accesible from the Project directory
# so we know that the movie stack file will be moved
# to final particles folder
newImageName = '%d@%s' % newLocation
frameRow.setValue(md.MDL_IMAGE, newImageName)
frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movieId))
frameRow.setValue(md.MDL_MICROGRAPH, str(movieId))
frameRow.setValue(md.MDL_FRAME_ID, long(frame))
frameRow.setValue(md.MDL_PARTICLE_ID, frameRow.getValue(md.MDL_ITEM_ID))
frameRow.writeToMd(movieMd, movieMd.addObject())
movieMd.addItemId()
movieMd.write(movieMdFile)
cleanPath(frameStk)
if self.doNormalize:
self._runNormalize(movieStk, numberOfFrames)
def _processMovie(self, movie):
movId = movie.getObjId()
x, y, n = movie.getDim()
iniFrame, lastFrame, _ = movie.getFramesRange()
frame0, frameN = self._getRange(movie)
boxSize = self.boxSize.get()
if movie.hasAlignment() and self.applyAlignment:
shiftX, shiftY = movie.getAlignment().getShifts() # lists.
else:
shiftX = [0] * (lastFrame-iniFrame+1)
shiftY = shiftX
stkIndex = 0
movieStk = self._getMovieName(movie, '.stk')
movieMdFile = self._getMovieName(movie, '.xmd')
movieMd = md.MetaData()
frameMd = md.MetaData()
frameMdImages = md.MetaData()
frameRow = md.Row()
if self._hasCoordinates(movie):
imgh = ImageHandler()
for frame in range(frame0, frameN+1):
indx = frame-iniFrame
frameName = self._getFnRelated('frameMic',movId, frame)
frameMdFile = self._getFnRelated('frameMdFile',movId, frame)
coordinatesName = self._getFnRelated('frameCoords',movId, frame)
frameImages = self._getFnRelated('frameImages',movId, frame)
frameStk = self._getFnRelated('frameStk', movId, frame)
self._writeXmippPosFile(movie, coordinatesName,
shiftX[indx], shiftY[indx])
self.info("Writing frame: %s" % frameName)
# TODO: there is no need to write the frame and then operate
# the input of the first operation should be the movie
movieName = imgh.fixXmippVolumeFileName(movie)
imgh.convert((frame, movieName), frameName)
if self.doRemoveDust:
self.info("Removing Dust")
self._runNoDust(frameName)
self.info("Extracting particles")
args = '-i %(frameName)s --pos %(coordinatesName)s ' \
'-o %(frameImages)s --Xdim %(boxSize)d' % locals()
if self.doInvert:
args += " --invert"
if self.doBorders:
args += " --fillBorders"
args += " --downsampling %f " % self.getBoxScale()
self.runJob('xmipp_micrograph_scissor', args)
cleanPath(frameName)
self.info("Combining particles into one stack.")
frameMdImages.read(frameMdFile)
frameMd.read('particles@%s' % coordinatesName)
frameMd.merge(frameMdImages)
for objId in frameMd:
stkIndex += 1
frameRow.readFromMd(frameMd, objId)
location = xmippToLocation(frameRow.getValue(md.MDL_IMAGE))
newLocation = (stkIndex, movieStk)
imgh.convert(location, newLocation)
# Fix the name to be accessible from the Project directory
# so we know that the movie stack file will be moved
# to final particles folder
newImageName = '%d@%s' % newLocation
frameRow.setValue(md.MDL_IMAGE, newImageName)
frameRow.setValue(md.MDL_MICROGRAPH_ID, long(movId))
frameRow.setValue(md.MDL_MICROGRAPH, str(movId))
frameRow.setValue(md.MDL_FRAME_ID, long(frame))
frameRow.setValue(md.MDL_PARTICLE_ID,
frameRow.getValue(md.MDL_ITEM_ID))
frameRow.writeToMd(movieMd, movieMd.addObject())
movieMd.addItemId()
movieMd.write(movieMdFile)
cleanPath(frameStk)
if self.doNormalize:
numberOfFrames = frameN - frame0 + 1
self._runNormalize(movieStk, numberOfFrames)
def runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName):
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
img.convert(reconstructedFilteredVolume, maskedFileName)
def runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName):
from pyworkflow.em.convert import ImageHandler
img = ImageHandler()
img.convert(reconstructedFilteredVolume, maskedFileName)
