def _estimateCTF(self, mic, *args):
""" Run ctftilt with required parameters """
micFn = mic.getFileName()
micDir = self._getTmpPath('mic_%04d' % mic.getObjId())
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize(
)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
em.ImageHandler().scaleFourier(micFn, micFnMrc, downFactor)
self._params[
'scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(
pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
except Exception as ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
try:
program, args = self._getCommand(
micFn=micFnMrc,
ctftiltOut=self._getCtfOutPath(micDir),
ctftiltPSD=self._getPsdPath(micDir))
self.runJob(program, args)
except Exception as ex:
print >> sys.stderr, "ctftilt has failed with micrograph %s" % micFnMrc
# Let's notify that this micrograph have been processed
# just creating an empty file at the end (after success or failure)
open(doneFile, 'w')
# Let's clean the temporary mrc micrographs
pwutils.cleanPath(micFnMrc)
def createOutputStep(self):
micSet = self.getInputMicrographs()
ih = em.ImageHandler()
coordSet = self._createSetOfCoordinates(micSet)
if self.boxSize and self.boxSize > 0:
coordSet.setBoxSize(self.boxSize.get())
else:
coordSet.setBoxSize(self.inputReferences.get().getXDim or 100)
readSetOfCoordinates(self.getMicrographsDir(), micSet, coordSet)
coordSetAux = self._createSetOfCoordinates(micSet, suffix='_rejected')
coordSetAux.setBoxSize(coordSet.getBoxSize())
readSetOfCoordinates(self.getMicrographsDir(),
micSet,
coordSetAux,
suffix='_rejected.star')
coordSetAux.write()
# debug output
if self.writeCC:
self.createDebugOutput(suffix='_ccmax')
if self.writeFilt:
self.createDebugOutput(suffix='_pref')
if self.writeBg:
self.createDebugOutput(suffix='_bg')
if self.writeBgSub:
self.createDebugOutput(suffix='_bgfree')
if self.writeSigma:
self.createDebugOutput(suffix='_lsigma')
if self.writeMsk:
self.createDebugOutput(suffix='_mask')
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(micSet, coordSet)
def convertInputStep(self, micsId, refsId):
""" This step will take of the convertions from the inputs.
Micrographs: they will be linked if are in '.mrc' format, converted otherwise.
References: will always be converted to '.mrc' format
Mask: either converted ('.tif' format) or generated a circular one
"""
ih = em.ImageHandler()
micDir = self._getTmpPath('micrographs')
pwutils.makePath(micDir)
for mic in self.getInputMicrographs():
# Create micrograph folder
micName = mic.getFileName()
# If micrographs are in .mrc format just link it
# otherwise convert them
outMic = join(micDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
ih.convert(mic, outMic)
refDir = self._getTmpPath('templates')
pwutils.makePath(refDir)
# We will always convert the templates, since
# they can be in an stack and link will not be possible sometimes
inputRefs = self.inputReferences.get()
for i, ref in enumerate(inputRefs):
outRef = join(refDir, 'ref%02d.mrc' % (i + 1))
ih.convert(ref, outRef)
self.createInputMasks(inputRefs)
def _writeGroupFiles(self, partSet):
"""Write files that are needed by each defocus group:
- stack
- selfile
- docfile
"""
ih = em.ImageHandler()
# Keep a dict with all groups found in particles
groupDict = {}
template = self._getExtraPath('group%03d_%s.stk')
for part in partSet:
defocusGroup = self._getDefocusGroup(part)
if defocusGroup not in groupDict:
groupInfo = DefocusGroupInfo(defocusGroup, template, ih)
groupDict[defocusGroup] = groupInfo
else:
groupInfo = groupDict[defocusGroup]
groupInfo.addParticle(part)
# Write the docfile with the defocus groups information
# like the number of particles and the defocus
groupsDoc = SpiderDocFile(self._getExtraPath('sel_group.stk'), 'w+')
for gi in groupDict.values():
groupsDoc.writeValues(gi.number, gi.counter, gi.defocus)
# Convert the endianness of the stack
convertEndian(gi.stackfile, gi.counter)
# Close each group docfile
gi.close()
groupsDoc.close()
def convertBinaryVol(vol, outputDir):
""" Convert binary volume to a format read by Relion.
Params:
vol: input volume object to be converted.
outputDir: where to put the converted file(s)
Return:
new file name of the volume (convrted or not).
"""
ih = em.ImageHandler()
# This approach can be extended when
# converting from a binary file format that
# is not read from Relion
def convertToMrc(fn):
""" Convert from a format that is not read by Relion
to mrc format.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrc'))
ih.convert(fn, newFn)
return newFn
ext = vol.getFileName()
if not ext.endswith('.mrc'):
fn = convertToMrc(vol.getFileName())
else:
fn = vol.getFileName()
return fn
def _restimateCTF(self, ctfId):
""" Run Gctf with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = out
pwutils.cleanPath(psdFile)
try:
self.runJob(self._getProgram(), self._args % self._params)
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
pwutils.cleanPattern(micFnMrc)
def _restimateCTF(self, ctfId):
""" Run ctffind3 with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = out
self._params['ctffindPSD'] = psdFile
pwutils.cleanPath(psdFile)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def createOutputStep(self):
micSet = self.getInputMicrographs()
ih = em.ImageHandler()
coordSet = self._createSetOfCoordinates(micSet)
if self.boxSize == 0:
coordSet.setBoxSize(self.inputReferences.get().getDim()[0])
else:
coordSet.setBoxSize(self.boxSize.get())
for mic in micSet:
fnCoords = pwutils.replaceBaseExt(mic.getFileName(), 'txt')
fn2parse = self._getExtraPath('pik_coord', fnCoords)
print fn2parse
xdim, _, _, _ = ih.getDimensions(mic)
#xdim, ydim, _ = em.getDimensions(micSet)
#print xdim, ydim
with open(fn2parse, "r") as source:
source.readline() # Skip first line
for line in source:
tokens = line.split()
coord = em.Coordinate()
coord.setPosition(xdim - int(tokens[3]), int(tokens[2]))
coord.setMicrograph(mic)
coordSet.append(coord)
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(micSet, coordSet)
def _getRecalCommand(self, ctfModel, **kwargs):
line = ctfModel.getObjComment().split()
params = self.getRecalCtfParamsDict(ctfModel)
# get the size and the image of psd
imgPsd = ctfModel.getPsdFile()
imgh = em.ImageHandler()
size, _, _, _ = imgh.getDimensions(imgPsd)
mic = ctfModel.getMicrograph()
# Convert digital frequencies to spatial frequencies
sampling = mic.getSamplingRate()
params['step_focus'] = 1000.0
params['sampling'] = sampling
params['lowRes'] = sampling / float(line[3])
params['highRes'] = sampling / float(line[4])
params['minDefocus'] = min([float(line[0]), float(line[1])])
params['maxDefocus'] = max([float(line[0]), float(line[1])])
params['astigmatism'] = self.astigmatism.get()
params['windowSize'] = size
params['pixelAvg'] = 1 # set to 1 since we have our own downsampling
params['tiltAngle'] = self.tiltA.get()
params['tiltR'] = self.tiltR.get()
params.update(kwargs)
return self._getCommandFromParams(params)
def _resizeVolume(self, volFn):
""" Resize input volume if not of the same size of referenceVol """
refDim = self.referenceVolume.get().getXDim()
volDim = em.ImageHandler().getDimensions(volFn)
if refDim != volDim:
self.runJob('xmipp_image_resize',
"-i %s --dim %d" % (volFn, refDim))
def _prepareRecalCommand(self, ctfModel):
line = ctfModel.getObjComment().split()
self._defineRecalValues(ctfModel)
# get the size and the image of psd
imgPsd = ctfModel.getPsdFile()
imgh = em.ImageHandler()
size, _, _, _ = imgh.getDimensions(imgPsd)
mic = ctfModel.getMicrograph()
micDir = self._getMicrographDir(mic)
# Convert digital frequencies to spatial frequencies
sampling = mic.getSamplingRate()
self._params['step_focus'] = 1000.0
self._params['sampling'] = sampling
self._params['lowRes'] = sampling / float(line[3])
self._params['highRes'] = sampling / float(line[4])
self._params['minDefocus'] = min([float(line[0]), float(line[1])])
self._params['maxDefocus'] = max([float(line[0]), float(line[1])])
self._params['windowSize'] = size
if not self.useCtffind4:
self._argsCtffind3()
else:
self._params['astigmatism'] = self.astigmatism.get()
if self.findPhaseShift:
self._params['phaseShift'] = "yes"
self._params['minPhaseShift'] = self.minPhaseShift.get()
self._params['maxPhaseShift'] = self.maxPhaseShift.get()
self._params['stepPhaseShift'] = self.stepPhaseShift.get()
else:
self._params['phaseShift'] = "no"
self._argsCtffind4()
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
#Replace extension by 'mrc' cause there are some formats that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
self.runJob("xmipp_transform_downsample",
"-i %s -o %s --step %f --method fourier" % (micFn, micFnMrc, downFactor),
env=xmipp3.getEnviron())
self._params['scannedPixelSize'] = self.inputMicrographs.get().getScannedPixelSize() * downFactor
else:
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, "mrc"))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
try:
self.runJob(self._program, self._args % self._params)
except Exception, ex:
print >> sys.stderr, "ctffind has failed with micrograph %s" % micFnMrc
def convertInputStep(self, particlesId):
""" Convert all needed inputs before running the refinement script. """
partSet = self.inputParticles.get()
ih = em.ImageHandler()
stackfile = self._getPath('particles.stk')
docfile = self._getPath('docfile.stk')
doc = SpiderDocFile(docfile, 'w+')
now = 'now' #FIXME
doc.writeComment("spi/dat Generated by Scipion on %s" % now)
doc.writeComment(" KEY PSI, THE, PHI, REF#, EXP#, CUM.{ROT, SX, SY}, NPROJ, DIFF, CCROT, ROT, SX, SY, MIR-CC")
for i, img in enumerate(partSet):
ind = i + 1
ih.convert(img, (ind, stackfile))
alignRow = {ANGLE_PSI: 0.,
ANGLE_THE: 0.,
ANGLE_PHI: 0.,
SHIFTX: 0.,
SHIFTY: 0.}
alignment = img.getTransform()
if alignment is not None:
alignmentToRow(alignment, alignRow, ALIGN_PROJ)
values = [0.00, alignRow[ANGLE_THE], alignRow[ANGLE_PHI],
0.00, ind,
alignRow[ANGLE_PSI], alignRow[SHIFTX], alignRow[SHIFTY],
0.00, 0.00, 0.00, 0.00, 0.00, 0.00, 0.0]
doc.writeValues(*values)
convertEndian(stackfile, partSet.getSize())
def convertInputStep(self, moviesId, partsId):
self._ih = em.ImageHandler() # used to convert movies
inputMovies = self.getInputMovies()
firstMovie = inputMovies.getFirstItem()
self._linkOrConvertMovie = self._getConvertFunc(firstMovie)
# convert movies to mrcs and write a movie star file
writeSetOfMovies(inputMovies,
self._getPath(self._getFileName('inputMovies')),
preprocessImageRow=self._preprocessMovieRow)
inputParts = self.getParticles()
firstPart = inputParts.getFirstItem()
self._linkOrConvertPart = self._getConvertFunc(firstPart)
# convert particle set to star file
# we avoid to use the defaul convertBinaryFiles because we want
# to set a different naming convention to map between movie mrcs
# and particles stack mrcs
writeSetOfParticles(
inputParts,
self._getPath(self._getFileName('inputParts')),
None, # do not use convertBinaryFiles
fillMagnification=True,
alignType=inputParts.getAlignment(),
postprocessImageRow=self._postprocessParticleRow)
def convertInputStep(self):
self._ih = em.ImageHandler() # used to convert micrographs
# Match ctf information against the micrographs
self.ctfDict = {}
if self.ctfRelations.get() is not None:
for ctf in self.ctfRelations.get():
self.ctfDict[ctf.getMicrograph().getMicName()] = ctf.clone()
micStar = self._getPath('input_micrographs.star')
writeSetOfMicrographs(self.getInputMicrographs(),
micStar,
alignType=em.ALIGN_NONE,
preprocessImageRow=self._preprocessMicrographRow)
# We need to compute a scale factor for the coordinates if extracting
# from other micrographs with a different pixel size
micDict = {}
for mic in self.getInputMicrographs():
micDict[mic.getMicName()] = mic.getFileName()
def _getCoordsStarFile(mic):
micName = mic.getMicName()
if not micName in micDict:
return None
micFn = micDict[micName]
return self._getExtraPath(
pwutils.replaceBaseExt(micFn, 'coords.star'))
self.info("Using scale: %s" % self.getScaleFactor())
writeSetOfCoordinates(self._getExtraPath(),
self.getInputCoords(),
_getCoordsStarFile,
scale=self.getScaleFactor())
def prepareReferenceStep(self, volId):
inputMask = self.mask.get()
ih = em.ImageHandler()
fnRef = self._getExtraPath("reference.vol")
ih.convert(self.referenceVolume.get(), fnRef)
if inputMask is not None:
fnMask = self._getExtraPath("mask.vol")
ih.convert(self.mask.get(), fnMask)
self._resizeVolume(fnMask)
self._maskVolume(fnRef)
def convertInputStep(self, particlesId):
""" Convert all needed inputs before running the refinement script. """
partSet = self.inputParticles.get()
self._writeParamsFile(partSet)
self._writeGroupFiles(partSet)
# Convert the input volume
volPath = self._getExtraPath('vol01.vol')
em.ImageHandler().convert(self.input3DReference.get(), volPath)
pwutils.moveFile(volPath, volPath.replace('.vol', '.stk'))
self._writeRefinementScripts()
def _prepareRecalCommand(self, ctfModel):
if self.recalculate:
self._defineRecalValues(ctfModel)
self._createFilenameTemplates()
self._program = 'xmipp_ctf_estimate_from_psd'
self._args = "--psd %(psdFn)s "
line = ctfModel.getObjComment().split()
# get the size and the image of psd
imgPsd = ctfModel.getPsdFile()
psdFile = pwutils.path.basename(imgPsd)
imgh = em.ImageHandler()
size, _, _, _ = imgh.getDimensions(imgPsd)
mic = ctfModel.getMicrograph()
micDir = self._getMicrographDir(mic)
fnCTFparam = self._getFileName('ctfparam', micDir=micDir)
mdCTFParam = md.MetaData(fnCTFparam)
downFactor = mdCTFParam.getValue(md.MDL_CTF_DOWNSAMPLE_PERFORMED,
mdCTFParam.firstObject())
# cleanPath(fnCTFparam)
params2 = {'psdFn': join(micDir, psdFile),
'defocusU': float(line[0]),
'defocusV': float(line[1]),
'angle': line[2],
}
self._params = dict(self._params.items() + params2.items())
# Mapping between base protocol parameters and the package specific
# command options
self.__params = {'sampling_rate': self._params['samplingRate']
* downFactor,
'downSamplingPerformed': downFactor,
'kV': self._params['voltage'],
'Cs': self._params['sphericalAberration'],
'min_freq': line[3],
'max_freq': line[4],
'defocusU': self._params['defocusU'],
'defocusV': self._params['defocusV'],
'azimuthal_angle': self._params['angle'],
'Q0': self._params['ampContrast'],
'defocus_range': 5000,
'ctfmodelSize': size
}
for par, val in self.__params.iteritems():
self._args += " --%s %s" % (par, str(val))
def runGautomatch(micName, refStack, workDir, args, env=None):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
if refStack is not None:
args = ' %s --T %s %s' % (outMic, refStack, args)
else:
args = ' %s %s' % (outMic, args)
pwutils.runJob(None, getProgram(), args, env=env)
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def convertBinaryFiles(imgSet, outputDir):
""" Convert binary images files to a format read by EMAN.
Params:
imgSet: input image set to be converted.
outputDir: where to put the converted file(s)
Return:
A dictionary with old-file as key and new-file as value
If empty, not conversion was done.
"""
filesDict = {}
ih = em.ImageHandler()
# This approach can be extended when
# converting from a binary file format that
# is not read from Relion
def linkMrcToMrcs(fn):
""" Just create a link named .mrc to Eman understand
that it is a mrc binary stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'mrcs'))
createLink(fn, newFn)
return newFn
def convertStack(fn):
""" Convert from a format that is not read by Relion
to an spider stack.
"""
newFn = join(outputDir, replaceBaseExt(fn, 'stk'))
ih.convertStack(fn, newFn)
return newFn
ext = imgSet.getFirstItem().getFileName()
if ext.endswith('.mrc'):
mapFunc = linkMrcToMrcs
ismrcs = True
# elif ext.endswith('.hdf'): # assume eman .hdf format
# mapFunc = convertStack
else:
mapFunc = None
ismrcs = False
if mapFunc is not None:
for fn in imgSet.getFiles():
filesDict[fn] = mapFunc(fn) # convert and map new filename
return ismrcs, filesDict
def convertInputStep(self, inputId, avgId, volId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameters just to force redo of convert if
the input particles are changed.
"""
inputSet = self.inputSet.get()
imgStar = self._getFileName('input_particles')
refStar = self._getFileName('input_refs')
# Pass stack file as None to avoid write the images files
self.info("Converting set from '%s' into '%s'" %
(inputSet.getFileName(), imgStar))
refSet = None
# case refine3D
if self.isInputClasses():
refSet = self.inputSet.get() # 2D or 3D classes
else:
if self.isInputAutoRefine():
em.ImageHandler().convert(self.referenceVolume.get(),
self._getFileName('input_refvol'))
else: # Autopicking case
refSet = self.referenceAverages.get()
self.classDict = {}
if refSet:
self.info("Converting reference from '%s' into %s" %
(refSet.getFileName(), refStar))
# Compute class mapping
classList = [cls.getObjId() for cls in refSet]
classList.sort()
for i, c in enumerate(classList):
self.classDict[c] = i + 1
writeReferences(refSet,
removeExt(refStar),
postprocessImageRow=self._updateClasses)
# Write particles star file
allParticles = self._allParticles(iterate=False)
writeSetOfParticles(allParticles,
imgStar,
self._getPath(),
postprocessImageRow=self._postProcessImageRow)
def _estimateCTF(self, micFn, micDir, micName):
""" Run ctffind, 3 or 4, with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
scannedPixelSize = self.inputMicrographs.get().getScannedPixelSize(
)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' because there are some formats
# that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
args = "-i %s -o %s --step %f --method fourier" % (
micFn, micFnMrc, downFactor)
self.runJob("xmipp_transform_downsample",
args,
env=xmipp3.getEnviron())
self._params[
'scannedPixelSize'] = scannedPixelSize * downFactor
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
micFnMrc = self._getTmpPath(
pwutils.replaceBaseExt(micFn, "mrc"))
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['ctffindOut'] = self._getCtfOutPath(micDir)
self._params['ctffindPSD'] = self._getPsdPath(micDir)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def runGempicker(micName, workingDir, useGPU, args):
# We convert the input micrograph on demand if not in .mrc
outMic = os.path.join(workingDir, pwutils.replaceBaseExt(micName, 'mrc'))
if micName.endswith('.mrc'):
pwutils.createLink(micName, outMic)
else:
em.ImageHandler().convert(micName, outMic)
refDir = join(workingDir, 'templates')
maskSchDir = join(workingDir, 'maskSch')
args += ' --dirTgt=%s --dirSch=%s --dirMskSch=%s ' % (workingDir, refDir,
maskSchDir)
# Run Gempicker:
for mode in [0, 1]:
pwutils.runJob(None, getProgram(useGPU), args + ' --mode=%d' % mode)
# After picking we can remove the temporary file.
pwutils.cleanPath(outMic)
def convertMask(img, outputDir, newDim=None):
""" Convert binary mask to a format read by Relion and truncate the
values between 0-1 values, due to Relion only support masks with this
values (0-1).
Params:
img: input image to be converted.
outputDir: where to put the converted file(s)
Return:
new file name of the mask.
"""
ih = em.ImageHandler()
imgFn = getImageLocation(img.getLocation())
newFn = join(outputDir, pwutils.replaceBaseExt(imgFn, 'mrc'))
ih.truncateMask(imgFn, newFn, newDim=newDim)
return newFn
def _insertInitialSteps(self):
self._setupBasicProperties()
# used to convert micrographs if not in .mrc format
self._ih = em.ImageHandler()
# dict to map between micrographs and its coordinates file
self._micCoordStarDict = {}
# When no streaming, it doesn't make sense the default value of
# batch size = 1, so let's use 0 to extract all micrographs at once
if not self._isStreamOpen() and self._getStreamingBatchSize() == 1:
self.info("WARNING: The batch size of 1 does not make sense when "
"not in streaming...changed value to 0 (extract all).")
self.streamingBatchSize.set(0)
return []
def createInputMasks(self, inputRefs, maskSchDir):
""" Create the needed mask for picking.
We should either generate a circular mask,
or convert the inputs (one or just one per reference 2d)
"""
ih = em.ImageHandler()
if self.maskType == MASK_CIRCULAR:
if self.maskRadius < 0: # usually -1
radius = inputRefs.getDim()[0] / 2 # use half of input dim
else:
radius = self.maskRadius.get()
outMask = join(maskSchDir, 'ref01.tif')
ih.createCircularMask(radius, inputRefs.getFirstItem(), outMask)
else:
for i, mask in enumerate(self.inputMasks):
outMask = join(maskSchDir, 'ref%02d.tif' % (i + 1))
ih.convert(mask.get(), outMask)
def initIterStep(self, iterN):
""" Prepare files and directories for the current iteration """
self._createIterWorkingDir(
iterN) # create the working directory for the current iteration.
prevIter = iterN - 1
if iterN == 1:
vol = self.input3DReference.get()
imgFn = self._getFileName('particles')
volFn = self._getFileName('init_vol')
refVol = self._getFileName(
'ref_vol', iter=iterN) # reference volume of the step.
#TODO check if the input is already a single mrc stack
self.writeParticlesByMic(imgFn)
em.ImageHandler().convert(
vol.getLocation(),
volFn) # convert the reference volume into a mrc volume
copyFile(
volFn,
refVol) #Copy the initial volume in the current directory.
for ref in self._allRefs():
refVol = self._getFileName(
'ref_vol_class', iter=iterN,
ref=ref) # reference volume of the step.
iterVol = self._getFileName('iter_vol_class', iter=iterN,
ref=ref) # refined volumes of the step
if iterN == 1:
copyFile(
volFn,
iterVol) #Copy the initial volume in current directory.
else:
self._splitParFile(iterN, ref, self.cpuList[ref - 1])
prevIterVol = self._getFileName(
'iter_vol_class', iter=prevIter,
ref=ref) # volumes of the previous iteration
copyFile(prevIterVol,
refVol) #Copy the reference volume as refined volume.
copyFile(
refVol,
iterVol) #Copy the reference volume as refined volume.
def writeReferences(inputSet, outputRoot, useBasename=False, **kwargs):
"""
Write references star and stack files from SetOfAverages or SetOfClasses2D/3D.
Params:
inputSet: the input SetOfParticles to be converted
outputRoot: where to write the output files.
basename: If True, use the basename of the stack for setting path.
"""
refsMd = md.MetaData()
stackFile = outputRoot + '.stk'
stackName = basename(stackFile) if useBasename else stackFile
starFile = outputRoot + '.star'
ih = em.ImageHandler()
row = md.Row()
def _convert(item, i, convertFunc):
index = i + 1
ih.convert(item, (index, stackFile))
item.setLocation(index, stackName)
convertFunc(item, row, **kwargs)
row.writeToMd(refsMd, refsMd.addObject())
if isinstance(inputSet, em.SetOfAverages):
for i, img in enumerate(inputSet):
_convert(img, i, particleToRow)
elif isinstance(inputSet, em.SetOfClasses2D):
for i, rep in enumerate(inputSet.iterRepresentatives()):
_convert(rep, i, particleToRow)
elif isinstance(inputSet, em.SetOfClasses3D):
for i, rep in enumerate(inputSet.iterRepresentatives()):
_convert(rep, i, imageToRow)
elif isinstance(inputSet, em.SetOfVolumes):
for i, vol in enumerate(inputSet):
_convert(vol, i, imageToRow)
else:
raise Exception('Invalid object type: %s' % type(inputSet))
refsMd.write(starFile)
def _estimateCTF(self, micFn, micDir, micName):
""" Run Gctf with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
ih = em.ImageHandler()
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(
pwutils.removeBaseExt(micFn) + '_EPA.log')
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self.inputMicrographs.get().getScannedPixelSize(
) * downFactor
self._params['scannedPixelSize'] = sps
else:
if ih.existsLocation(micFn):
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = self._getCtfOutPath(micDir)
print >> sys.stdout, "Updated the paths dictionary with %s %s %s" % (
micFnMrc, micDir, self._getCtfOutPath)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
def createOutputStep(self):
inputAverages = self.inputAverages.get()
imgxdim = inputAverages.getXDim()
fnVol = self._getExtraPath("refine.mrc")
volxdim, _, _, _ = em.ImageHandler().getDimensions(fnVol)
if imgxdim != volxdim:
fnVolOut = self._getPath("volume.vol")
args3 = "-i %s --dim %d -o %s" % (fnVol, imgxdim, fnVolOut)
#TODO maybe change later using Python binding to resize images
import pyworkflow.em.packages.xmipp3 as xmipp3
self.runJob("xmipp_image_resize", args3, env=xmipp3.getEnviron())
else:
fnVolOut = fnVol
vol = em.Volume()
vol.setLocation(fnVolOut)
vol.setSamplingRate(inputAverages.getSamplingRate())
self._defineOutputs(outputVol=vol)
self._defineSourceRelation(self.inputAverages, vol)
