def _restimateCTF(self, ctfId):
""" Run Gctf with required parameters """
ctfModel = self.recalculateSet[ctfId]
mic = ctfModel.getMicrograph()
micFn = mic.getFileName()
micDir = self._getMicrographDir(mic)
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')
out = self._getCtfOutPath(micDir)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.cleanPath(out)
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
em.ImageHandler().convert(micFn, micFnMrc, em.DT_FLOAT)
# Update _params dictionary
self._prepareRecalCommand(ctfModel)
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = out
pwutils.cleanPath(psdFile)
try:
self.runJob(self._getProgram(), self._args % self._params)
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
pwutils.cleanPattern(micFnMrc)
def _renameFiles(self, pattern1, pattern2):
# find files by pattern1, move and rename them by replacing pattern2
filesList = sorted(glob(self._getPath(pattern1)))
for fn in filesList:
oldFn = os.path.basename(fn)
newFn = pwutils.join(self._getExtraPath('shiny'), oldFn.replace(pattern2, ''))
pwutils.moveFile(fn, newFn)
def refineAnglesStep(self):
fnTmpDir = self._getTmpPath()
fnDirectional = self._getDirectionalClassesFn()
inputParticles = self.inputParticles.get()
newTs = self.readInfoField(self._getExtraPath(),"sampling",xmippLib.MDL_SAMPLINGRATE)
newXdim = self.readInfoField(self._getExtraPath(),"size",xmippLib.MDL_XSIZE)
# Generate projections
fnGallery=join(fnTmpDir,"gallery.stk")
fnGalleryMd=join(fnTmpDir,"gallery.doc")
fnVol = self._getInputVolFn()
args="-i %s -o %s --sampling_rate %f --sym %s"%\
(fnVol,fnGallery,5.0,self.symmetryGroup)
args+=" --compute_neighbors --angular_distance -1 --experimental_images %s"%fnDirectional
self.runJob("xmipp_angular_project_library",args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
# Global angular assignment
maxShift=0.15*newXdim
args='-i %s --initgallery %s --maxShift %d --odir %s --dontReconstruct --useForValidation 0'%\
(fnDirectional,fnGalleryMd,maxShift,fnTmpDir)
self.runJob('xmipp_reconstruct_significant',args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
fnAngles = join(fnTmpDir,"angles_iter001_00.xmd")
self.runJob("xmipp_metadata_utilities","-i %s --operate drop_column ref"%fnAngles,numberOfMpi=1)
self.runJob("xmipp_metadata_utilities","-i %s --set join %s ref2"%(fnAngles,fnDirectional),numberOfMpi=1)
# Local angular assignment
fnAnglesLocalStk = self._getPath("directional_local_classes.stk")
args="-i %s -o %s --sampling %f --Rmax %d --padding %d --ref %s --max_resolution %f --applyTo image1 --Nsimultaneous %d"%\
(fnAngles,fnAnglesLocalStk,newTs,newXdim/2,2,fnVol,self.targetResolution,8)
args+=" --optimizeShift --max_shift %f"%maxShift
args+=" --optimizeAngles --max_angular_change %f"%self.angularDistance
self.runJob("xmipp_angular_continuous_assign2",args,numberOfMpi=self.numberOfMpi.get()*self.numberOfThreads.get())
moveFile(self._getPath("directional_local_classes.xmd"),self._getDirectionalClassesFn())
cleanPattern(self._getExtraPath("direction_*"))
def convertToPseudoAtomsStep(self, inputFn, fnMask, sampling, prefix=''):
pseudoatoms = 'pseudoatoms%s' % prefix
outputFn = self._getPath(pseudoatoms)
sigma = sampling * self.pseudoAtomRadius.get()
targetErr = self.pseudoAtomTarget.get()
#volume-to-pseudoatom conversion was not MPI-parallelized and the number of MPIs was removed from the gui
#nthreads = self.numberOfThreads.get() * self.numberOfMpi.get()
nthreads = self.numberOfThreads.get()
params = "-i %(inputFn)s -o %(outputFn)s --sigma %(sigma)f --thr " \
"%(nthreads)d "
params += "--targetError %(targetErr)f --sampling_rate %(sampling)f " \
"-v 2 --intensityColumn Bfactor"
if fnMask:
params += " --mask binary_file %(fnMask)s"
self.runJob("xmipp_volume_to_pseudoatoms", params % locals())
for suffix in ["_approximation.vol", "_distance.hist"]:
moveFile(self._getPath(pseudoatoms + suffix),
self._getExtraPath(pseudoatoms + suffix))
self.runJob(
"xmipp_image_convert",
"-i %s_approximation.vol -o %s_approximation.mrc -t vol" %
(self._getExtraPath(pseudoatoms), self._getExtraPath(pseudoatoms)))
self.runJob(
"xmipp_image_header",
"-i %s_approximation.mrc --sampling_rate %f" %
(self._getExtraPath(pseudoatoms), sampling))
cleanPattern(self._getPath(pseudoatoms + '_*'))
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and self.checkedMics == self.processedMics
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
newFiles = getFiles(self._getTmpPath())
if newFiles or self.finished: # when finished to close the output set
outSet = self._loadOutputSet(SetOfCoordinates,
'coordinates.sqlite')
for fnTmp in newFiles:
coords = np.loadtxt(fnTmp)
moveFile(fnTmp, self._getExtraPath())
if coords.size == 2: # special case with only one coordinate
coords = [coords]
for coord in coords:
newCoord = Coordinate()
micrographs = self.getMainInput().getMicrographs()
newCoord.setMicrograph(micrographs[self.getMicId(fnTmp)])
newCoord.setPosition(coord[0], coord[1])
outSet.append(newCoord)
firstTime = not self.hasAttribute(self.outputName)
self._updateOutputSet(self.outputName, outSet, streamMode)
if firstTime:
self.defineRelations(outSet)
outSet.close()
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(STATUS_NEW)
def _renameFiles(self, pattern1, pattern2):
# find files by pattern1, move and rename them by replacing pattern2
filesList = sorted(glob(self._getPath(pattern1)))
for fn in filesList:
oldFn = os.path.basename(fn)
newFn = pwutils.join(self._getExtraPath('shiny'),
oldFn.replace(pattern2, ''))
pwutils.moveFile(fn, newFn)
def readPartsFromMics(self, micList, outputParts):
""" Read the particles extract for the given list of micrographs
and update the outputParts set with new items.
"""
relionToLocation = relion.convert.relionToLocation
p = Particle()
p._rlnOpticsGroup = Integer()
acq = self.getInputMicrographs().getAcquisition()
# JMRT: Ideally I would like to disable the whole Acquisition for each
# particle row, but the SetOfImages will set it again.
# Another option could be to disable in the set, but then in
# streaming, other protocols might get the wrong optics info
pAcq = Acquisition(magnification=acq.getMagnification(),
voltage=acq.getVoltage(),
amplitudeContrast=acq.getAmplitudeContrast(),
sphericalAberration=acq.getSphericalAberration())
p.setAcquisition(pAcq)
tmp = self._getTmpPath()
extra = self._getExtraPath()
for mic in micList:
posSet = set()
coordDict = {self._getPos(c): c
for c in self.coordDict[mic.getObjId()]}
del self.coordDict[mic.getObjId()]
ogNumber = mic.getAttributeValue('_rlnOpticsGroup', 1)
partsStar = self.__getMicFile(mic, '_extract.star', folder=tmp)
partsTable = relion.convert.Table(fileName=partsStar)
stackFile = self.__getMicFile(mic, '.mrcs', folder=tmp)
endStackFile = self.__getMicFile(mic, '.mrcs', folder=extra)
pwutils.moveFile(stackFile, endStackFile)
for part in partsTable:
pos = (int(float(part.rlnCoordinateX)),
int(float(part.rlnCoordinateY)))
if pos in posSet:
print("Duplicate coordinate at: %s, IGNORED. " % str(pos))
coord = None
else:
coord = coordDict.get(pos, None)
if coord is not None:
# scale the coordinates according to particles dimension.
coord.scale(self.getBoxScale())
p.copyObjId(coord)
idx, fn = relionToLocation(part.rlnImageName)
p.setLocation(idx, endStackFile)
p.setCoordinate(coord)
p.setMicId(mic.getObjId())
p.setCTF(mic.getCTF())
p._rlnOpticsGroup.set(ogNumber)
outputParts.append(p)
posSet.add(pos)
def filterStep(self, args):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particlesStk = self._getPath("particles.spi")
tmpStk = particlesStk.replace(".spi", "_tmp.spi")
self.runJob("bfilter", args + " %s %s" % (particlesStk, tmpStk))
pwutils.moveFile(tmpStk, particlesStk.replace(".spi", ".stk")) # just we prefer stk as stack of spider images
pwutils.cleanPath(particlesStk)
def filterStep(self, args):
""" Apply the selected filter to particles.
Create the set of particles.
"""
particlesStk = self._getPath('particles.spi')
tmpStk = particlesStk.replace('.spi', '_tmp.spi')
self.runJob('bfilter', args + ' %s %s' % (particlesStk, tmpStk))
pwutils.moveFile(tmpStk, particlesStk.replace('.spi', '.stk')) # just we prefer stk as stack of spider images
pwutils.cleanPath(particlesStk)
def createReport(self):
fnTex = "report.tex"
fhTex = open(self._getExtraPath(fnTex), "w")
template = """
\\documentclass[12pt]{article}
\\usepackage{amsmath,amsthm,amssymb,amsfonts}
\\usepackage{graphicx}
\\usepackage{pdfpages}
\\DeclareGraphicsExtensions{.pdf,.png,.jpg}
\\begin{document}
\\title{User Report}
\\author{Created by Scipion}
\\maketitle
"""
fhTex.write(template)
fnDir = self.filesPath.get()
if not os.path.isdir(fnDir):
fnDir = os.path.basename(fnDir)
for fileName in sorted(glob.glob(os.path.join(fnDir, "*"))):
fnDest = os.path.basename(fileName).lower()
fnDest = fnDest.replace(" ", "_")
fnDest = fnDest.replace(":", "_")
fnDest = fnDest.replace(";", "_")
pwutils.copyFile(fileName, self._getExtraPath(fnDest))
if fnDest.endswith(".tex") or fnDest.endswith(".txt"):
fhTex.write("\\input{%s}\n" % fnDest)
fhTex.write("\n")
elif fnDest.endswith(".png") or fnDest.endswith(".jpg"):
fhTex.write("\\begin{center}\n")
fhTex.write("\\includegraphics[width=14cm]{%s}\n" % fnDest)
fhTex.write("\\end{center}\n")
fhTex.write("\n")
elif fnDest.endswith(".pdf"):
fhTex.write("\\includepdf[pages=-]{%s}\n" % fnDest)
fhTex.write("\\clearpage\n")
fhTex.write("\n")
template = """
\\end{document}
"""
fhTex.write(template)
fhTex.close()
args = "-interaction=nonstopmode " + fnTex
self.runJob(PDFLATEX, args, cwd=self._getExtraPath())
fnPDF = self._getExtraPath("report.pdf")
if os.path.exists(fnPDF):
pwutils.moveFile(fnPDF, self._getPath("report.pdf"))
else:
raise Exception("PDF file was not produced.")
def _fixMovie(self, movie):
if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport():
outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
movieFn = outputMicFn.replace('_aligned_mic.mrc',
'_aligned_mic_Stk.mrc')
pwutils.moveFile(movieFn, outputMovieFn)
if self.useMotioncor2 and not self.doSaveUnweightedMic:
fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
pwutils.cleanPath(fnToDelete)
def _fixMovie(self, movie):
if self.doSaveMovie and self.useMotioncor2 and self._isOutStackSupport(
):
outputMicFn = self._getExtraPath(self._getOutputMicName(movie))
outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
movieFn = outputMicFn.replace('_aligned_mic.mrc',
'_aligned_mic_Stk.mrc')
pwutils.moveFile(movieFn, outputMovieFn)
if self.useMotioncor2 and not self.doSaveUnweightedMic:
fnToDelete = self._getExtraPath(self._getOutputMicName(movie))
pwutils.cleanPath(fnToDelete)
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the micrographs in
# the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
# create tmp set with all mics from coords set
coordMics = SetOfMicrographs(filename=':memory:')
coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())
for micU, micT in izip(
self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micU.cleanObjId()
micT.cleanObjId()
coordMics.append(micU)
coordMics.append(micT)
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeExt(mic.getMicName())] = micPos
# now match micDict and inputMics
if any(
pwutils.removeExt(mic.getMicName()) in micDict
for mic in self.inputMics):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception(
'Could not match input micrographs and coordinates '
'by micName.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' %
(micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _convertInput(self, imgSet):
newDim = self._getNewDim()
bg = int(newDim / 2)
args = '--operate_on %s --operate_out %s --norm --bg_radius %d'
params = args % (self._getFileName('input_star'),
self._getFileName('preprocess_parts_star'), bg)
self.runJob(self._getProgram(program='relion_preprocess'), params)
from pyworkflow.utils import moveFile
moveFile(self._getFileName('preprocess_parts'),
self._getTmpPath('particles_subset.mrcs'))
def convertInputStep(self, particlesId):
""" Convert all needed inputs before running the refinement script. """
partSet = self.inputParticles.get()
self._writeParamsFile(partSet)
self._writeGroupFiles(partSet)
# Convert the input volume
volPath = self._getExtraPath('vol01.vol')
em.ImageHandler().convert(self.input3DReference.get(), volPath)
pwutils.moveFile(volPath, volPath.replace('.vol', '.stk'))
self._writeRefinementScripts()
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the
# micrographs in the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
# create tmp set with all mics from coords set
coordMics = SetOfMicrographs(filename=':memory:')
coordMics.copyInfo(self.inputCoords.getUntilted().getMicrographs())
for micU, micT in izip(self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micU.cleanObjId()
micT.cleanObjId()
coordMics.append(micU)
coordMics.append(micT)
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeExt(mic.getMicName())] = micPos
# now match micDict and inputMics
if any(pwutils.removeExt(mic.getMicName()) in micDict for mic in self.inputMics):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception('Could not match input micrographs and coordinates '
'by micName.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' % (micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _pickMicrographStep(self, mics, args):
""" Main func that runs the picking job.
:param mics: micrograph list
:param args: programs args
"""
for mic in mics:
micName = mic.getFileName()
outMic = os.path.join(self._getTmpPath(),
pwutils.replaceBaseExt(micName, 'mrc'))
ctf = self.ctfDict[mic.getMicName()]
args.update({'micName': outMic,
'logFn': self._getLogFn(mic),
'outStack': self._getStackFn(mic),
'phaseShift': ctf.getPhaseShift() or 0.0,
'defocusU': ctf.getDefocusU(),
'defocusV': ctf.getDefocusV(),
'defocusAngle': ctf.getDefocusAngle()
})
if self.pickType == 1:
args.update({
'refsFn': self._getExtraPath('references.mrc'),
'useRadAvg': 'YES' if self.useRadAvg else 'NO',
'rotateRef': self.rotateRef.get(),
})
argsStr = self._getArgsStr()
cmdArgs = argsStr % args
try:
self.runJob(self._getProgram(), cmdArgs,
env=Plugin.getEnviron())
# Move output coords from tmp to extra
pltFn = pwutils.replaceExt(self._getStackFn(mic), 'plt')
pwutils.moveFile(pltFn, self._getPltFn(mic))
# Clean tmp folder
pwutils.cleanPath(outMic)
pwutils.cleanPath(self._getLogFn(mic))
pwutils.cleanPath(self._getStackFn(mic))
except Exception as e:
self.error("ERROR: Picking has failed for %s. %s" % (
outMic, self._getErrorFromPickerTxt(mic, e)))
def _processMovie(self, movieId, movieName, movieFolder):
inputName = movieName
micName = self._getMicName(movieId)
logFile = self._getLogFile(movieId)
gainFile = self.inputMovies.get().getGain()
gpuId = self.gpuId.get()
# TODO Check better way to handle gain correction
# if gainFile is not None:
# # Apply the gain correction to flat the raw movie
# correctedName = movieName.replace('.mrc', '_corrected.mrc')
#
# self.runJob('dosefgpu_flat',
# '%(inputName)s %(correctedName)s %(gainFile)s %(gpuId)s' % locals(),
# cwd=movieFolder)
#
# inputName = correctedName
args = {
'-crx': self.cropOffsetX.get(),
'-cry': self.cropOffsetY.get(),
'-cdx': self.cropDimX.get(),
'-cdy': self.cropDimY.get(),
'-bin': self.binFactor.get(),
'-nst': self.alignFrame0.get(),
'-ned': self.alignFrameN.get(),
'-nss': self.sumFrame0.get(),
'-nes': self.sumFrameN.get(),
'-gpu': gpuId,
'-flg': logFile,
}
#TODO: check the gain can be handle in dosefgpu_driftcoor program
#if gainFile is not None:
# args['-fgr'] = gainFile
command = '%(inputName)s -fcs %(micName)s ' % locals()
command += ' '.join(['%s %s' % (k, v) for k, v in args.iteritems()])
command += ' ' + self.extraParams.get()
self.runJob('dosefgpu_driftcorr', command, cwd=movieFolder)
# Move the micrograph and alignment text file
# before clean of movie folder
moveFile(join(movieFolder, micName), self._getExtraPath())
moveFile(join(movieFolder, logFile), self._getExtraPath())
def writePosFilesStep(self):
""" Write the pos file for each micrograph in metadata format
(both untilted and tilted). """
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getUntilted(),
scale=self.getBoxScale())
writeSetOfCoordinates(self._getExtraPath(),
self.inputCoords.getTilted(),
scale=self.getBoxScale())
# We need to find the mapping by micName (without ext) between the
# micrographs in the SetOfCoordinates and the Other micrographs
if self._micsOther():
micDict = {}
for micU, micT in izip(
self.inputCoords.getUntilted().getMicrographs(),
self.inputCoords.getTilted().getMicrographs()):
micBaseU = pwutils.removeBaseExt(micU.getFileName())
micPosU = self._getExtraPath(micBaseU + ".pos")
micDict[pwutils.removeExt(micU.getMicName())] = micPosU
micBaseT = pwutils.removeBaseExt(micT.getFileName())
micPosT = self._getExtraPath(micBaseT + ".pos")
micDict[pwutils.removeExt(micT.getMicName())] = micPosT
# now match micDict and other mics (in self.ctfDict)
if any(
pwutils.removeExt(mic.getMicName()) in micDict
for mic in self.ctfDict):
micKey = lambda mic: pwutils.removeExt(mic.getMicName())
else:
raise Exception(
'Could not match input micrographs and coordinates '
'by micName.')
for mic in self.ctfDict:
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' %
(micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def _processMovie(self, movieId, movieName, movieFolder):
inputName = movieName
micName = self._getMicName(movieId)
logFile = self._getLogFile(movieId)
gainFile = self.inputMovies.get().getGain()
gpuId = self.gpuId.get()
# TODO Check better way to handle gain correction
# if gainFile is not None:
# # Apply the gain correction to flat the raw movie
# correctedName = movieName.replace('.mrc', '_corrected.mrc')
#
# self.runJob('dosefgpu_flat',
# '%(inputName)s %(correctedName)s %(gainFile)s %(gpuId)s' % locals(),
# cwd=movieFolder)
#
# inputName = correctedName
args = {'-crx': self.cropOffsetX.get(),
'-cry': self.cropOffsetY.get(),
'-cdx': self.cropDimX.get(),
'-cdy': self.cropDimY.get(),
'-bin': self.binFactor.get(),
'-nst': self.alignFrame0.get(),
'-ned': self.alignFrameN.get(),
'-nss': self.sumFrame0.get(),
'-nes': self.sumFrameN.get(),
'-gpu': gpuId,
'-flg': logFile,
}
#TODO: check the gain can be handle in dosefgpu_driftcoor program
#if gainFile is not None:
# args['-fgr'] = gainFile
command = '%(inputName)s -fcs %(micName)s ' % locals()
command += ' '.join(['%s %s' % (k, v) for k, v in args.iteritems()])
command += ' ' + self.extraParams.get()
self.runJob('dosefgpu_driftcorr', command, cwd=movieFolder)
# Move the micrograph and alignment text file
# before clean of movie folder
moveFile(join(movieFolder, micName), self._getExtraPath())
moveFile(join(movieFolder, logFile), self._getExtraPath())
def _combineTrainDataFiles(pattern, outputFile):
files = glob.glob(pattern)
if len(files) == 1:
moveFile(files[0], outputFile)
else:
# Create a dictionary with the data fields contained in each npz file
dataDict = {}
with np.load(files[0]) as data:
for field in data.files:
dataDict[field] = []
# Read and combine the data from all files
for i, name in enumerate(files):
with np.load(name) as data:
for field in data.files:
dataDict[field].append(data[field])
# Save the combined data into a npz file
np.savez(outputFile, **dataDict)
def _moveOutput(self, movie):
""" Move output from tmp to extra folder. """
def _moveToExtra(fn):
""" Move file from movies tmp folder to extra """
pwutils.moveFile(self._getCwdPath(movie, fn),
self._getExtraPath(fn))
_moveToExtra(self._getMovieLogFile(movie))
_moveToExtra(self._getMicFn(movie))
if self._doSaveUnweightedMic():
_moveToExtra(self._getOutputMicName(movie))
if self.doSaveMovie:
outputMicFn = self._getCwdPath(movie,
self._getOutputMicName(movie))
outputMovieFn = self._getExtraPath(self._getOutputMovieName(movie))
movieFn = outputMicFn.replace('_aligned_mic.mrc',
'_aligned_mic_Stk.mrc')
pwutils.moveFile(movieFn, outputMovieFn)
def createMaskStep(self):
inputObj = self.inputObj.get()
maskSrc=self.maskFile.get()
basename = os.path.basename(maskSrc)
maskDst = self._getPath(basename)
moveFile(maskSrc, maskDst)
samplingRate = None
if(hasattr(inputObj, "getSamplingRate")):
samplingRate = inputObj.getSamplingRate()
else:
for key, attr in inputObj.iterInputAttributes():
if hasattr(attr.get(), "getSamplingRate"):
samplingRate = attr.get().getSamplingRate()
if not samplingRate:
raise Exception("sampling rate required")
mask = Mask()
mask.setFileName(maskDst)
mask.setSamplingRate(samplingRate)
self._defineOutputs(outputMask=mask)
self._defineSourceRelation(self.inputObj, self.outputMask)
def createMaskStep(self):
inputObj = self.inputObj.get()
maskSrc = self.maskFile.get()
basename = os.path.basename(maskSrc)
maskDst = self._getPath(basename)
moveFile(maskSrc, maskDst)
samplingRate = None
if hasattr(inputObj, "getSamplingRate"):
samplingRate = inputObj.getSamplingRate()
else:
for key, attr in inputObj.iterInputAttributes():
if hasattr(attr.get(), "getSamplingRate"):
samplingRate = attr.get().getSamplingRate()
if not samplingRate:
raise Exception("sampling rate required")
mask = emobj.Mask()
mask.setFileName(maskDst)
mask.setSamplingRate(samplingRate)
self._defineOutputs(outputMask=mask)
self._defineSourceRelation(self.inputObj, self.outputMask)
def _estimateCtfList(self, micList, *args, **kwargs):
""" Estimate several micrographs at once, probably a bit more
efficient. """
try:
micPath = self._getMicrographDir(micList[0])
if len(micList) > 1:
micPath += ('-%04d' % micList[-1].getObjId())
pwutils.makePath(micPath)
ih = emlib.image.ImageHandler()
for mic in micList:
micFn = mic.getFileName()
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
micFnMrc = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self._params['scannedPixelSize'] * downFactor
kwargs['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
program, args = self._gctfProgram.getCommand(**kwargs)
args += ' %s/*.mrc' % micPath
self.runJob(program, args) # , env=gctf.Plugin.getEnviron())
def _getFile(micBase, suffix):
return os.path.join(micPath, micBase + suffix)
for mic in micList:
micFn = mic.getFileName()
micBase = pwutils.removeBaseExt(micFn)
micFnMrc = _getFile(micBase, '.mrc')
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = _getFile(micBase, self._gctfProgram.getExt())
micFnCtfLog = _getFile(micBase, '_gctf.log')
micFnCtfFit = _getFile(micBase, '_EPA.log')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.cleanPath(micPath)
except:
print("ERROR: Gctf has failed on %s/*.mrc" % micPath)
import traceback
traceback.print_exc()
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _moveFiles(self, movie):
# It really annoying that Relion default names changes if you use DW or not
# if use DW, the default name are DW and the others noDW
if self.doDW:
pwutils.moveFile(
self._getMovieOutFn(movie, '.mrc'),
self._getExtraPath(self._getOutputMicWtName(movie)))
if self.saveNonDW:
pwutils.moveFile(
self._getMovieOutFn(movie, '_noDW.mrc'),
self._getExtraPath(self._getOutputMicName(movie)))
else:
pwutils.moveFile(self._getMovieOutFn(movie, '.mrc'),
self._getExtraPath(self._getOutputMicName(movie)))
# Keep some local files of this movie in the extra folder
for suffix in ['.star', '.log']:
pwutils.moveFile(self._getMovieOutFn(movie, suffix),
self._getMovieExtraFn(movie, suffix))
suffix = 'corrected_micrographs.star'
fn = os.path.join(self._getOutputMovieFolder(movie), 'output', suffix)
pwutils.moveFile(fn, self._getMovieExtraFn(movie, suffix))
def organizeDataStep(self):
from pyworkflow.utils import moveFile
import pyworkflow.em.metadata as md
from convert import relionToLocation, locationToRelion
# moving shiny.star form project base path to the current protocol extra path.
shinyStar = "shiny.star"
pathFixedShiny = self._getExtraPath(shinyStar)
if exists(shinyStar):
moveFile(shinyStar, pathFixedShiny)
mdShiny = md.MetaData(pathFixedShiny)
oldImgPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = self._getExtraPath(os.path.basename(imgPath))
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldImgPath != imgPath and exists(imgPath):
moveFile(imgPath, newPath)
oldImgPath = imgPath
mdShiny.write(pathFixedShiny)
def writePosFilesStep(self):
""" Write the pos file for each micrograph on metadata format. """
writeSetOfCoordinates(self._getExtraPath(), self.inputCoords,
scale=self.getBoxScale())
# We need to find the mapping (either by micName (without ext) or micId)
# between the micrographs in the SetOfCoordinates and
# the Other micrographs if necessary
if self._micsOther():
micDict = {}
coordMics = self.inputCoords.getMicrographs()
for mic in coordMics:
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
micDict[pwutils.removeBaseExt(mic.getMicName())] = micPos
micDict[mic.getObjId()] = micPos
if any(pwutils.removeBaseExt(mic.getMicName()) in micDict
for mic in self.inputMics):
micKey = lambda mic: pwutils.removeBaseExt(mic.getMicName())
elif any(mic.getObjId() in micDict for mic in self.inputMics):
self.warning('Could not match input micrographs and coordinates'
' micrographs by micName, using micId.')
micKey = lambda mic: mic.getObjId()
else:
raise Exception('Could not match input micrographs and '
'coordinates neither by micName or micId.')
for mic in self.inputMics: # micrograph from input (other)
mk = micKey(mic)
if mk in micDict:
micPosCoord = micDict[mk]
if exists(micPosCoord):
micBase = pwutils.removeBaseExt(mic.getFileName())
micPos = self._getExtraPath(micBase + ".pos")
if micPos != micPosCoord:
self.info('Moving %s -> %s' % (micPosCoord, micPos))
pwutils.moveFile(micPosCoord, micPos)
def organizeDataStep(self):
from pyworkflow.utils import moveFile
import pyworkflow.em.metadata as md
from convert import relionToLocation, locationToRelion
# moving shiny.star form project base path to the current protocol extra path.
shinyStar = "shiny.star"
pathFixedShiny = self._getExtraPath(shinyStar)
if exists(shinyStar):
moveFile(shinyStar, pathFixedShiny)
mdShiny = md.MetaData(pathFixedShiny)
oldImgPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = self._getExtraPath(os.path.basename(imgPath))
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldImgPath != imgPath and exists(imgPath):
moveFile(imgPath, newPath)
oldImgPath = imgPath
mdShiny.write(pathFixedShiny)
def denoisingStep(self):
input_mics = self.inputMicrographs.get()
# Create links to the movies desired to denoise in tmp folder (subset case, janni only accepts directories
# and work with all the files contained in them)
for mic in input_mics:
micName = mic.getFileName()
createLink(mic.getFileName(), self._getTmpPath(basename(micName)))
args = "denoise {}/ {}/ {}".format(self._getTmpPath(),
self._getTmpPath(),
self.getInputModel())
Plugin.runCryolo(self, 'janni_denoise.py', args)
# Move the resulting denoised files to the extra folder
[
moveFile(file, self._getExtraPath(basename(file)))
for file in glob.glob(self._getTmpPath(join('tmp', '*.mrc')))
]
class ProtGctf(em.ProtCTFMicrographs):
"""
Estimates CTF on a set of micrographs
using GPU-accelerated Gctf program.
To find more information about Gctf go to:
http://www.mrc-lmb.cam.ac.uk/kzhang
"""
_label = 'CTF estimation on GPU'
_lastUpdateVersion = VERSION_1_1
def _defineParams(self, form):
form.addSection(label=Message.LABEL_CTF_ESTI)
form.addParam('recalculate', params.BooleanParam, default=False,
condition='recalculate',
label="Do recalculate ctf?")
form.addParam('continueRun', params.PointerParam, allowsNull=True,
condition='recalculate', label="Input previous run",
pointerClass=self.getClassName())
form.addHidden('sqliteFile', params.FileParam, condition='recalculate',
allowsNull=True)
form.addParam('inputMicrographs', params.PointerParam, important=True,
condition='not recalculate', label=Message.LABEL_INPUT_MIC,
pointerClass='SetOfMicrographs')
form.addParam('ctfDownFactor', params.FloatParam, default=1.,
label='CTF Downsampling factor',
condition='not recalculate',
help='Set to 1 for no downsampling. Non-integer '
'downsample factors are possible. This downsampling '
'is only used for estimating the CTF and it does not '
'affect any further calculation. Ideally the estimation '
'of the CTF is optimal when the Thon rings are not too '
'concentrated at the origin (too small to be seen) and '
'not occupying the whole power spectrum (since this '
'downsampling might entail aliasing).')
line = form.addLine('Resolution', condition='not recalculate',
help='Give a value in digital frequency (i.e. between '
'0.0 and 0.5). These cut-offs prevent the typical '
'peak at the center of the PSD and high-resolution '
'terms where only noise exists, to interfere with '
'CTF estimation. The default lowest value is 0.05 '
'but for micrographs with a very fine sampling this '
'may be lowered towards 0. The default highest '
'value is 0.35, but it should be increased for '
'micrographs with signals extending beyond this '
'value. However, if your micrographs extend further '
'than 0.35, you should consider sampling them at a '
'finer rate.')
line.addParam('lowRes', params.FloatParam, default=0.05,
label='Lowest')
line.addParam('highRes', params.FloatParam, default=0.35,
label='Highest')
line = form.addLine('Defocus search range (microns)',
expertLevel=params.LEVEL_ADVANCED,
condition='not recalculate',
help='Select _minimum_ and _maximum_ values for '
'defocus search range (in microns). '
'Underfocus is represented by a positive '
'number.')
line.addParam('minDefocus', params.FloatParam, default=0.25,
label='Min')
line.addParam('maxDefocus', params.FloatParam, default=4.,
label='Max')
form.addParam('astigmatism', params.FloatParam, default=100.0,
label='Expected (tolerated) astigmatism',
help='Estimated astigmatism in Angstroms',
expertLevel=params.LEVEL_ADVANCED)
form.addParam('windowSize', params.IntParam, default=512,
expertLevel=params.LEVEL_ADVANCED,
label='Window size', condition='not recalculate',
help='The PSD is estimated from small patches of this '
'size. Bigger patches allow identifying more '
'details. However, since there are fewer windows, '
'estimations are noisier.')
form.addParam('plotResRing', params.BooleanParam, default=True,
label='Plot a resolution ring on a PSD file',
help='Whether to plot an estimated resolution ring '
'on the power spectrum',
expertLevel=params.LEVEL_ADVANCED)
form.addParam('GPUCore', params.IntParam, default=0,
expertLevel=params.LEVEL_ADVANCED,
label="Choose GPU core",
help='GPU may have several cores. Set it to zero if '
'you do not know what we are talking about. '
'First core index is 0, second 1 and so on.')
form.addSection(label='Advanced')
form.addParam('doEPA', params.BooleanParam, default=False,
label="Do EPA",
help='Do Equiphase average used for output CTF file. '
'Only for nice output, will NOT be used for CTF '
'determination.')
form.addParam('EPAsmp', params.IntParam, default=4,
expertLevel=params.LEVEL_ADVANCED,
condition='not _oldVersion',
label="Over-sampling factor for EPA")
form.addParam('doBasicRotave', params.BooleanParam, default=False,
expertLevel=params.LEVEL_ADVANCED,
condition='_oldVersion',
label="Do rotational average",
help='Do rotational average used for output CTF file. '
'Only for nice output, will NOT be used for CTF '
'determination.')
form.addParam('bfactor', params.IntParam, default=150,
expertLevel=params.LEVEL_ADVANCED,
label="B-factor",
help='B-factors used to decrease high resolution '
'amplitude, A^2; suggested range 50~300 except '
'using REBS method')
form.addParam('overlap', params.FloatParam, default=0.5,
expertLevel=params.LEVEL_ADVANCED,
label="Overlap factor",
help='Overlapping factor for grid boxes sampling, '
'for windowsize=512, 0.5 means 256 pixels overlapping.')
form.addParam('convsize', params.IntParam, default=85,
expertLevel=params.LEVEL_ADVANCED,
label="Boxsize for smoothing",
help='Boxsize to be used for smoothing, '
'suggested 1/5 ~ 1/20 of window size in pixel, '
'e.g. 99 for 512 window')
group = form.addGroup('High-res refinement')
group.addParam('doHighRes', params.BooleanParam, default=False,
label="Do high-resolution refinement",
help='Whether to do High-resolution refinement or not, '
'very useful for selecting high quality micrographs. '
'Especially useful when your data has strong '
'low-resolution bias')
group.addParam('HighResL', params.FloatParam, default=15.0,
condition='doHighRes',
label="Lowest resolution",
help='Lowest resolution to be used for High-resolution '
'refinement, in Angstroms')
group.addParam('HighResH', params.FloatParam, default=4.0,
condition='doHighRes',
label="Highest resolution",
help='Highest resolution to be used for High-resolution '
'refinement, in Angstroms')
group.addParam('HighResBf', params.IntParam, default=50,
condition='doHighRes',
label="B-factor",
help='B-factor to be used for High-resolution '
'refinement, in Angstroms')
form.addParam('doValidate', params.BooleanParam, default=False,
expertLevel=params.LEVEL_ADVANCED,
label="Do validation",
help='Whether to validate the CTF determination.')
form.addSection(label='Phase shift')
form.addParam('doPhShEst', params.BooleanParam, default=False,
label="Estimate phase shift?",
help='For micrographs collected with phase-plate. '
'It is suggested to import such micrographs with '
'amplitude contrast = 0. Also, using smaller '
'_lowest resolution_ (e.g. 15A) and smaller '
'_boxsize for smoothing_ (e.g. 50 for 1024 '
'window size) might be better.')
line = form.addLine('Phase shift range range (deg)',
condition='doPhShEst',
help='Select _lowest_ and _highest_ phase shift '
'(in degrees).')
line.addParam('phaseShiftL', params.FloatParam, default=0.0,
condition='doPhShEst',
label="Min")
line.addParam('phaseShiftH', params.FloatParam, default=180.0,
condition='doPhShEst',
label="Max")
form.addParam('phaseShiftS', params.FloatParam, default=10.0,
condition='doPhShEst',
label="Step",
help='Phase shift search step. Do not worry about '
'the accuracy; this is just the search step, '
'Gctf will refine the phase shift anyway.')
form.addParam('phaseShiftT', params.EnumParam, default=CCC,
condition='doPhShEst',
label='Target',
choices=['CCC', 'Resolution limit'],
display=params.EnumParam.DISPLAY_HLIST,
help='Phase shift target in the search: CCC or '
'resolution limit')
#--------------------------- STEPS functions -------------------------------
def _estimateCTF(self, micFn, micDir, micName):
""" Run Gctf with required parameters """
doneFile = os.path.join(micDir, 'done.txt')
if self.isContinued() and os.path.exists(doneFile):
return
try:
# Create micrograph dir
pwutils.makePath(micDir)
downFactor = self.ctfDownFactor.get()
micFnMrc = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'mrc'))
micFnCtf = self._getTmpPath(pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfFit = self._getTmpPath(pwutils.removeBaseExt(micFn) + '_EPA.log')
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
import pyworkflow.em.packages.xmipp3 as xmipp3
self.runJob("xmipp_transform_downsample",
"-i %s -o %s --step %f --method fourier"
% (micFn, micFnMrc, downFactor),
env=xmipp3.getEnviron())
sps = self.inputMicrographs.get().getScannedPixelSize() * downFactor
self._params['scannedPixelSize'] = sps
else:
ih = em.ImageHandler()
if ih.existsLocation(micFn):
ih.convert(micFn, micFnMrc, em.DT_FLOAT)
else:
print >> sys.stderr, "Missing input micrograph %s" % micFn
# Update _params dictionary
self._params['micFn'] = micFnMrc
self._params['micDir'] = micDir
self._params['gctfOut'] = self._getCtfOutPath(micDir)
except Exception, ex:
print >> sys.stderr, "Some error happened: %s" % ex
import traceback
traceback.print_exc()
try:
self.runJob(self._getProgram(), self._args % self._params, env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % micFnMrc)
psdFile = self._getPsdPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))
# Let's notify that this micrograph has been processed
# just creating an empty file at the end (after success or failure)
open(doneFile, 'w')
# Let's clean the temporary mrc micrographs
pwutils.cleanPath(micFnMrc)
def _processMovie(self, movie):
numberOfFrames = self._getNumberOfFrames(movie)
#FIXME: Figure out how to properly write shifts for unblur
#self._writeMovieAlignment(movie, numberOfFrames)
a0, aN = self._getRange(movie, 'align')
_, lstFrame, _ = movie.getFramesRange()
movieBaseName = pwutils.removeExt(movie.getFileName())
aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
if a0 > 1 or aN < lstFrame:
from pyworkflow.em import ImageHandler
ih = ImageHandler()
movieInputFn = movie.getFileName()
if movieInputFn.endswith("mrc"):
movieInputFn += ":mrcs"
movieConverted = pwutils.removeExt(movieInputFn) + "_tmp.mrcs"
ih.convertStack(movieInputFn, movieConverted, a0, aN)
# Here, only temporal movie file (or link) stored in
# tmp/movie_?????? is removed before move the converted file. It
# is necessary 'cause if it is overwritten you may lost your
# original data.
os.remove(movie.getFileName())
pwutils.moveFile(movieConverted, movie.getFileName())
movieSet = self.inputMovies.get()
self._createLink(movie)
range = aN - a0 + 1
self._argsUnblur(movie, range)
try:
self.runJob(self._program, self._args)
outMicFn = self._getExtraPath(self._getOutputMicName(movie))
if not os.path.exists(outMicFn):
# if only DW mic is saved
outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))
if self.doComputePSD:
# Compute uncorrected avg mic
roi = [0, 0, 0, 0]
fakeShiftsFn = self.writeZeroShifts(movie)
self.averageMovie(movie, fakeShiftsFn, aveMicFn,
binFactor=1,
roi=roi, dark=None,
gain=movieSet.getGain())
self.computePSDs(movie, aveMicFn, outMicFn,
outputFnCorrected=self._getPsdJpeg(movie))
self._saveAlignmentPlots(movie)
if self._doComputeMicThumbnail():
self.computeThumbnail(outMicFn,
outputFn=self._getOutputMicThumbnail(
movie))
except:
print("ERROR: Movie %s failed\n" % movie.getFileName())
import traceback
traceback.print_exc()
def refineCtfStep(self):
self._defineValues()
self._prepareCommand()
for mic in self.matchingMics:
micName = mic.getFileName()
micBase = pwutils.removeBaseExt(micName)
micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
outMic = pwutils.join(micDirTmp,
pwutils.replaceBaseExt(micName, 'mrc'))
micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
micFnOut = self._getCtfOutPath(micDirTmp)
micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')
# Update _params dictionary
self._params['micFn'] = outMic
self._params['gctfOut'] = micFnOut
if self.useInputCtf and self.ctfRelations.get():
# get input CTFs from a mic
ctfs = self.ctfRelations.get()
micKey = mic.getMicName() if self.hasMicName else mic.getObjId(
)
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({
'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params[
'refine_input_ctf']
self._args += "--defU_init %f " % self._params[
'defU_init']
self._args += "--defV_init %f " % self._params[
'defV_init']
self._args += "--defA_init %f " % self._params[
'defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# final args
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
try:
self.runJob(self._getProgram(),
self._args % self._params,
env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % outMic)
# move results from tmp to extra folder
micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
psdFile = self._getPsdPath(micDir)
ctfOutFile = self._getCtfOutPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
ctflocalFile = self._getCtfLocalPath(micDir, micBase)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnOut, ctfOutFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.moveFile(micFnLocalCtf, ctflocalFile)
# Let's clean the temporary micrographs
pwutils.cleanPath(outMic)
pwutils.cleanPath(micDirTmp)
pwutils.cleanPath(self.matchingMics.getFileName())
pwutils.cleanPath(
self.getProject().getPath('micrographs_all_gctf.star'))
def cryoloModelingStep(self, extraArgs=''):
self.runCryoloTrain(5, extraArgs=extraArgs)
pwutils.moveFile(self._getWorkDir(self.MODEL),
self.getOutputModelPath())
def refineCtfStep(self, micFn, micKey):
micPath = self._getTmpPath(pwutils.removeBaseExt(micFn))
# We convert the input micrograph on demand if not in .mrc
downFactor = self.ctfDownFactor.get()
ih = emlib.image.ImageHandler()
micFnMrc = os.path.join(micPath, pwutils.replaceBaseExt(micFn, 'mrc'))
if downFactor != 1:
# Replace extension by 'mrc' cause there are some formats
# that cannot be written (such as dm3)
ih.scaleFourier(micFn, micFnMrc, downFactor)
sps = self.inputMicrographs.get().getScannedPixelSize(
) * downFactor
self._params['scannedPixelSize'] = sps
else:
ih.convert(micFn, micFnMrc, emlib.DT_FLOAT)
# Refine input CTFs, match ctf by micName
if self.useInputCtf and self.ctfRelations.hasValue():
ctfs = self._getCtfs()
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({
'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params[
'refine_input_ctf']
self._args += "--defU_init %f " % self._params['defU_init']
self._args += "--defV_init %f " % self._params['defV_init']
self._args += "--defA_init %f " % self._params['defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# Run Gctf refine
try:
args = self._args % self._params
args += ' %s' % micFnMrc
self.runJob(Plugin.getProgram(), args, env=Plugin.getEnviron())
# Let's clean the temporary mrc micrograph
pwutils.cleanPath(micFnMrc)
# move output from tmp to extra
micFnCtf = os.path.join(micPath,
pwutils.replaceBaseExt(micFn, 'ctf'))
micFnCtfLog = os.path.join(
micPath,
pwutils.removeBaseExt(micFn) + '_gctf.log')
micFnCtfFit = os.path.join(
micPath,
pwutils.removeBaseExt(micFn) + '_EPA.log')
micFnCtfLocal = os.path.join(
micPath,
pwutils.removeBaseExt(micFn) + '_local.star')
micFnCtfOut = self._getPsdPath(micFn)
micFnCtfLogOut = self._getCtfOutPath(micFn)
micFnCtfFitOut = self._getCtfFitOutPath(micFn)
micFnCtfLocalOut = self._getCtfLocalOutPath(micFn)
pwutils.moveFile(micFnCtf, micFnCtfOut)
pwutils.moveFile(micFnCtfLog, micFnCtfLogOut)
pwutils.moveFile(micFnCtfFit, micFnCtfFitOut)
pwutils.moveFile(micFnCtfLocal, micFnCtfLocalOut)
except:
print("ERROR: Gctf has failed on %s" % micFnMrc)
import traceback
traceback.print_exc()
def _processMovie(self, movie):
numberOfFrames = self._getNumberOfFrames(movie)
#FIXME: Figure out how to properly write shifts for unblur
#self._writeMovieAlignment(movie, numberOfFrames)
a0, aN = self._getRange(movie, 'align')
_, lstFrame, _ = movie.getFramesRange()
movieBaseName = pwutils.removeExt(movie.getFileName())
aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
if a0 > 1 or aN < lstFrame:
from pyworkflow.em import ImageHandler
ih = ImageHandler()
movieInputFn = movie.getFileName()
if movieInputFn.endswith("mrc"):
movieInputFn += ":mrcs"
movieConverted = pwutils.removeExt(movieInputFn) + "_tmp.mrcs"
ih.convertStack(movieInputFn, movieConverted, a0, aN)
# Here, only temporal movie file (or link) stored in
# tmp/movie_?????? is removed before move the converted file. It
# is necessary 'cause if it is overwritten you may lost your
# original data.
os.remove(movie.getFileName())
pwutils.moveFile(movieConverted, movie.getFileName())
movieSet = self.inputMovies.get()
self._createLink(movie)
range = aN - a0 + 1
self._argsUnblur(movie, range)
try:
self.runJob(self._program, self._args)
outMicFn = self._getExtraPath(self._getOutputMicName(movie))
if not os.path.exists(outMicFn):
# if only DW mic is saved
outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))
if self.doComputePSD:
# Compute uncorrected avg mic
roi = [0, 0, 0, 0]
fakeShiftsFn = self.writeZeroShifts(movie)
self.averageMovie(movie, fakeShiftsFn, aveMicFn,
binFactor=1,
roi=roi, dark=None,
gain=movieSet.getGain())
self.computePSDs(movie, aveMicFn, outMicFn,
outputFnCorrected=self._getPsdJpeg(movie))
self._saveAlignmentPlots(movie)
if self._doComputeMicThumbnail():
self.computeThumbnail(outMicFn,
outputFn=self._getOutputMicThumbnail(
movie))
except:
print("ERROR: Movie %s failed\n" % movie.getFileName())
def createOutputStep(self):
inputMovies = self.getInputMovies()
nFrames = inputMovies.getFirstItem().getNumberOfFrames()
inputParts = self.getParticles()
movieParticles = self._createSetOfMovieParticles()
movieParticles.copyInfo(inputParts)
movieParticles.setSamplingRate(self._getNewSampling())
self.lastMicName = None
self.partList = []
def _addPartsFromMic():
# To avoid parsing the Relion star files...we are assuming here
# the order in which Relion is generating the movie-particles per stack
# it start part 1, 2, N of frame 1, then 1, 2..N of frame 2 and so on.
# If this way changes in the future, the following code could break.
# For the sake of performance, I will take the risk now.
count = 0
avgFrames = self.avgFrames.get()
for frame in range(0, nFrames, avgFrames):
frameId = min(frame + avgFrames, nFrames)
for mPart in self.partList:
mPart.setObjId(None) # clear objId to insert a new one
mPart.setFrameId(frameId)
count += 1
mPart.setIndex(count)
mPart._rlnAverageNrOfFrames = em.Integer(avgFrames)
movieParticles.append(mPart)
del self.partList # free unnecessary particle list memory
self.partList = []
for part in inputParts.iterItems(orderBy='_micId'):
micName = part.getCoordinate().getMicName()
if micName != self.lastMicName:
if self.lastMicName is not None:
_addPartsFromMic()
self.lastMicName = micName
movieBase = '%s_movie.mrcs' % pwutils.removeBaseExt(micName)
def _replaceSuffix(suffix):
return movieBase.replace('_movie.mrcs', suffix)
# Move the resulting stack of movie-particles to extra directly
movieStack = self._getExtraPath('output', 'extra', movieBase)
self.newMovieStack = self._getExtraPath(
_replaceSuffix('_ptcls.mrcs'))
pwutils.moveFile(movieStack, self.newMovieStack)
# Clean up intermediate files (either links or converted)
# plus generated files not needed anymore
if not pwutils.envVarOn("SCIPION_DEBUG_NOCLEAN"):
pwutils.cleanPath(
self._getExtraPath(movieBase),
self._getExtraPath(_replaceSuffix('.mrcs')))
# Create a movie particles based on that one and
# store in the list of this movie
mPart = em.MovieParticle()
mPart.copy(part) # copy all information from part
mPart.setParticleId(part.getObjId())
mPart.setFileName(self.newMovieStack)
self.partList.append(mPart)
pwutils.cleanPath(self._getExtraPath('output'))
_addPartsFromMic()
self._defineOutputs(outputParticles=movieParticles)
self._defineSourceRelation(self.inputMovies, movieParticles)
self._defineSourceRelation(self.inputParticles, movieParticles)
def refineCtfStep(self):
self._defineValues()
self._prepareCommand()
for mic in self.matchingMics:
micName = mic.getFileName()
micBase = pwutils.removeBaseExt(micName)
micDirTmp = self._getTmpPath(pwutils.removeBaseExt(micName))
outMic = pwutils.join(micDirTmp, pwutils.replaceBaseExt(micName, 'mrc'))
micFnCtf = pwutils.join(micDirTmp, micBase + '.ctf')
micFnOut = self._getCtfOutPath(micDirTmp)
micFnCtfFit = pwutils.join(micDirTmp, micBase + '_EPA.log')
micFnLocalCtf = pwutils.join(micDirTmp, micBase + '_local.star')
# Update _params dictionary
self._params['micFn'] = outMic
self._params['gctfOut'] = micFnOut
if self.useInputCtf and self.ctfRelations.get():
# get input CTFs from a mic
ctfs = self.ctfRelations.get()
micKey = mic.getMicName() if self.hasMicName else mic.getObjId()
for ctf in ctfs:
ctfMicName = ctf.getMicrograph().getMicName()
ctfMicId = ctf.getMicrograph().getObjId()
if micKey == ctfMicName or micKey == ctfMicId:
# add CTF refine options
self._params.update({'refine_input_ctf': 1,
'defU_init': ctf.getDefocusU(),
'defV_init': ctf.getDefocusV(),
'defA_init': ctf.getDefocusAngle(),
'B_init': self.bfactor.get()
})
self._args += "--refine_input_ctf %d " % self._params['refine_input_ctf']
self._args += "--defU_init %f " % self._params['defU_init']
self._args += "--defV_init %f " % self._params['defV_init']
self._args += "--defA_init %f " % self._params['defA_init']
self._args += "--B_init %f " % self._params['B_init']
self._args += "--defU_err %f " % self.defUerr.get()
self._args += "--defV_err %f " % self.defVerr.get()
self._args += "--defA_err %f " % self.defAerr.get()
self._args += "--B_err %f " % self.Berr.get()
break
# final args
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
try:
self.runJob(self._getProgram(), self._args % self._params,
env=self._getEnviron())
except:
print("ERROR: Gctf has failed for micrograph %s" % outMic)
# move results from tmp to extra folder
micDir = self._getExtraPath(pwutils.removeBaseExt(micName))
pwutils.makePath(micDir)
psdFile = self._getPsdPath(micDir)
ctfOutFile = self._getCtfOutPath(micDir)
ctffitFile = self._getCtfFitOutPath(micDir)
ctflocalFile = self._getCtfLocalPath(micDir, micBase)
pwutils.moveFile(micFnCtf, psdFile)
pwutils.moveFile(micFnOut, ctfOutFile)
pwutils.moveFile(micFnCtfFit, ctffitFile)
pwutils.moveFile(micFnLocalCtf, ctflocalFile)
# Let's clean the temporary micrographs
pwutils.cleanPath(outMic)
pwutils.cleanPath(micDirTmp)
pwutils.cleanPath(self.matchingMics.getFileName())
pwutils.cleanPath(self.getProject().getPath('micrographs_all_gctf.star'))