def getSubparticles(self, localrec, partItem, symMatrices,
params, subpartVectorList, filters):
if getVersion() == '1.1.0':
subparticles, _ = localrec.create_subparticles(partItem,
symMatrices,
subpartVectorList,
params["dim"],
self.relaxSym,
self.randomize,
"subparticles",
params["unique"],
0,
self.alignSubparticles,
"",
True,
filters)
else:
subparticles, _ = localrec.create_subparticles(partItem,
symMatrices,
subpartVectorList,
params["dim"],
self.randomize,
"subparticles",
params["unique"],
0,
self.alignSubparticles,
"",
False,
filters)
return subparticles
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet, imgStarTmp, self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath), 'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def getSubparticles(self, localrec, partItem, symMatrices, params,
subpartVectorList, filters, pxSize):
if getVersion() == '1.2.1':
subparticles, _ = localrec.create_subparticles(
partItem, symMatrices, subpartVectorList, params["dim"],
self.randomize, "subparticles", params["unique"], 0,
self.alignSubparticles, "", False, filters, pxSize)
elif getVersion() == '1.1.0':
subparticles, _ = localrec.create_subparticles(
partItem, symMatrices, subpartVectorList, params["dim"],
self.relaxSym, self.randomize, "subparticles",
params["unique"], 0, self.alignSubparticles, "", True, filters)
else:
subparticles, _ = localrec.create_subparticles(
partItem, symMatrices, subpartVectorList, params["dim"],
self.randomize, "subparticles", params["unique"], 0,
self.alignSubparticles, "", False, filters)
return subparticles
def _validate(self):
errors = []
# Check that the program exists
if not exists(self._getProgram()):
errors.append("Binary '%s' does not exits.\n"
"Check configuration file: \n"
"~/.config/scipion/scipion.conf\n"
"and set GCTF variables properly."
% self._getProgram())
if self.doPhShEst and getVersion() == '0.50':
errors.append('This version of Gctf (0.50) does not support phase shift estimation!'
' Please update to a newer version.')
return errors
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameters just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('input_star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStar))
# Pass stack file as None to avoid write the images files
writeSetOfParticles(imgSet, imgStar, self._getExtraPath())
if self.doCtfManualGroups:
self._splitInCTFGroups(imgStar)
if not self.IS_CLASSIFY:
if self.realignMovieFrames:
movieParticleSet = self.inputMovieParticles.get()
auxMovieParticles = self._createSetOfMovieParticles(suffix='tmp')
auxMovieParticles.copyInfo(movieParticleSet)
# Discard the movie particles that are not present in the refinement set
for movieParticle in movieParticleSet:
particle = imgSet[movieParticle.getParticleId()]
if particle is not None:
auxMovieParticles.append(movieParticle)
writeSetOfParticles(auxMovieParticles,
self._getFileName('movie_particles'), None, originalSet=imgSet,
postprocessImageRow=self._postprocessImageRow)
mdMovies = md.MetaData(self._getFileName('movie_particles'))
mdParts = md.MetaData(self._getFileName('input_star'))
if getVersion() == "1.4":
mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_ORI_NAME)
else:
mdParts.renameColumn(md.RLN_IMAGE_NAME, md.RLN_PARTICLE_NAME)
mdParts.removeLabel(md.RLN_MICROGRAPH_NAME)
detectorPxSize = movieParticleSet.getAcquisition().getMagnification() * movieParticleSet.getSamplingRate() / 10000
mdAux = md.MetaData()
mdMovies.fillConstant(md.RLN_CTF_DETECTOR_PIXEL_SIZE, detectorPxSize)
mdAux.join2(mdMovies, mdParts, md.RLN_PARTICLE_ID, md.RLN_IMAGE_ID, md.INNER_JOIN)
mdAux.write(self._getFileName('movie_particles'), md.MD_OVERWRITE)
cleanPath(auxMovieParticles.getFileName())
def _argsGctf(self):
self._args = " --apix %f " % self._params['sampling']
self._args += "--kV %f " % self._params['voltage']
self._args += "--cs %f " % self._params['sphericalAberration']
self._args += "--ac %f " % self._params['ampContrast']
self._args += "--dstep %f " % self._params['scannedPixelSize']
self._args += "--defL %f " % self._params['minDefocus']
self._args += "--defH %f " % self._params['maxDefocus']
self._args += "--defS %f " % self._params['step_focus']
self._args += "--astm %f " % self.astigmatism.get()
self._args += "--resL %f " % self._params['lowRes']
self._args += "--resH %f " % self._params['highRes']
self._args += "--do_EPA %d " % (1 if self.doEPA else 0)
self._args += "--boxsize %d " % self._params['windowSize']
self._args += "--plot_res_ring %d " % (1 if self.plotResRing else 0)
self._args += "--gid %d " % self.GPUCore.get()
self._args += "--bfac %d " % self.bfactor.get()
self._args += "--B_resH %f " % (2 * self._params['sampling'])
self._args += "--overlap %f " % self.overlap.get()
self._args += "--convsize %d " % self.convsize.get()
self._args += "--do_Hres_ref %d " % (1 if self.doHighRes else 0)
# local refine options
self._args += "--do_local_refine 1 --boxsuffix _coords.star "
self._args += "--local_radius %d " % self.locRad.get()
self._args += "--local_avetype %d " % self.locAveType.get()
self._args += "--local_boxsize %d " % self.locBoxSize.get()
self._args += "--local_overlap % 0.2f " % self.locOverlap.get()
self._args += "--local_resL %d " % self.locResL.get()
self._args += "--local_resH %d " % self.locResH.get()
self._args += "--refine_local_astm %d " % (1 if self.locAstm else 0)
if getVersion() == '0.50':
self._args += "--do_basic_rotave %d " % (1 if self.doBasicRotave
else 0)
else:
self._args += "--EPA_oversmp %d " % self.EPAsmp.get()
if self.doPhShEst:
self._args += "--phase_shift_L %f " % self.phaseShiftL.get()
self._args += "--phase_shift_H %f " % self.phaseShiftH.get()
self._args += "--phase_shift_S %f " % self.phaseShiftS.get()
self._args += "--phase_shift_T %d " % (1 +
self.phaseShiftT.get())
if self.doHighRes:
self._args += "--Href_resL %d " % self.HighResL.get()
self._args += "--Href_resH %d " % self.HighResH.get()
self._args += "--Href_bfac %d " % self.HighResBf.get()
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _argsGctf(self):
self._args = " --apix %f " % self._params['sampling']
self._args += "--kV %f " % self._params['voltage']
self._args += "--cs %f " % self._params['sphericalAberration']
self._args += "--ac %f " % self._params['ampContrast']
self._args += "--dstep %f " % self._params['scannedPixelSize']
self._args += "--defL %f " % self._params['minDefocus']
self._args += "--defH %f " % self._params['maxDefocus']
self._args += "--defS %f " % self._params['step_focus']
self._args += "--astm %f " % self.astigmatism.get()
self._args += "--resL %f " % self._params['lowRes']
self._args += "--resH %f " % self._params['highRes']
self._args += "--do_EPA %d " % (1 if self.doEPA else 0)
self._args += "--boxsize %d " % self._params['windowSize']
self._args += "--plot_res_ring %d " % (1 if self.plotResRing else 0)
self._args += "--gid %d " % self.GPUCore.get()
self._args += "--bfac %d " % self.bfactor.get()
self._args += "--B_resH %f " % (2 * self._params['sampling'])
self._args += "--overlap %f " % self.overlap.get()
self._args += "--convsize %d " % self.convsize.get()
self._args += "--do_Hres_ref %d " % (1 if self.doHighRes else 0)
# local refine options
self._args += "--do_local_refine 1 --boxsuffix _coords.star "
self._args += "--local_radius %d " % self.locRad.get()
self._args += "--local_avetype %d " % self.locAveType.get()
self._args += "--local_boxsize %d " % self.locBoxSize.get()
self._args += "--local_overlap % 0.2f " % self.locOverlap.get()
self._args += "--local_resL %d " % self.locResL.get()
self._args += "--local_resH %d " % self.locResH.get()
self._args += "--refine_local_astm %d " % (1 if self.locAstm else 0)
if getVersion() == '0.50':
self._args += "--do_basic_rotave %d " % (1 if self.doBasicRotave else 0)
else:
self._args += "--EPA_oversmp %d " % self.EPAsmp.get()
if self.doPhShEst:
self._args += "--phase_shift_L %f " % self.phaseShiftL.get()
self._args += "--phase_shift_H %f " % self.phaseShiftH.get()
self._args += "--phase_shift_S %f " % self.phaseShiftS.get()
self._args += "--phase_shift_T %d " % (1 + self.phaseShiftT.get())
if self.doHighRes:
self._args += "--Href_resL %d " % self.HighResL.get()
self._args += "--Href_resH %d " % self.HighResH.get()
self._args += "--Href_bfac %d " % self.HighResBf.get()
def versionGE(self, version):
""" Return True if current version of motioncor2 is greater
or equal than the input argument.
Params:
version: string version (semantic version, e.g 1.0.1)
"""
if not self.isSemVersion():
return False
v1 = map(int, getVersion('MOTIONCOR2').split('.'))
v2 = map(int, version.split('.'))
for x1, x2 in zip(v1, v2):
if x1 < x2:
return False
return True
def versionGE(self, version):
""" Return True if current version of motioncor2 is greater
or equal than the input argument.
Params:
version: string version (semantic version, e.g 1.0.1)
"""
if not self.isSemVersion():
return False
v1 = map(int, getVersion('MOTIONCOR2').split('.'))
v2 = map(int, version.split('.'))
for x1, x2 in zip(v1, v2):
if x1 < x2:
return False
return True
def _argsGctf(self):
self._args = " --apix %f " % self._params['sampling']
self._args += "--kV %f " % self._params['voltage']
self._args += "--cs %f " % self._params['sphericalAberration']
self._args += "--ac %f " % self._params['ampContrast']
self._args += "--dstep %f " % self._params['scannedPixelSize']
self._args += "--defL %f " % self._params['minDefocus']
self._args += "--defH %f " % self._params['maxDefocus']
self._args += "--defS %f " % self._params['step_focus']
self._args += "--astm %f " % self.astigmatism.get()
self._args += "--resL %f " % self._params['lowRes']
self._args += "--resH %f " % self._params['highRes']
self._args += "--do_EPA %d " % (1 if self.doEPA else 0)
self._args += "--boxsize %d " % self._params['windowSize']
self._args += "--plot_res_ring %d " % (1 if self.plotResRing else 0)
self._args += "--gid %%(GPU)s " # Use %% to escape when formatting
self._args += "--bfac %d " % self.bfactor.get()
self._args += "--B_resH %f " % (2 * self._params['sampling'])
self._args += "--overlap %f " % self.overlap.get()
self._args += "--convsize %d " % self.convsize.get()
self._args += "--do_Hres_ref %d " % (1 if self.doHighRes else 0)
if getVersion() == '0.50':
self._args += "--do_basic_rotave %d " % (1 if self.doBasicRotave
else 0)
else:
self._args += "--EPA_oversmp %d " % self.EPAsmp.get()
if self.doPhShEst:
self._args += "--phase_shift_L %f " % self.phaseShiftL.get()
self._args += "--phase_shift_H %f " % self.phaseShiftH.get()
self._args += "--phase_shift_S %f " % self.phaseShiftS.get()
self._args += "--phase_shift_T %d " % (1 +
self.phaseShiftT.get())
if self.doHighRes:
self._args += "--Href_resL %d " % self.HighResL.get()
self._args += "--Href_resH %d " % self.HighResH.get()
self._args += "--Href_bfac %d " % self.HighResBf.get()
self._args += "--do_validation %d " % (1 if self.doValidate else 0)
self._args += "%(micFn)s "
self._args += "> %(gctfOut)s"
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameter just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
imgStarTmp = self._getTmpPath('movie_particles.star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStarTmp))
writeSetOfParticles(imgSet,
imgStarTmp,
self._getExtraPath(),
alignType=imgSet.getAlignment(),
extraLabels=MOVIE_EXTRA_LABELS)
mdImg = md.MetaData(imgStarTmp)
# replace mdColumn from *.stk to *.mrcs as Relion2 requires
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
from convert import relionToLocation, locationToRelion
for objId in mdImg:
index, imgPath = relionToLocation(mdImg.getValue(mdColumn, objId))
if not imgPath.endswith('mrcs'):
newName = pwutils.replaceBaseExt(os.path.basename(imgPath),
'mrcs')
newPath = self._getTmpPath(newName)
newLoc = locationToRelion(index, newPath)
if not exists(newPath):
pwutils.createLink(imgPath, newPath)
mdImg.setValue(mdColumn, newLoc, objId)
mdImg.write(imgStar)
def _validate(self):
errors = []
# Check that the program exists
if not exists(self._getProgram()):
errors.append("Binary '%s' does not exits.\n"
"Check configuration file: \n"
"~/.config/scipion/scipion.conf\n"
"and set GCTF variables properly."
% self._getProgram())
if self.doPhShEst and getVersion() == '0.50':
errors.append('This version of Gctf (0.50) does not support phase '
'shift estimation! Please update to a newer version.')
nprocs = max(self.numberOfMpi.get(), self.numberOfThreads.get())
if nprocs < len(self.getGpuList()):
errors.append("Multiple GPUs can not be used by a single process. "
"Make sure you specify more processors than GPUs. ")
if self._getStreamingBatchSize() > 1:
errors.append("Batch steps are not implemented yet for Gctf. ")
return errors
def isDisabled(cls):
return getVersion() in [V1_3, V1_4, V2_0]
def _supportsMagCorrection(self):
return getVersion('MOTIONCOR2') not in ['03162016', '10192016']
def isSemVersion(self):
""" Return True if it is a semantic version of motioncor2.
It started with release 1.0.0.
"""
return getVersion('MOTIONCOR2').startswith('1.0.')
def isSemVersion(self):
""" Return True if it is a semantic version of motioncor2.
It started with release 1.0.0.
"""
return getVersion('MOTIONCOR2').startswith('1.0.')
def _isOutStackSupport(self):
# checks if output aligned movies can be saved by motioncor2
return True if getVersion('MOTIONCOR2') != '03162016' else False
def _processMovie(self, movie):
inputMovies = self.inputMovies.get()
movieFolder = self._getOutputMovieFolder(movie)
outputMicFn = self._getRelPath(self._getOutputMicName(movie),
movieFolder)
outputMovieFn = self._getRelPath(self._getOutputMovieName(movie),
movieFolder)
movieBaseName = pwutils.removeExt(movie.getFileName())
aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
logFile = self._getRelPath(self._getMovieLogFile(movie), movieFolder)
a0, aN = self._getRange(movie, 'align')
if not self.useMotioncor2:
# Get the number of frames and the range to be used
# for alignment and sum
s0, sN = self._getRange(movie, 'sum')
argsDict = {
'-crx': self.cropOffsetX.get(),
'-cry': self.cropOffsetY.get(),
'-cdx': self.cropDimX.get(),
'-cdy': self.cropDimY.get(),
'-bin': self.binFactor.get(),
'-nst': '%d' % a0,
'-ned': '%d' % aN,
'-nss': '%d' % s0,
'-nes': '%d' % sN,
'-gpu': self.GPUIDs.get(),
'-flg': logFile,
}
args = '"%s" ' % movie.getBaseName()
args += ' '.join(
['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -fgr "%s"' % inputMovies.getGain()
if inputMovies.getDark():
args += ' -fdr "%s"' % inputMovies.getDark()
if self.doSaveAveMic:
args += ' -fcs "%s" ' % outputMicFn
if self.doSaveMovie:
args += ' -fct "%s" -ssc 1' % outputMovieFn
args += ' ' + self.extraParams.get()
program = MOTIONCORR_PATH
else:
logFileBase = (logFile.replace('0-Full.log',
'').replace('0-Patch-Full.log', ''))
# default values for motioncor2 are (1, 1)
cropDimX = self.cropDimX.get() or 1
cropDimY = self.cropDimY.get() or 1
numbOfFrames = self._getNumberOfFrames(movie)
if self.doApplyDoseFilter:
preExp, dose = self._getCorrectedDose(inputMovies)
else:
preExp, dose = 0.0, 0.0
# reset values = 1 to 0 (motioncor2 does it automatically,
# but we need to keep this for consistency)
if self.patchX.get() == 1:
self.patchX.set(0)
if self.patchY.get() == 1:
self.patchY.set(0)
argsDict = {
'-OutMrc': '"%s"' % outputMicFn,
'-Patch': '%d %d' % (self.patchX, self.patchY),
'-MaskCent': '%d %d' % (self.cropOffsetX, self.cropOffsetY),
'-MaskSize': '%d %d' % (cropDimX, cropDimY),
'-FtBin': self.binFactor.get(),
'-Tol': self.tol.get(),
'-Group': self.group.get(),
'-FmDose': dose,
'-Throw': '%d' % a0,
'-Trunc': '%d' % (abs(aN - numbOfFrames + 1)),
'-PixSize': inputMovies.getSamplingRate(),
'-kV': inputMovies.getAcquisition().getVoltage(),
'-Gpu': self.GPUIDs.get(),
'-LogFile': logFileBase,
}
if getVersion('MOTIONCOR2') != '03162016':
argsDict['-InitDose'] = preExp
argsDict['-OutStack'] = 1 if self.doSaveMovie else 0
if self.isSemVersion():
if self.defectFile.get():
argsDict['-DefectFile'] = self.defectFile.get()
if self.versionGE(
'1.0.1'): # Patch overlap was introduced in 1.0.1
patchOverlap = self.getAttributeValue('patchOverlap', None)
if patchOverlap: # 0 or None is False
argsDict['-Patch'] += " %d" % patchOverlap
if self._supportsMagCorrection() and self.doMagCor:
if self.useEst:
inputEst = self.inputEst.get().getOutputLog()
if getVersion('MOTIONCOR2') == '01302017':
input_params = parseMagCorrInput(inputEst)
# this version uses stretch parameters as following:
# 1/maj, 1/min, -angle
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (
1.0 / input_params[1], 1.0 / input_params[2],
-1 * input_params[0])
else:
# While motioncor2 >=1.0.0 uses estimation params AS IS
input_params = parseMagEstOutput(inputEst)
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (
input_params[1], input_params[2], input_params[0])
else:
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (
self.scaleMaj, self.scaleMin, self.angDist)
ext = pwutils.getExt(movie.getFileName()).lower()
if ext in ['.mrc', '.mrcs']:
args = ' -InMrc "%s" ' % movie.getBaseName()
elif ext in ['.tif', '.tiff']:
args = ' -InTiff "%s" ' % movie.getBaseName()
else:
raise Exception("Unsupported format: %s" % ext)
args += ' '.join(
['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -Gain "%s" ' % inputMovies.getGain()
if inputMovies.getDark():
args += ' -Dark "%s"' % inputMovies.getDark()
args += ' ' + self.extraParams2.get()
program = MOTIONCOR2_PATH
try:
self.runJob(program,
args,
cwd=movieFolder,
env=getEnviron(self.useMotioncor2))
self._fixMovie(movie)
# Compute PSDs
outMicFn = self._getExtraPath(self._getOutputMicName(movie))
if not os.path.exists(outMicFn):
# if only DW mic is saved
outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))
if self.doComputePSD:
# Compute uncorrected avg mic
roi = [
self.cropOffsetX.get(),
self.cropOffsetY.get(),
self.cropDimX.get(),
self.cropDimY.get()
]
fakeShiftsFn = self.writeZeroShifts(movie)
self.averageMovie(movie,
fakeShiftsFn,
aveMicFn,
binFactor=self.binFactor.get(),
roi=roi,
dark=inputMovies.getDark(),
gain=inputMovies.getGain())
self.computePSDs(movie,
aveMicFn,
outMicFn,
outputFnCorrected=self._getPsdJpeg(movie))
self._saveAlignmentPlots(movie)
if self._doComputeMicThumbnail():
self.computeThumbnail(
outMicFn, outputFn=self._getOutputMicThumbnail(movie))
except:
print("ERROR: Movie %s failed\n" % movie.getName())
def isDisabled(cls):
return getVersion() in [V1_3, V1_4]
def _isOutStackSupport(self):
# checks if output aligned movies can be saved by motioncor2
return True if getVersion('MOTIONCOR2') != '03162016' else False
def createOutputStep(self):
setEnviron() # Set the environment to access localrec modules
import localrec
import pyrelion
# return
inputSet = self._getInputParticles()
outputSet = self._createSetOfCoordinates(inputSet)
params = {
"symmetryGroup": self.symmetryGroup.get(),
"vector": self.vector.get(),
"vectorFile": self.vectorFile.get(),
"length": self.length.get(),
"unique": self.unique.get(),
"mindist": self.mindist.get(),
"side": self.side.get(),
"top": self.top.get(),
"pxSize": self.inputParticles.get().getSamplingRate(),
"dim": self.inputParticles.get().getXDim()
}
symMatrices = localrec.matrix_from_symmetry(self.symmetryGroup.get())
if self.defineVector == CMM:
cmmFn = params["vectorFile"]
vector = " "
else:
cmmFn = ""
vector = params["vector"]
subpartVectorList = localrec.load_vectors(cmmFn, vector,
params["length"],
params["pxSize"])
# Define some conditions to filter subparticles
filters = localrec.load_filters(math.radians(params["side"]),
math.radians(params["top"]),
params["mindist"])
coord = Coordinate()
for part in inputSet:
partItem = pyrelion.Item()
particleToRow(part, partItem)
if getVersion() == '1.2.1':
subparticles = self.getSubparticles(localrec, partItem,
symMatrices, params,
subpartVectorList, filters,
params["pxSize"])
else:
subparticles = self.getSubparticles(localrec, partItem,
symMatrices, params,
subpartVectorList, filters)
for subpart in subparticles:
rowToSubcoordinate(subpart, coord, part)
coord.setObjId(None) # Force to insert as a new item
outputSet.append(coord)
if part.hasAttribute('_rlnRandomSubset'):
coord._subparticle.copyAttributes(part, '_rlnRandomSubset')
restituteRelionHome()
self._defineOutputs(outputCoordinates=outputSet)
self._defineSourceRelation(self.inputParticles, outputSet)
def _processMovie(self, movie):
inputMovies = self.inputMovies.get()
movieFolder = self._getOutputMovieFolder(movie)
outputMicFn = self._getRelPath(self._getOutputMicName(movie),
movieFolder)
outputMovieFn = self._getRelPath(self._getOutputMovieName(movie),
movieFolder)
movieBaseName = pwutils.removeExt(movie.getFileName())
aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
logFile = self._getRelPath(self._getMovieLogFile(movie), movieFolder)
a0, aN = self._getRange(movie, 'align')
if not self.useMotioncor2:
# Get the number of frames and the range to be used
# for alignment and sum
s0, sN = self._getRange(movie, 'sum')
argsDict = {
'-crx': self.cropOffsetX.get(),
'-cry': self.cropOffsetY.get(),
'-cdx': self.cropDimX.get(),
'-cdy': self.cropDimY.get(),
'-bin': self.binFactor.get(),
'-nst': '%d' % a0,
'-ned': '%d' % aN,
'-nss': '%d' % s0,
'-nes': '%d' % sN,
'-gpu': self.GPUIDs.get(),
'-flg': logFile,
}
args = '"%s" ' % movie.getBaseName()
args += ' '.join(
['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -fgr "%s"' % inputMovies.getGain()
if inputMovies.getDark():
args += ' -fdr "%s"' % inputMovies.getDark()
if self.doSaveAveMic:
args += ' -fcs "%s" ' % outputMicFn
if self.doSaveMovie:
args += ' -fct "%s" -ssc 1' % outputMovieFn
args += ' ' + self.extraParams.get()
program = MOTIONCORR_PATH
else:
logFileBase = (logFile.replace('0-Full.log',
'').replace('0-Patch-Full.log', ''))
# default values for motioncor2 are (1, 1)
cropDimX = self.cropDimX.get() or 1
cropDimY = self.cropDimY.get() or 1
numbOfFrames = self._getNumberOfFrames(movie)
if self.doApplyDoseFilter:
preExp, dose = self._getCorrectedDose(inputMovies)
else:
preExp, dose = 0.0, 0.0
argsDict = {
'-OutMrc': '"%s"' % outputMicFn,
'-Patch': '%d %d' % (self.patchX, self.patchY),
'-MaskCent': '%d %d' % (self.cropOffsetX, self.cropOffsetY),
'-MaskSize': '%d %d' % (cropDimX, cropDimY),
'-FtBin': self.binFactor.get(),
'-Tol': self.tol.get(),
'-Group': self.group.get(),
'-FmDose': dose,
'-Throw': '%d' % a0,
'-Trunc': '%d' % (abs(aN - numbOfFrames + 1)),
'-PixSize': inputMovies.getSamplingRate(),
'-kV': inputMovies.getAcquisition().getVoltage(),
'-Gpu': self.GPUIDs.get(),
'-LogFile': logFileBase,
}
if getVersion('MOTIONCOR2') != '03162016':
argsDict['-InitDose'] = preExp
argsDict['-OutStack'] = 1 if self.doSaveMovie else 0
if self._supportsMagCorrection() and self.doMagCor:
if self.useEst:
inputEst = self.inputEst.get().getOutputLog()
input_params = parseMagCorrInput(inputEst)
# mag dist angle is inverted due to a different convention
argsDict['-Mag'] = '%0.2f %0.2f %0.2f' % (
input_params[1], input_params[2], -1 * input_params[0])
else:
argsDict['-Mag'] = '%0.2f %0.2f %0.2f' % (
self.scaleMaj.get(), self.scaleMin.get(),
self.angDist.get())
args = ' -InMrc "%s" ' % movie.getBaseName()
args += ' '.join(
['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -Gain "%s" ' % inputMovies.getGain()
args += ' ' + self.extraParams2.get()
program = MOTIONCOR2_PATH
try:
self.runJob(program,
args,
cwd=movieFolder,
env=getEnviron(self.useMotioncor2))
self._fixMovie(movie)
# Compute PSDs
outMicFn = self._getExtraPath(self._getOutputMicName(movie))
if not os.path.exists(outMicFn):
# if only DW mic is saved
outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))
if self.doComputePSD:
uncorrectedPSD = movieBaseName + '_uncorrected'
correctedPSD = movieBaseName + '_corrected'
# Compute uncorrected avg mic
roi = [
self.cropOffsetX.get(),
self.cropOffsetY.get(),
self.cropDimX.get(),
self.cropDimY.get()
]
fakeShiftsFn = self.writeZeroShifts(movie)
self.averageMovie(movie,
fakeShiftsFn,
aveMicFn,
binFactor=self.binFactor.get(),
roi=roi,
dark=None,
gain=inputMovies.getGain())
self.computePSD(aveMicFn, uncorrectedPSD)
self.computePSD(outMicFn, correctedPSD)
self.composePSD(uncorrectedPSD + ".psd", correctedPSD + ".psd",
self._getPsdCorr(movie))
self._saveAlignmentPlots(movie)
if self._doComputeMicThumbnail():
self.computeThumbnail(
outMicFn, outputFn=self._getOutputMicThumbnail(movie))
except:
print("ERROR: Movie %s failed\n" % movie.getName())
def convertInputStep(self, particlesId):
""" Create the input file in STAR format as expected by Relion.
If the input particles comes from Relion, just link the file.
Params:
particlesId: use this parameters just to force redo of convert if
the input particles are changed.
"""
imgSet = self._getInputParticles()
imgStar = self._getFileName('input_star')
self.info("Converting set from '%s' into '%s'" %
(imgSet.getFileName(), imgStar))
# Pass stack file as None to avoid write the images files
writeSetOfParticles(imgSet, imgStar, self._getExtraPath())
if self.doCtfManualGroups:
self._splitInCTFGroups(imgStar)
if not self.IS_CLASSIFY:
if self.realignMovieFrames:
movieParticleSet = self.inputMovieParticles.get()
auxMovieParticles = self._createSetOfMovieParticles(
suffix='tmp')
auxMovieParticles.copyInfo(movieParticleSet)
# Discard the movie particles that are not present in the refinement set
for movieParticle in movieParticleSet:
particle = imgSet[movieParticle.getParticleId()]
if particle is not None:
auxMovieParticles.append(movieParticle)
writeSetOfParticles(
auxMovieParticles,
self._getFileName('movie_particles'),
None,
originalSet=imgSet,
postprocessImageRow=self._postprocessImageRow)
mdMovies = md.MetaData(self._getFileName('movie_particles'))
mdParts = md.MetaData(self._getFileName('input_star'))
if getVersion() == "1.4":
mdParts.renameColumn(md.RLN_IMAGE_NAME,
md.RLN_PARTICLE_ORI_NAME)
else:
mdParts.renameColumn(md.RLN_IMAGE_NAME,
md.RLN_PARTICLE_NAME)
mdParts.removeLabel(md.RLN_MICROGRAPH_NAME)
detectorPxSize = movieParticleSet.getAcquisition(
).getMagnification() * movieParticleSet.getSamplingRate(
) / 10000
mdAux = md.MetaData()
mdMovies.fillConstant(md.RLN_CTF_DETECTOR_PIXEL_SIZE,
detectorPxSize)
mdAux.join2(mdMovies, mdParts, md.RLN_PARTICLE_ID,
md.RLN_IMAGE_ID, md.INNER_JOIN)
mdAux.write(self._getFileName('movie_particles'),
md.MD_OVERWRITE)
cleanPath(auxMovieParticles.getFileName())
def _supportsMagCorrection(self):
return getVersion('MOTIONCOR2') not in ['03162016', '10192016']
def _oldVersion(self):
return True if getVersion() == '0.50' else False
def _processMovie(self, movie):
inputMovies = self.inputMovies.get()
movieFolder = self._getOutputMovieFolder(movie)
outputMicFn = self._getRelPath(self._getOutputMicName(movie),
movieFolder)
outputMovieFn = self._getRelPath(self._getOutputMovieName(movie),
movieFolder)
movieBaseName = pwutils.removeExt(movie.getFileName())
aveMicFn = movieBaseName + '_uncorrected_avg.mrc'
logFile = self._getRelPath(self._getMovieLogFile(movie),
movieFolder)
a0, aN = self._getRange(movie, 'align')
if not self.useMotioncor2:
# Get the number of frames and the range to be used
# for alignment and sum
s0, sN = self._getRange(movie, 'sum')
argsDict = {'-crx': self.cropOffsetX.get(),
'-cry': self.cropOffsetY.get(),
'-cdx': self.cropDimX.get(),
'-cdy': self.cropDimY.get(),
'-bin': self.binFactor.get(),
'-nst': '%d' % a0,
'-ned': '%d' % aN,
'-nss': '%d' % s0,
'-nes': '%d' % sN,
'-flg': logFile,
}
args = '"%s" ' % movie.getBaseName()
args += ' '.join(
['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -fgr "%s"' % inputMovies.getGain()
if inputMovies.getDark():
args += ' -fdr "%s"' % inputMovies.getDark()
if self.doSaveAveMic:
args += ' -fcs "%s" ' % outputMicFn
if self.doSaveMovie:
args += ' -fct "%s" -ssc 1' % outputMovieFn
args += ' -gpu %(GPU)s'
args += ' ' + self.extraParams.get()
program = MOTIONCORR_PATH
else:
logFileBase = (logFile.replace('0-Full.log', '').replace(
'0-Patch-Full.log', ''))
# default values for motioncor2 are (1, 1)
cropDimX = self.cropDimX.get() or 1
cropDimY = self.cropDimY.get() or 1
numbOfFrames = self._getNumberOfFrames(movie)
if self.doApplyDoseFilter:
preExp, dose = self._getCorrectedDose(inputMovies)
else:
preExp, dose = 0.0, 0.0
# reset values = 1 to 0 (motioncor2 does it automatically,
# but we need to keep this for consistency)
if self.patchX.get() == 1:
self.patchX.set(0)
if self.patchY.get() == 1:
self.patchY.set(0)
argsDict = {'-OutMrc': '"%s"' % outputMicFn,
'-Patch': '%d %d' % (self.patchX, self.patchY),
'-MaskCent': '%d %d' % (self.cropOffsetX,
self.cropOffsetY),
'-MaskSize': '%d %d' % (cropDimX, cropDimY),
'-FtBin': self.binFactor.get(),
'-Tol': self.tol.get(),
'-Group': self.group.get(),
'-FmDose': dose,
'-Throw': '%d' % a0,
'-Trunc': '%d' % (abs(aN - numbOfFrames + 1)),
'-PixSize': inputMovies.getSamplingRate(),
'-kV': inputMovies.getAcquisition().getVoltage(),
'-LogFile': logFileBase,
}
if getVersion('MOTIONCOR2') != '03162016':
argsDict['-InitDose'] = preExp
argsDict['-OutStack'] = 1 if self.doSaveMovie else 0
if self.isSemVersion():
if self.defectFile.get():
argsDict['-DefectFile'] = self.defectFile.get()
if self.versionGE('1.0.1'): # Patch overlap was introduced in 1.0.1
patchOverlap = self.getAttributeValue('patchOverlap', None)
if patchOverlap: # 0 or None is False
argsDict['-Patch'] += " %d" % patchOverlap
if self._supportsMagCorrection() and self.doMagCor:
if self.useEst:
inputEst = self.inputEst.get().getOutputLog()
if getVersion('MOTIONCOR2') == '01302017':
input_params = parseMagCorrInput(inputEst)
# this version uses stretch parameters as following:
# 1/maj, 1/min, -angle
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (
1.0 / input_params[1],
1.0 / input_params[2],
-1 * input_params[0])
else:
# While motioncor2 >=1.0.0 uses estimation params AS IS
input_params = parseMagEstOutput(inputEst)
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (
input_params[1],
input_params[2],
input_params[0])
else:
argsDict['-Mag'] = '%0.3f %0.3f %0.3f' % (self.scaleMaj,
self.scaleMin,
self.angDist)
ext = pwutils.getExt(movie.getFileName()).lower()
if ext in ['.mrc', '.mrcs']:
args = ' -InMrc "%s" ' % movie.getBaseName()
elif ext in ['.tif', '.tiff']:
args = ' -InTiff "%s" ' % movie.getBaseName()
else:
raise Exception("Unsupported format: %s" % ext)
args += ' '.join(['%s %s' % (k, v)
for k, v in argsDict.iteritems()])
if inputMovies.getGain():
args += ' -Gain "%s" ' % inputMovies.getGain()
if inputMovies.getDark():
args += ' -Dark "%s"' % inputMovies.getDark()
args += ' -Gpu %(GPU)s'
args += ' ' + self.extraParams2.get()
program = MOTIONCOR2_PATH
try:
self.runJob(program, args, cwd=movieFolder,
env=getEnviron(self.useMotioncor2))
self._fixMovie(movie)
# Compute PSDs
outMicFn = self._getExtraPath(self._getOutputMicName(movie))
if not os.path.exists(outMicFn):
# if only DW mic is saved
outMicFn = self._getExtraPath(self._getOutputMicWtName(movie))
def _extraWork():
if self.doComputePSD:
# Compute uncorrected avg mic
roi = [self.cropOffsetX.get(), self.cropOffsetY.get(),
self.cropDimX.get(), self.cropDimY.get()]
fakeShiftsFn = self.writeZeroShifts(movie)
self.averageMovie(movie, fakeShiftsFn, aveMicFn,
binFactor=self.binFactor.get(),
roi=roi, dark=inputMovies.getDark(),
gain=inputMovies.getGain())
self.computePSDs(movie, aveMicFn, outMicFn,
outputFnCorrected=self._getPsdJpeg(movie))
self._saveAlignmentPlots(movie)
if self._doComputeMicThumbnail():
self.computeThumbnail(outMicFn,
outputFn=self._getOutputMicThumbnail(
movie))
# This protocols takes control of clean up the temporary movie folder
# which is required mainly when using a thread for this extra work
self._cleanMovieFolder(movieFolder)
if self._useWorkerThread():
thread = Thread(target=_extraWork)
thread.start()
else:
_extraWork()
except:
print("ERROR: Movie %s failed\n" % movie.getName())
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam(
'protRefine',
PointerParam,
pointerClass="ProtRefine3D",
label='Select a previous refinement protocol',
help='Select any previous refinement protocol to get the '
'3D half maps. Note that is recomended that the '
'refinement protocol uses a gold-standard method.')
form.addSection(label='Masking')
form.addParam('doAutoMask',
BooleanParam,
default=True,
label='Perform automated masking?',
help='Perform automated masking, based on a density '
'threshold')
form.addParam('initMaskThreshold',
FloatParam,
default=0.02,
condition='doAutoMask',
label='Initial binarisation threshold',
help='This threshold is used to make an initial binary '
'mask from the average of the two unfiltered '
'half-reconstructions. If you do not know what '
'value to use, display one of the unfiltered '
'half-maps in a 3D surface rendering viewer, that '
'gives no noise peaks outside the reconstruction.')
form.addParam('extendInitMask',
IntParam,
default=3,
label='Mask pixels extension (px)',
condition='doAutoMask',
help='The initial binary mask is extended this number '
'of pixels in all directions.')
form.addParam('addMaskEdge',
IntParam,
default=6,
label='add soft-edge width (px)',
condition='doAutoMask',
help='The extended binary mask is further extended with '
'a raised-cosine soft edge of the specified width.')
form.addParam(
'mask',
PointerParam,
pointerClass='VolumeMask',
label='Provide a mask',
allowsNull=True,
condition='not doAutoMask',
help='Provide a soft mask where the protein is white (1) '
'and the solvent is black (0). Often, the softer '
'the mask the higher resolution estimates you will '
'get. A soft edge of 5-10 pixels is often a good '
'edge width.')
form.addSection(label='Sharpening')
form.addParam('mtf',
FileParam,
label='MTF-curve file',
help='User-provided STAR-file with the MTF-curve '
'of the detector.'
'Relion param: <--mtf>')
form.addParam('doAutoBfactor',
BooleanParam,
default=True,
label='Estimate B-factor automatically?',
help='If set to Yes, then the program will use the '
'automated procedure described by Rosenthal and '
'Henderson (2003, JMB) to estimate an overall '
'B-factor for your map, and sharpen it accordingly.')
line = form.addLine('B-factor resolution (A): ',
condition='doAutoBfactor',
help='There are the frequency (in Angstroms), '
'lowest and highest, that will be included in '
'the linear fit of the Guinier plot as '
'described in Rosenthal and Henderson '
'(2003, JMB).')
line.addParam('bfactorLowRes', FloatParam, default='10.0', label='low')
line.addParam('bfactorHighRes',
FloatParam,
default='0.0',
label='high')
form.addParam('bfactor',
FloatParam,
default=-350,
condition='not doAutoBfactor',
label='Provide B-factor:',
help='User-provided B-factor (in A^2) for map '
'sharpening, e.g. -400. Use negative values for '
'sharpening. Be careful: if you over-sharpen\n'
'your map, you may end up interpreting noise for '
'signal!\n'
'Relion param: *--adhoc_bfac*')
form.addSection(label='Filtering')
form.addParam('skipFscWeighting',
BooleanParam,
default=False,
label='Skip FSC-weighting for sharpening?',
help='If set to No (the default), then the output map '
'will be low-pass filtered according to the '
'mask-corrected, gold-standard FSC-curve. '
'Sometimes, it is also useful to provide an ad-hoc '
'low-pass filter (option below), as due to local '
'resolution variations some parts of the map may '
'be better and other parts may be worse than the '
'overall resolution as measured by the FSC. In '
'such cases, set this option to Yes and provide '
'an ad-hoc filter as described below.')
form.addParam('lowRes',
FloatParam,
default=5,
condition='skipFscWeighting',
label='Low-pass filter (A):',
help='This option allows one to low-pass filter the map '
'at a user-provided frequency (in Angstroms). When '
'using a resolution that is higher than the '
'gold-standard FSC-reported resolution, take care '
'not to interpret noise in the map for signal...')
if getVersion() == "2.0":
pass
form.addParam('filterEdgeWidth',
IntParam,
default=2,
expertLevel=LEVEL_ADVANCED,
label='Low-pass filter edge width:',
help='Width of the raised cosine on the low-pass filter '
'edge (in resolution shells)\n'
'Relion param: *--filter_edge_width*')
form.addParam('randomizeAtFsc',
FloatParam,
default=0.8,
expertLevel=LEVEL_ADVANCED,
label='Randomize phases threshold',
help='Randomize phases from the resolution where FSC '
'drops below this value\n'
'Relion param: *--randomize_at_fsc*')
form.addParallelSection(threads=0, mpi=0)
def _oldVersion(self):
return True if getVersion() == '0.50' else False
