def _insertPolishStep(self):
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
params = ' --i %s' % imgStar
if isVersion2():
params += ' --o %s/shiny' % self._getExtraPath()
else:
params += ' --o shiny'
params += ' --angpix %0.3f' % imgSet.getSamplingRate()
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --sigma_nb %d' % self.stddevParticleDistance.get()
params += ' --perframe_highres %0.3f' % self.highresLimitPerFrameMaps.get()
params += ' --autob_lowres %0.3f' % self.lowresLimitBfactorEstimation.get()
if not isVersion2():
params += ' --movie_frames_running_avg %d' % self.movieAvgWindow.get()
params += ' --dont_read_old_files'
else:
params += ' --bfactor_running_avg %d' % self.avgFramesBfac.get()
x, _, _ = imgSet.getDimensions()
if self.maskRadius.get() >= x/2 or self.maskRadius.get() < 0:
params += ' --bg_radius %d' % (x/2)
else:
params += ' --bg_radius %d' % self.maskRadius.get()
if self.performBfactorWeighting:
params += ' --mask %s' % self.maskForReconstructions.get().getFileName()
else:
params += ' --skip_bfactor_weighting'
params += ' ' + self.extraParams.get()
self._insertFunctionStep('polishStep', params)
def createMaskStep(self):
argsDict = {
'--i ': self.inputVolFn,
'--ini_threshold ': self.threshold.get(),
'--extend_inimask ': self.extend.get(),
'--width_soft_edge ': self.edge.get()
}
if isVersion2() and self.initialLowPassFilterA.get() != -1:
argsDict['--lowpass '] = self.initialLowPassFilterA.get()
args = ' --o %s ' % self.maskFile
args += ' '.join(['%s %s' % (k, v) for k, v in argsDict.iteritems()])
if self.doCompare:
if self.operation.get() == AND:
args += ' --and %s' % self.inputVol2Fn
elif self.operation.get() == OR:
args += ' --or %s' % self.inputVol2Fn
elif self.operation.get() == AND_NOT:
args += ' --and_not %s' % self.inputVol2Fn
elif self.operation.get() == OR_NOT:
args += ' --or_not %s' % self.inputVol2Fn
if self.doInvert:
args += ' --invert'
self.runJob("relion_mask_create", args)
return [self.maskFile]
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(
self._getPath(inputFit),
self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(
self._getPath(
'movie_particles_shiny_%sclass001_unfil.mrc' %
half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' %
half))
self._renameFiles('movie_particles_shiny_post*',
'movie_particles_')
self._renameFiles('movie_particles_shiny*',
'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(
mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(
mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _insertPolishStep(self):
imgSet = self._getInputParticles()
imgStar = self._getFileName('movie_particles')
params = ' --i %s' % imgStar
if isVersion2():
params += ' --o %s/shiny' % self._getExtraPath()
else:
params += ' --o shiny'
params += ' --angpix %0.3f' % imgSet.getSamplingRate()
params += ' --sym %s' % self.symmetryGroup.get()
params += ' --sigma_nb %d' % self.stddevParticleDistance.get()
params += ' --perframe_highres %0.3f' % self.highresLimitPerFrameMaps.get(
)
params += ' --autob_lowres %0.3f' % self.lowresLimitBfactorEstimation.get(
)
if not isVersion2():
params += ' --movie_frames_running_avg %d' % self.movieAvgWindow.get(
)
params += ' --dont_read_old_files'
else:
params += ' --bfactor_running_avg %d' % self.avgFramesBfac.get()
x, _, _ = imgSet.getDimensions()
if self.maskRadius.get() >= x / 2 or self.maskRadius.get() < 0:
params += ' --bg_radius %d' % (x / 2)
else:
params += ' --bg_radius %d' % self.maskRadius.get()
if self.performBfactorWeighting:
params += ' --mask %s' % self.maskForReconstructions.get(
).getFileName()
else:
params += ' --skip_bfactor_weighting'
params += ' ' + self.extraParams.get()
self._insertFunctionStep('polishStep', params)
def _validateNormal(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
errors = []
self._validateDim(self._getInputParticles(),
self.referenceVolume.get(), errors,
'Input particles', 'Reference volume')
if isVersion2() and self.IS_3D:
if self.solventFscMask and not self.referenceMask.get():
errors.append('When using solvent-corrected FSCs, '
'please provide a reference mask.')
return errors
def organizeDataStep(self):
from convert import relionToLocation, locationToRelion
if getVersion() == V1_3:
mdColumn = md.RLN_PARTICLE_NAME
else:
mdColumn = md.RLN_PARTICLE_ORI_NAME
shinyStar = self._getFileName('shiny')
newDir = self._getExtraPath('polished_particles')
pwutils.makePath(newDir)
if not isVersion2():
pwutils.makePath(self._getExtraPath('shiny'))
shinyOld = "shiny.star"
inputFit = "movie_particles_shiny.star"
try:
pwutils.moveFile(shinyOld, shinyStar)
pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star"))
for half in self.PREFIXES:
pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half),
self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half))
self._renameFiles('movie_particles_shiny_post*', 'movie_particles_')
self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_')
except:
raise Exception('ERROR: some file(s) were not found!')
# move polished particles from Tmp to Extra path
# and restore previous mdColumn
mdShiny = md.MetaData(shinyStar)
oldPath = ""
for objId in mdShiny:
index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId))
newPath = pwutils.join(newDir, str(imgPath).split('/')[-1])
newLoc = locationToRelion(index, newPath)
mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId)
if oldPath != imgPath and exists(imgPath):
pwutils.moveFile(imgPath, newPath)
oldPath = imgPath
index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId))
absPath = os.path.realpath(imgPath2)
newPath2 = 'Runs' + str(absPath).split('Runs')[1]
newLoc2 = locationToRelion(index2, newPath2)
mdShiny.setValue(mdColumn, newLoc2, objId)
mdShiny.write(shinyStar, md.MD_OVERWRITE)
pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _validateNormal(self):
""" Should be overwritten in subclasses to
return summary message for NORMAL EXECUTION.
"""
errors = []
# We we scale the input volume to have the same size as the particles...
# so no need to validate the following
# self._validateDim(self._getInputParticles(), self.referenceVolume.get(),
# errors, 'Input particles', 'Reference volume')
if isVersion2() and self.IS_3D:
if self.solventFscMask and not self.referenceMask.get():
errors.append('When using solvent-corrected FSCs, '
'please provide a reference mask.')
return errors
def isDisabled(cls):
return isVersion2() or isVersion3()
def isDisabled(cls):
return not isVersion2()
def isDisabled(cls):
return not isVersion2()
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMovieParticles', PointerParam,
pointerClass='SetOfMovieParticles',
label='Input refined movie particles',
help='Select the refined movie particles '
'from Relion 3D auto-refine run.')
form.addParam('maskRadius', IntParam, default=100,
label='Particle mask RADIUS (px)',
help='Radius of the circular mask that will be used '
'to define the background area.')
form.addParam('extraParams', StringParam, default='',
expertLevel=LEVEL_ADVANCED,
label='Additional parameters',
help="In this box command-line arguments may be "
"provided that are not generated by the GUI. This "
"may be useful for testing developmental options "
"and/or expert use of the program.")
form.addSection('Movement')
form.addParam('linerFitParticleMovements', BooleanParam, default=True,
label='Linear fit particle movements?',
help='If set to Yes, then the program will fit linear '
'tracks (in X-Y and in time) through the estimated '
'movie tracks in the input STAR file. For small '
'particles (e.g. < 1MDa) this will lead to more '
'robust beam-induced movement modelling. Because '
'particles that are close to each other on a '
'micrograph often move in similar directions, the '
'estimated tracks from neighbouring particles may '
'also be included in the fitting of each particle. '
'Again, in particular for smaller particles this '
'may improve the robustness of the fits.')
if not isVersion2():
form.addParam('movieAvgWindow', FloatParam, default=5,
label='Running average window',
help='The individual movie frames will be '
'averaged using a running average window '
'with the specified width. Use an odd '
'number. The optimal value will depend on '
'the SNR in the individual movie frames. '
'For ribosomes, we used a value of 5, '
'where each movie frame integrated '
'approximately 1 electron per squared '
'Angstrom.')
form.addParam('stddevParticleDistance', IntParam, default=100,
condition='linerFitParticleMovements',
label='Stddev on particle distance (px)',
help='This value determines how much neighbouring '
'particles contribute to the fit of the movements '
'of each particle. This value is the standard '
'deviation of a Gaussian on the inter-particle '
'distance. Larger values mean that particles that '
'are further away still contribute more. Particles '
'beyond 3 standard deviations are excluded from '
'the fit. Very large values will lead to all '
'fitted tracks pointing in the same direction. '
'A value of zero means that each particle is '
'fitted independently.')
form.addSection('Damage')
form.addParam('performBfactorWeighting', BooleanParam, default=True,
label='Perform B-factor weighting?',
help='If set to Yes, then the program will estimate '
'a resolution-dependent weight for each movie '
'frames by calculating independent '
'half-reconstructions for each movie frame '
'separately. Gold-standard FSCs between these '
'are then converted into relative Guinier plots, '
'through which straight lines are fitted. Linear '
'fits are often suitable beyond 20A resolution. '
'Small particles may not yield such high '
'resolutions for the individual-frame '
'reconstructions. Therefore, in some cases '
'it may be better to skip this step. It is '
'however recommended to always try and perform '
'B-factor weighting, and to inspect the output '
'bfactors.star and guinier.star files, as '
'adequate weighting may significantly improve '
'resolution in the final map.')
form.addParam('highresLimitPerFrameMaps', FloatParam, default=5,
condition='performBfactorWeighting',
label='Highres-limit per-frame maps (A)',
help='To estimate the resolution and dose dependency of '
'the radiation damage, the program will calculate '
'reconstructions from all first movie frames, '
'second movie frames, etc. These per-frame '
'reconstructions will have lower resolution than '
'the reconstruction from all-frames. To speed up '
'the calculations (and reduce memory '
'requirements), the per-frame reconstructions may '
'be limited in resolution using this parameter. '
'One can inspect the output STAR files of the '
'per-frame reconstructions to check afterwards '
'that this value was not chosen lower than the '
'actual resolution of these reconstruction')
form.addParam('lowresLimitBfactorEstimation', FloatParam, default=20,
condition='performBfactorWeighting',
label='Lowres-limit B-factor estimation (A)',
help='This value describes the lowest resolution that '
'is included in the B-factor estimation of the '
'per-frame reconstructions. Because the power '
'spectrum of per-frame reconstructions is compared '
'to the power spectrum of the reconstruction from '
'all frames, a much lower value than the 10A '
'described in the Rosenthal and Henderson (2003) '
'paper in JMB can be used. Probably a value around '
'20A is still OK.')
if isVersion2():
form.addParam('avgFramesBfac', IntParam, default=1,
condition='performBfactorWeighting',
label='Average frames B-factor estimation',
help='B-factors for each movie frame will be estimated '
'from reconstructions of all particles for that '
'movie frame. Single-frame reconstructions '
'sometimes give not enough signal to estimate '
'reliable B-factors. This option allows one to '
'calculate the B-factors from running averages of '
'movie frames. The value specified should be an '
'odd number. Calculating B-factors from multiple '
'movie frames improves the SNR in the reconstructions, '
'but limits the estimation of sudden changes in '
'B-factors throughout the movie, for example in '
'the first few frames when beam-induced movement '
'is very rapid. Therefore, one should not use '
'higher values than strictly necessary.')
form.addParam('maskForReconstructions', PointerParam,
pointerClass='VolumeMask',
label='Mask for the reconstructions', allowsNull=True,
help='A continuous mask with values between 0 (solvent) '
'and 1 (protein). You may provide the same mask '
'that was used in the post-processing of the '
'corresponding 3D auto-refine jobs before the movie '
'processing.')
self.addSymmetry(form)
form.addParallelSection(threads=0, mpi=3)
def _defineParams(self, form):
form.addSection(label='Mask generation')
form.addParam(
'inputVolume',
params.PointerParam,
pointerClass="Volume",
label="Input volume",
help="Select the volume that will be used to create the mask")
if isVersion2():
form.addParam(
'initialLowPassFilterA',
params.FloatParam,
default=-1,
label='Lowpass filter map by (A)',
help='Lowpass filter that will be applied to the input map, '
'prior to binarization. To calculate solvent masks, a '
'lowpass filter of 15-20A may work well.')
# TODO: add wizard
form.addParam('threshold',
params.FloatParam,
default=0.02,
label='Initial binarisation threshold',
help="This threshold is used to make an initial binary "
"mask from the average of the two unfiltered "
"half-reconstructions. If you don't know what "
"value to use, display one of the unfiltered "
"half-maps in a 3D surface rendering viewer and "
"find the lowest threshold that gives no noise "
"peaks outside the reconstruction.")
form.addParam(
'doCompare',
params.BooleanParam,
default=False,
expertLevel=params.LEVEL_ADVANCED,
label='Compare with another volume to produce a mask?',
help='Logical comparison of two input volumes to produce a mask')
form.addParam(
'inputVolume2',
params.PointerParam,
pointerClass="Volume",
condition='doCompare',
expertLevel=params.LEVEL_ADVANCED,
label="Input volume (second)",
help="Select the volume that will be compared to the first one")
form.addParam(
'operation',
params.EnumParam,
default=AND,
condition='doCompare',
expertLevel=params.LEVEL_ADVANCED,
label='Operation',
choices=['AND', 'OR', 'AND_NOT', 'OR_NOT'],
help='*AND*: Pixels in the initial mask will be one if the input '
'AND the second volume are above the threshold value.\n'
'*OR*: Pixels in the initial mask will be one if the input '
'OR the second volume are above the threshold value.\n'
'*AND_NOT*: pixels in the initial mask will be one if '
'the input is above the threshold AND the second volume '
'is below it.\n*OR_NOT*: pixels in the initial mask will be '
'one if the input is above the threshold OR the second '
'volume is below it.')
form.addParam(
'extend',
params.IntParam,
default=3,
label='Extend binary mask by (px)',
help='The initial binary mask is extended this number of '
'pixels in all directions.')
form.addParam('edge',
params.IntParam,
default=3,
label='Add a soft-edge (px)',
help='The extended binary mask is further extended with '
'a raised-cosine soft edge of the specified width.')
form.addParam('doInvert',
params.BooleanParam,
default=False,
expertLevel=params.LEVEL_ADVANCED,
label='Invert final mask',
help='Invert the final mask')
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMovieParticles',
PointerParam,
pointerClass='SetOfMovieParticles',
label='Input refined movie particles',
help='Select the refined movie particles '
'from Relion 3D auto-refine run.')
form.addParam('maskRadius',
IntParam,
default=100,
label='Particle mask RADIUS (px)',
help='Radius of the circular mask that will be used '
'to define the background area.')
form.addParam('extraParams',
StringParam,
default='',
expertLevel=LEVEL_ADVANCED,
label='Additional parameters',
help="In this box command-line arguments may be "
"provided that are not generated by the GUI. This "
"may be useful for testing developmental options "
"and/or expert use of the program.")
form.addSection('Movement')
form.addParam('linerFitParticleMovements',
BooleanParam,
default=True,
label='Linear fit particle movements?',
help='If set to Yes, then the program will fit linear '
'tracks (in X-Y and in time) through the estimated '
'movie tracks in the input STAR file. For small '
'particles (e.g. < 1MDa) this will lead to more '
'robust beam-induced movement modelling. Because '
'particles that are close to each other on a '
'micrograph often move in similar directions, the '
'estimated tracks from neighbouring particles may '
'also be included in the fitting of each particle. '
'Again, in particular for smaller particles this '
'may improve the robustness of the fits.')
if not isVersion2():
form.addParam('movieAvgWindow',
FloatParam,
default=5,
label='Running average window',
help='The individual movie frames will be '
'averaged using a running average window '
'with the specified width. Use an odd '
'number. The optimal value will depend on '
'the SNR in the individual movie frames. '
'For ribosomes, we used a value of 5, '
'where each movie frame integrated '
'approximately 1 electron per squared '
'Angstrom.')
form.addParam('stddevParticleDistance',
IntParam,
default=100,
condition='linerFitParticleMovements',
label='Stddev on particle distance (px)',
help='This value determines how much neighbouring '
'particles contribute to the fit of the movements '
'of each particle. This value is the standard '
'deviation of a Gaussian on the inter-particle '
'distance. Larger values mean that particles that '
'are further away still contribute more. Particles '
'beyond 3 standard deviations are excluded from '
'the fit. Very large values will lead to all '
'fitted tracks pointing in the same direction. '
'A value of zero means that each particle is '
'fitted independently.')
form.addSection('Damage')
form.addParam('performBfactorWeighting',
BooleanParam,
default=True,
label='Perform B-factor weighting?',
help='If set to Yes, then the program will estimate '
'a resolution-dependent weight for each movie '
'frames by calculating independent '
'half-reconstructions for each movie frame '
'separately. Gold-standard FSCs between these '
'are then converted into relative Guinier plots, '
'through which straight lines are fitted. Linear '
'fits are often suitable beyond 20A resolution. '
'Small particles may not yield such high '
'resolutions for the individual-frame '
'reconstructions. Therefore, in some cases '
'it may be better to skip this step. It is '
'however recommended to always try and perform '
'B-factor weighting, and to inspect the output '
'bfactors.star and guinier.star files, as '
'adequate weighting may significantly improve '
'resolution in the final map.')
form.addParam('highresLimitPerFrameMaps',
FloatParam,
default=5,
condition='performBfactorWeighting',
label='Highres-limit per-frame maps (A)',
help='To estimate the resolution and dose dependency of '
'the radiation damage, the program will calculate '
'reconstructions from all first movie frames, '
'second movie frames, etc. These per-frame '
'reconstructions will have lower resolution than '
'the reconstruction from all-frames. To speed up '
'the calculations (and reduce memory '
'requirements), the per-frame reconstructions may '
'be limited in resolution using this parameter. '
'One can inspect the output STAR files of the '
'per-frame reconstructions to check afterwards '
'that this value was not chosen lower than the '
'actual resolution of these reconstruction')
form.addParam('lowresLimitBfactorEstimation',
FloatParam,
default=20,
condition='performBfactorWeighting',
label='Lowres-limit B-factor estimation (A)',
help='This value describes the lowest resolution that '
'is included in the B-factor estimation of the '
'per-frame reconstructions. Because the power '
'spectrum of per-frame reconstructions is compared '
'to the power spectrum of the reconstruction from '
'all frames, a much lower value than the 10A '
'described in the Rosenthal and Henderson (2003) '
'paper in JMB can be used. Probably a value around '
'20A is still OK.')
if isVersion2():
form.addParam(
'avgFramesBfac',
IntParam,
default=1,
condition='performBfactorWeighting',
label='Average frames B-factor estimation',
help='B-factors for each movie frame will be estimated '
'from reconstructions of all particles for that '
'movie frame. Single-frame reconstructions '
'sometimes give not enough signal to estimate '
'reliable B-factors. This option allows one to '
'calculate the B-factors from running averages of '
'movie frames. The value specified should be an '
'odd number. Calculating B-factors from multiple '
'movie frames improves the SNR in the reconstructions, '
'but limits the estimation of sudden changes in '
'B-factors throughout the movie, for example in '
'the first few frames when beam-induced movement '
'is very rapid. Therefore, one should not use '
'higher values than strictly necessary.')
form.addParam('maskForReconstructions',
PointerParam,
pointerClass='VolumeMask',
label='Mask for the reconstructions',
allowsNull=True,
help='A continuous mask with values between 0 (solvent) '
'and 1 (protein). You may provide the same mask '
'that was used in the post-processing of the '
'corresponding 3D auto-refine jobs before the movie '
'processing.')
self.addSymmetry(form)
form.addParallelSection(threads=0, mpi=3)
