def _insertPolishStep(self): imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') params = ' --i %s' % imgStar if isVersion2(): params += ' --o %s/shiny' % self._getExtraPath() else: params += ' --o shiny' params += ' --angpix %0.3f' % imgSet.getSamplingRate() params += ' --sym %s' % self.symmetryGroup.get() params += ' --sigma_nb %d' % self.stddevParticleDistance.get() params += ' --perframe_highres %0.3f' % self.highresLimitPerFrameMaps.get() params += ' --autob_lowres %0.3f' % self.lowresLimitBfactorEstimation.get() if not isVersion2(): params += ' --movie_frames_running_avg %d' % self.movieAvgWindow.get() params += ' --dont_read_old_files' else: params += ' --bfactor_running_avg %d' % self.avgFramesBfac.get() x, _, _ = imgSet.getDimensions() if self.maskRadius.get() >= x/2 or self.maskRadius.get() < 0: params += ' --bg_radius %d' % (x/2) else: params += ' --bg_radius %d' % self.maskRadius.get() if self.performBfactorWeighting: params += ' --mask %s' % self.maskForReconstructions.get().getFileName() else: params += ' --skip_bfactor_weighting' params += ' ' + self.extraParams.get() self._insertFunctionStep('polishStep', params)
def createMaskStep(self): argsDict = { '--i ': self.inputVolFn, '--ini_threshold ': self.threshold.get(), '--extend_inimask ': self.extend.get(), '--width_soft_edge ': self.edge.get() } if isVersion2() and self.initialLowPassFilterA.get() != -1: argsDict['--lowpass '] = self.initialLowPassFilterA.get() args = ' --o %s ' % self.maskFile args += ' '.join(['%s %s' % (k, v) for k, v in argsDict.iteritems()]) if self.doCompare: if self.operation.get() == AND: args += ' --and %s' % self.inputVol2Fn elif self.operation.get() == OR: args += ' --or %s' % self.inputVol2Fn elif self.operation.get() == AND_NOT: args += ' --and_not %s' % self.inputVol2Fn elif self.operation.get() == OR_NOT: args += ' --or_not %s' % self.inputVol2Fn if self.doInvert: args += ' --invert' self.runJob("relion_mask_create", args) return [self.maskFile]
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile( self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile( self._getPath( 'movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation( mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation( mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _insertPolishStep(self): imgSet = self._getInputParticles() imgStar = self._getFileName('movie_particles') params = ' --i %s' % imgStar if isVersion2(): params += ' --o %s/shiny' % self._getExtraPath() else: params += ' --o shiny' params += ' --angpix %0.3f' % imgSet.getSamplingRate() params += ' --sym %s' % self.symmetryGroup.get() params += ' --sigma_nb %d' % self.stddevParticleDistance.get() params += ' --perframe_highres %0.3f' % self.highresLimitPerFrameMaps.get( ) params += ' --autob_lowres %0.3f' % self.lowresLimitBfactorEstimation.get( ) if not isVersion2(): params += ' --movie_frames_running_avg %d' % self.movieAvgWindow.get( ) params += ' --dont_read_old_files' else: params += ' --bfactor_running_avg %d' % self.avgFramesBfac.get() x, _, _ = imgSet.getDimensions() if self.maskRadius.get() >= x / 2 or self.maskRadius.get() < 0: params += ' --bg_radius %d' % (x / 2) else: params += ' --bg_radius %d' % self.maskRadius.get() if self.performBfactorWeighting: params += ' --mask %s' % self.maskForReconstructions.get( ).getFileName() else: params += ' --skip_bfactor_weighting' params += ' ' + self.extraParams.get() self._insertFunctionStep('polishStep', params)
def _validateNormal(self): """ Should be overwritten in subclasses to return summary message for NORMAL EXECUTION. """ errors = [] self._validateDim(self._getInputParticles(), self.referenceVolume.get(), errors, 'Input particles', 'Reference volume') if isVersion2() and self.IS_3D: if self.solventFscMask and not self.referenceMask.get(): errors.append('When using solvent-corrected FSCs, ' 'please provide a reference mask.') return errors
def organizeDataStep(self): from convert import relionToLocation, locationToRelion if getVersion() == V1_3: mdColumn = md.RLN_PARTICLE_NAME else: mdColumn = md.RLN_PARTICLE_ORI_NAME shinyStar = self._getFileName('shiny') newDir = self._getExtraPath('polished_particles') pwutils.makePath(newDir) if not isVersion2(): pwutils.makePath(self._getExtraPath('shiny')) shinyOld = "shiny.star" inputFit = "movie_particles_shiny.star" try: pwutils.moveFile(shinyOld, shinyStar) pwutils.moveFile(self._getPath(inputFit), self._getExtraPath("shiny/all_movies_input_fit.star")) for half in self.PREFIXES: pwutils.moveFile(self._getPath('movie_particles_shiny_%sclass001_unfil.mrc' % half), self._getExtraPath('shiny/shiny_%sclass001_unfil.mrc' % half)) self._renameFiles('movie_particles_shiny_post*', 'movie_particles_') self._renameFiles('movie_particles_shiny*', 'movie_particles_shiny_') except: raise Exception('ERROR: some file(s) were not found!') # move polished particles from Tmp to Extra path # and restore previous mdColumn mdShiny = md.MetaData(shinyStar) oldPath = "" for objId in mdShiny: index, imgPath = relionToLocation(mdShiny.getValue(md.RLN_IMAGE_NAME, objId)) newPath = pwutils.join(newDir, str(imgPath).split('/')[-1]) newLoc = locationToRelion(index, newPath) mdShiny.setValue(md.RLN_IMAGE_NAME, newLoc, objId) if oldPath != imgPath and exists(imgPath): pwutils.moveFile(imgPath, newPath) oldPath = imgPath index2, imgPath2 = relionToLocation(mdShiny.getValue(mdColumn, objId)) absPath = os.path.realpath(imgPath2) newPath2 = 'Runs' + str(absPath).split('Runs')[1] newLoc2 = locationToRelion(index2, newPath2) mdShiny.setValue(mdColumn, newLoc2, objId) mdShiny.write(shinyStar, md.MD_OVERWRITE) pwutils.cleanPath(self._getExtraPath('shiny/Runs'))
def _validateNormal(self): """ Should be overwritten in subclasses to return summary message for NORMAL EXECUTION. """ errors = [] # We we scale the input volume to have the same size as the particles... # so no need to validate the following # self._validateDim(self._getInputParticles(), self.referenceVolume.get(), # errors, 'Input particles', 'Reference volume') if isVersion2() and self.IS_3D: if self.solventFscMask and not self.referenceMask.get(): errors.append('When using solvent-corrected FSCs, ' 'please provide a reference mask.') return errors
def isDisabled(cls): return isVersion2() or isVersion3()
def isDisabled(cls): return not isVersion2()
def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMovieParticles', PointerParam, pointerClass='SetOfMovieParticles', label='Input refined movie particles', help='Select the refined movie particles ' 'from Relion 3D auto-refine run.') form.addParam('maskRadius', IntParam, default=100, label='Particle mask RADIUS (px)', help='Radius of the circular mask that will be used ' 'to define the background area.') form.addParam('extraParams', StringParam, default='', expertLevel=LEVEL_ADVANCED, label='Additional parameters', help="In this box command-line arguments may be " "provided that are not generated by the GUI. This " "may be useful for testing developmental options " "and/or expert use of the program.") form.addSection('Movement') form.addParam('linerFitParticleMovements', BooleanParam, default=True, label='Linear fit particle movements?', help='If set to Yes, then the program will fit linear ' 'tracks (in X-Y and in time) through the estimated ' 'movie tracks in the input STAR file. For small ' 'particles (e.g. < 1MDa) this will lead to more ' 'robust beam-induced movement modelling. Because ' 'particles that are close to each other on a ' 'micrograph often move in similar directions, the ' 'estimated tracks from neighbouring particles may ' 'also be included in the fitting of each particle. ' 'Again, in particular for smaller particles this ' 'may improve the robustness of the fits.') if not isVersion2(): form.addParam('movieAvgWindow', FloatParam, default=5, label='Running average window', help='The individual movie frames will be ' 'averaged using a running average window ' 'with the specified width. Use an odd ' 'number. The optimal value will depend on ' 'the SNR in the individual movie frames. ' 'For ribosomes, we used a value of 5, ' 'where each movie frame integrated ' 'approximately 1 electron per squared ' 'Angstrom.') form.addParam('stddevParticleDistance', IntParam, default=100, condition='linerFitParticleMovements', label='Stddev on particle distance (px)', help='This value determines how much neighbouring ' 'particles contribute to the fit of the movements ' 'of each particle. This value is the standard ' 'deviation of a Gaussian on the inter-particle ' 'distance. Larger values mean that particles that ' 'are further away still contribute more. Particles ' 'beyond 3 standard deviations are excluded from ' 'the fit. Very large values will lead to all ' 'fitted tracks pointing in the same direction. ' 'A value of zero means that each particle is ' 'fitted independently.') form.addSection('Damage') form.addParam('performBfactorWeighting', BooleanParam, default=True, label='Perform B-factor weighting?', help='If set to Yes, then the program will estimate ' 'a resolution-dependent weight for each movie ' 'frames by calculating independent ' 'half-reconstructions for each movie frame ' 'separately. Gold-standard FSCs between these ' 'are then converted into relative Guinier plots, ' 'through which straight lines are fitted. Linear ' 'fits are often suitable beyond 20A resolution. ' 'Small particles may not yield such high ' 'resolutions for the individual-frame ' 'reconstructions. Therefore, in some cases ' 'it may be better to skip this step. It is ' 'however recommended to always try and perform ' 'B-factor weighting, and to inspect the output ' 'bfactors.star and guinier.star files, as ' 'adequate weighting may significantly improve ' 'resolution in the final map.') form.addParam('highresLimitPerFrameMaps', FloatParam, default=5, condition='performBfactorWeighting', label='Highres-limit per-frame maps (A)', help='To estimate the resolution and dose dependency of ' 'the radiation damage, the program will calculate ' 'reconstructions from all first movie frames, ' 'second movie frames, etc. These per-frame ' 'reconstructions will have lower resolution than ' 'the reconstruction from all-frames. To speed up ' 'the calculations (and reduce memory ' 'requirements), the per-frame reconstructions may ' 'be limited in resolution using this parameter. ' 'One can inspect the output STAR files of the ' 'per-frame reconstructions to check afterwards ' 'that this value was not chosen lower than the ' 'actual resolution of these reconstruction') form.addParam('lowresLimitBfactorEstimation', FloatParam, default=20, condition='performBfactorWeighting', label='Lowres-limit B-factor estimation (A)', help='This value describes the lowest resolution that ' 'is included in the B-factor estimation of the ' 'per-frame reconstructions. Because the power ' 'spectrum of per-frame reconstructions is compared ' 'to the power spectrum of the reconstruction from ' 'all frames, a much lower value than the 10A ' 'described in the Rosenthal and Henderson (2003) ' 'paper in JMB can be used. Probably a value around ' '20A is still OK.') if isVersion2(): form.addParam('avgFramesBfac', IntParam, default=1, condition='performBfactorWeighting', label='Average frames B-factor estimation', help='B-factors for each movie frame will be estimated ' 'from reconstructions of all particles for that ' 'movie frame. Single-frame reconstructions ' 'sometimes give not enough signal to estimate ' 'reliable B-factors. This option allows one to ' 'calculate the B-factors from running averages of ' 'movie frames. The value specified should be an ' 'odd number. Calculating B-factors from multiple ' 'movie frames improves the SNR in the reconstructions, ' 'but limits the estimation of sudden changes in ' 'B-factors throughout the movie, for example in ' 'the first few frames when beam-induced movement ' 'is very rapid. Therefore, one should not use ' 'higher values than strictly necessary.') form.addParam('maskForReconstructions', PointerParam, pointerClass='VolumeMask', label='Mask for the reconstructions', allowsNull=True, help='A continuous mask with values between 0 (solvent) ' 'and 1 (protein). You may provide the same mask ' 'that was used in the post-processing of the ' 'corresponding 3D auto-refine jobs before the movie ' 'processing.') self.addSymmetry(form) form.addParallelSection(threads=0, mpi=3)
def _defineParams(self, form): form.addSection(label='Mask generation') form.addParam( 'inputVolume', params.PointerParam, pointerClass="Volume", label="Input volume", help="Select the volume that will be used to create the mask") if isVersion2(): form.addParam( 'initialLowPassFilterA', params.FloatParam, default=-1, label='Lowpass filter map by (A)', help='Lowpass filter that will be applied to the input map, ' 'prior to binarization. To calculate solvent masks, a ' 'lowpass filter of 15-20A may work well.') # TODO: add wizard form.addParam('threshold', params.FloatParam, default=0.02, label='Initial binarisation threshold', help="This threshold is used to make an initial binary " "mask from the average of the two unfiltered " "half-reconstructions. If you don't know what " "value to use, display one of the unfiltered " "half-maps in a 3D surface rendering viewer and " "find the lowest threshold that gives no noise " "peaks outside the reconstruction.") form.addParam( 'doCompare', params.BooleanParam, default=False, expertLevel=params.LEVEL_ADVANCED, label='Compare with another volume to produce a mask?', help='Logical comparison of two input volumes to produce a mask') form.addParam( 'inputVolume2', params.PointerParam, pointerClass="Volume", condition='doCompare', expertLevel=params.LEVEL_ADVANCED, label="Input volume (second)", help="Select the volume that will be compared to the first one") form.addParam( 'operation', params.EnumParam, default=AND, condition='doCompare', expertLevel=params.LEVEL_ADVANCED, label='Operation', choices=['AND', 'OR', 'AND_NOT', 'OR_NOT'], help='*AND*: Pixels in the initial mask will be one if the input ' 'AND the second volume are above the threshold value.\n' '*OR*: Pixels in the initial mask will be one if the input ' 'OR the second volume are above the threshold value.\n' '*AND_NOT*: pixels in the initial mask will be one if ' 'the input is above the threshold AND the second volume ' 'is below it.\n*OR_NOT*: pixels in the initial mask will be ' 'one if the input is above the threshold OR the second ' 'volume is below it.') form.addParam( 'extend', params.IntParam, default=3, label='Extend binary mask by (px)', help='The initial binary mask is extended this number of ' 'pixels in all directions.') form.addParam('edge', params.IntParam, default=3, label='Add a soft-edge (px)', help='The extended binary mask is further extended with ' 'a raised-cosine soft edge of the specified width.') form.addParam('doInvert', params.BooleanParam, default=False, expertLevel=params.LEVEL_ADVANCED, label='Invert final mask', help='Invert the final mask')
def _defineParams(self, form): form.addSection(label='Input') form.addParam('inputMovieParticles', PointerParam, pointerClass='SetOfMovieParticles', label='Input refined movie particles', help='Select the refined movie particles ' 'from Relion 3D auto-refine run.') form.addParam('maskRadius', IntParam, default=100, label='Particle mask RADIUS (px)', help='Radius of the circular mask that will be used ' 'to define the background area.') form.addParam('extraParams', StringParam, default='', expertLevel=LEVEL_ADVANCED, label='Additional parameters', help="In this box command-line arguments may be " "provided that are not generated by the GUI. This " "may be useful for testing developmental options " "and/or expert use of the program.") form.addSection('Movement') form.addParam('linerFitParticleMovements', BooleanParam, default=True, label='Linear fit particle movements?', help='If set to Yes, then the program will fit linear ' 'tracks (in X-Y and in time) through the estimated ' 'movie tracks in the input STAR file. For small ' 'particles (e.g. < 1MDa) this will lead to more ' 'robust beam-induced movement modelling. Because ' 'particles that are close to each other on a ' 'micrograph often move in similar directions, the ' 'estimated tracks from neighbouring particles may ' 'also be included in the fitting of each particle. ' 'Again, in particular for smaller particles this ' 'may improve the robustness of the fits.') if not isVersion2(): form.addParam('movieAvgWindow', FloatParam, default=5, label='Running average window', help='The individual movie frames will be ' 'averaged using a running average window ' 'with the specified width. Use an odd ' 'number. The optimal value will depend on ' 'the SNR in the individual movie frames. ' 'For ribosomes, we used a value of 5, ' 'where each movie frame integrated ' 'approximately 1 electron per squared ' 'Angstrom.') form.addParam('stddevParticleDistance', IntParam, default=100, condition='linerFitParticleMovements', label='Stddev on particle distance (px)', help='This value determines how much neighbouring ' 'particles contribute to the fit of the movements ' 'of each particle. This value is the standard ' 'deviation of a Gaussian on the inter-particle ' 'distance. Larger values mean that particles that ' 'are further away still contribute more. Particles ' 'beyond 3 standard deviations are excluded from ' 'the fit. Very large values will lead to all ' 'fitted tracks pointing in the same direction. ' 'A value of zero means that each particle is ' 'fitted independently.') form.addSection('Damage') form.addParam('performBfactorWeighting', BooleanParam, default=True, label='Perform B-factor weighting?', help='If set to Yes, then the program will estimate ' 'a resolution-dependent weight for each movie ' 'frames by calculating independent ' 'half-reconstructions for each movie frame ' 'separately. Gold-standard FSCs between these ' 'are then converted into relative Guinier plots, ' 'through which straight lines are fitted. Linear ' 'fits are often suitable beyond 20A resolution. ' 'Small particles may not yield such high ' 'resolutions for the individual-frame ' 'reconstructions. Therefore, in some cases ' 'it may be better to skip this step. It is ' 'however recommended to always try and perform ' 'B-factor weighting, and to inspect the output ' 'bfactors.star and guinier.star files, as ' 'adequate weighting may significantly improve ' 'resolution in the final map.') form.addParam('highresLimitPerFrameMaps', FloatParam, default=5, condition='performBfactorWeighting', label='Highres-limit per-frame maps (A)', help='To estimate the resolution and dose dependency of ' 'the radiation damage, the program will calculate ' 'reconstructions from all first movie frames, ' 'second movie frames, etc. These per-frame ' 'reconstructions will have lower resolution than ' 'the reconstruction from all-frames. To speed up ' 'the calculations (and reduce memory ' 'requirements), the per-frame reconstructions may ' 'be limited in resolution using this parameter. ' 'One can inspect the output STAR files of the ' 'per-frame reconstructions to check afterwards ' 'that this value was not chosen lower than the ' 'actual resolution of these reconstruction') form.addParam('lowresLimitBfactorEstimation', FloatParam, default=20, condition='performBfactorWeighting', label='Lowres-limit B-factor estimation (A)', help='This value describes the lowest resolution that ' 'is included in the B-factor estimation of the ' 'per-frame reconstructions. Because the power ' 'spectrum of per-frame reconstructions is compared ' 'to the power spectrum of the reconstruction from ' 'all frames, a much lower value than the 10A ' 'described in the Rosenthal and Henderson (2003) ' 'paper in JMB can be used. Probably a value around ' '20A is still OK.') if isVersion2(): form.addParam( 'avgFramesBfac', IntParam, default=1, condition='performBfactorWeighting', label='Average frames B-factor estimation', help='B-factors for each movie frame will be estimated ' 'from reconstructions of all particles for that ' 'movie frame. Single-frame reconstructions ' 'sometimes give not enough signal to estimate ' 'reliable B-factors. This option allows one to ' 'calculate the B-factors from running averages of ' 'movie frames. The value specified should be an ' 'odd number. Calculating B-factors from multiple ' 'movie frames improves the SNR in the reconstructions, ' 'but limits the estimation of sudden changes in ' 'B-factors throughout the movie, for example in ' 'the first few frames when beam-induced movement ' 'is very rapid. Therefore, one should not use ' 'higher values than strictly necessary.') form.addParam('maskForReconstructions', PointerParam, pointerClass='VolumeMask', label='Mask for the reconstructions', allowsNull=True, help='A continuous mask with values between 0 (solvent) ' 'and 1 (protein). You may provide the same mask ' 'that was used in the post-processing of the ' 'corresponding 3D auto-refine jobs before the movie ' 'processing.') self.addSymmetry(form) form.addParallelSection(threads=0, mpi=3)