class Angles(EMObject):
"""Represents a triplet of angles"""
def __init__(self, **args):
EMObject.__init__(self, **args)
self._angleY = Float()
self._angleY2 = Float()
self._angleTilt = Float()
def setAngles(self, angleY, angleY2, angleTilt):
self._angleY.set(angleY)
self._angleY2.set(angleY2)
self._angleTilt.set(angleTilt)
def getAngles(self):
return (self._angleY.get(), self._angleY2.get(), self._angleTilt.get())
def createOutputStep(self):
outputVols = self._createSetOfVolumes()
for vol in self._iterInputVols():
volume = vol.clone()
volDir = self._getVolDir(vol.getObjId())
volPrefix = 'vol%03d_' % (vol.getObjId())
validationMd = self._getExtraPath(volPrefix + 'validation.xmd')
moveFile(join(volDir, 'validation.xmd'), validationMd)
clusterMd = self._getExtraPath(volPrefix +
'clusteringTendency.xmd')
moveFile(join(volDir, 'clusteringTendency.xmd'), clusterMd)
mData = md.MetaData(validationMd)
weight = mData.getValue(md.MDL_WEIGHT, mData.firstObject())
volume._xmipp_weight = Float(weight)
volume.clusterMd = String(clusterMd)
volume.cleanObjId(
) # clean objects id to assign new ones inside the set
outputVols.append(volume)
outputVols.setSamplingRate(self.partSet.getSamplingRate())
self._defineOutputs(outputVolumes=outputVols)
self._defineTransformRelation(self.inputVolumes, outputVols)
def createOutputStep(self, coordinates):
outSet = self._createSetOfCoordinates3D(coordinates)
outSet.setPrecedents(coordinates.getPrecedents())
outSet.setBoxSize(coordinates.getBoxSize())
outSet.setSamplingRate(coordinates.getSamplingRate())
scoreCarbon = dict(self.scoreCarbon) if self.carbon.get() else None
scoreOutliers = dict(
self.scoreOutliers) if self.outliers.get() else None
for coord in coordinates.iterCoordinates():
newCoord = coord.clone()
newCoord.setBoxSize(outSet.getBoxSize())
newCoord.setVolume(coord.getVolume())
scoreCoordOutlier = scoreOutliers[
coord.getObjId()] if scoreOutliers else None
scoreCoordCarbon = scoreCarbon[
coord.getObjId()] if scoreCarbon else None
if self.filter.get():
if self.outliers.get() and not self.carbon.get():
if self.outliersThreshold.get() >= scoreCoordOutlier:
newCoord.outlierScore = Float(scoreCoordOutlier)
outSet.append(newCoord)
elif not self.outliers.get() and self.carbon.get():
if self.carbonThreshold.get() <= scoreCoordCarbon:
newCoord.carbonScore = Float(scoreCoordCarbon)
outSet.append(newCoord)
elif self.outliers.get() and self.carbon.get():
if self.outliersThreshold.get() >= scoreCoordOutlier and \
self.carbonThreshold.get() <= scoreCoordCarbon:
newCoord.outlierScore = Float(scoreCoordOutlier)
newCoord.carbonScore = Float(scoreCoordCarbon)
outSet.append(newCoord)
else:
print("All scoring modes are disabled. Exting")
break
else:
if self.outliers.get():
newCoord.outlierScore = Float(scoreCoordOutlier)
if self.carbon.get():
newCoord.carbonScore = Float(scoreCoordCarbon)
outSet.append(newCoord)
self._defineOutputs(outputCoordinates=outSet)
self._defineSourceRelation(coordinates, outSet)
def _updateZScore(self, img, imgRow):
zscore = imgRow.getValue(md.RLN_SELECT_PARTICLES_ZSCORE)
img._rlnSelectParticlesZscore = Float(zscore)
def _initializeZscores(self):
# Store the set for later access , ;-(
self.minZScore = Float()
self.maxZScore = Float()
self.sumZScore = Float()
class XmippProtMonoRes(ProtAnalysis3D):
"""
Given a map the protocol assigns local resolutions to each voxel of the map.
"""
_label = 'local MonoRes'
_lastUpdateVersion = VERSION_1_1
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
# --------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('halfVolumes', BooleanParam, default=False,
label="Would you like to use half volumes?",
help='The noise estimation for determining the local resolution '
'is performed via half volumes.')
form.addParam('inputVolumes', PointerParam, pointerClass='Volume',
label="Input Volume", important=True,
condition = 'not halfVolumes',
help='Select a volume for determining its local resolution.')
form.addParam('inputVolume', PointerParam, pointerClass='Volume',
label="Volume Half 1", important=True,
condition = 'halfVolumes',
help='Select a volume for determining its local resolution.')
form.addParam('inputVolume2', PointerParam, pointerClass='Volume',
label="Volume Half 2", important=True,
condition='halfVolumes',
help='Select a second volume for determining a local resolution.')
form.addParam('Mask', PointerParam, pointerClass='VolumeMask',
condition='(halfVolumes) or (not halfVolumes)',
label="Binary Mask", important=True,
help='The mask determines which points are specimen and which ones not')
group = form.addGroup('Extra parameters')
group.addParam('symmetry', StringParam, default='c1',
label="Symmetry",
help='Symmetry group. By default = c1.'
'See [[http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry][Symmetry]]'
'for a description of the symmetry groups format, If no symmetry is present, give c1.')
line = group.addLine('Resolution Range (A)',
help="If the user knows the range of resolutions or only a"
" range of frequency needs to be analysed")
group.addParam('significance', FloatParam, default=0.95, expertLevel=LEVEL_ADVANCED,
label="Significance",
help='Relution is computed using hipothesis tests, this value determines'
'the significance of that test')
group.addParam('isPremasked', BooleanParam, default=False,
label="Is the original premasked?",
help='Sometimes the original volume is masked inside a spherical mask. In this case'
'please select yes')
form.addParam('noiseonlyinhalves', BooleanParam, expertLevel=LEVEL_ADVANCED,
default=True,
label="Use noise outside the mask?",
condition = 'halfVolumes',
help='Select yes if the volume present noise outside the mask.'
' Otherwise, select No.')
group.addParam('volumeRadius', FloatParam, default=-1,
label="Spherical mask radius (px)",
condition = 'isPremasked and not halfVolumes',
help='When the original volume is originally premasked, the noise estimation ought'
'to be performed inside that premask, and out of the provieded mask asked in the previus'
'box. The radius value, determines the radius of the spherical premask. By default'
'radius = -1 use the half of the volume size as radius')
group.addParam('volumeRadiusHalf', FloatParam, default=-1,
label="Spherical mask radius (px)",
condition = 'halfVolumes and isPremasked',
help='When the origianl volume is originally premasked,'
'the noise estimation ought to be performed inside that'
'premask, and out of the provieded mask asked in the previus'
'box. The radius value, determines the radius in pixels of '
'the spherical premask. By default radius = -1 use the half'
'of the volume size as radius')
line.addParam('minRes', FloatParam, default=1, label='High')
line.addParam('maxRes', FloatParam, default=30, label='Low')
line.addParam('stepSize', FloatParam, allowsNull=True,
expertLevel=LEVEL_ADVANCED, label='Step')
group.addParam('filterInput', BooleanParam, default=False,
label="Filter input volume with local resolution?",
help='The input map is locally filtered at the local resolution map.')
# --------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
self.micsFn = self._getPath()
if self.halfVolumes:
self.vol1Fn = self.inputVolume.get().getFileName()
self.vol2Fn = self.inputVolume2.get().getFileName()
self.maskFn = self.Mask.get().getFileName()
self.inputVolumes.set(None)
else:
self.vol0Fn = self.inputVolumes.get().getFileName()
self.maskFn = self.Mask.get().getFileName()
self.inputVolume.set(None)
self.inputVolume2.set(None)
# Convert input into xmipp Metadata format
convertId = self._insertFunctionStep('convertInputStep', )
MS = self._insertFunctionStep('resolutionMonogenicSignalStep',
prerequisites=[convertId])
self._insertFunctionStep('createOutputStep', prerequisites=[MS])
self._insertFunctionStep("createHistrogram")
def convertInputStep(self):
""" Read the input volume.
"""
if (self.halfVolumes.get() is False):
extVol0 = getExt(self.vol0Fn)
if (extVol0 == '.mrc') or (extVol0 == '.map'):
self.vol0Fn = self.vol0Fn + ':mrc'
if self.halfVolumes.get() is True:
extVol1 = getExt(self.vol1Fn)
extVol2 = getExt(self.vol2Fn)
if (extVol1 == '.mrc') or (extVol1 == '.map'):
self.vol1Fn = self.vol1Fn + ':mrc'
if (extVol2 == '.mrc') or (extVol2 == '.map'):
self.vol2Fn = self.vol2Fn + ':mrc'
extMask = getExt(self.maskFn)
if (extMask == '.mrc') or (extMask == '.map'):
self.maskFn = self.maskFn + ':mrc'
def resolutionMonogenicSignalStep(self):
# Number of frequencies
if self.stepSize.hasValue():
Nfreqs = round((self.maxRes.get() - self.minRes.get())/self.stepSize.get())
else:
Nfreqs = 50
if self.halfVolumes:
if self.isPremasked:
if self.volumeRadiusHalf == -1:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim*0.5
else:
xdim = self.volumeRadiusHalf.get()
else:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim*0.5
else:
if self.isPremasked:
if self.volumeRadius == -1:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim*0.5
else:
xdim = self.volumeRadius.get()
else:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim*0.5
if self.halfVolumes.get() is False:
params = ' --vol %s' % self.vol0Fn
params += ' --mask %s' % self.maskFn
else:
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getExtraPath(FN_MEAN_VOL)
params += ' --mask %s' % self.maskFn
params += ' --mask_out %s' % self._getExtraPath(OUTPUT_MASK_FILE)
params += ' -o %s' % self._getExtraPath(OUTPUT_RESOLUTION_FILE)
if (self.halfVolumes):
params += ' --sampling_rate %f' % self.inputVolume.get().getSamplingRate()
if (self.noiseonlyinhalves is False):
params += ' --noiseonlyinhalves'
else:
params += ' --sampling_rate %f' % self.inputVolumes.get().getSamplingRate()
params += ' --number_frequencies %f' % Nfreqs
params += ' --minRes %f' % self.minRes.get()
params += ' --maxRes %f' % self.maxRes.get()
params += ' --volumeRadius %f' % xdim
params += ' --chimera_volume %s' % self._getExtraPath(OUTPUT_RESOLUTION_FILE_CHIMERA)
params += ' --sym %s' % self.symmetry.get()
params += ' --significance %f' % self.significance.get()
params += ' --md_outputdata %s' % self._getExtraPath('mask_data.xmd')
if self.filterInput.get():
params += ' --filtered_volume %s' % self._getExtraPath(FN_FILTERED_MAP)
else:
params += ' --filtered_volume %s' % ''
self.runJob('xmipp_resolution_monogenic_signal', params)
def createHistrogram(self):
params = ' -i %s' % self._getExtraPath(OUTPUT_RESOLUTION_FILE)
params += ' --mask binary_file %s' % self._getExtraPath(OUTPUT_MASK_FILE)
params += ' --steps %f' % 30
params += ' --range %f %f' % (self.min_res_init, self.max_res_init)#(self.minRes.get(), self.maxRes.get())
params += ' -o %s' % self._getExtraPath('hist.xmd')
self.runJob('xmipp_image_histogram', params)
def readMetaDataOutput(self):
mData = md.MetaData(self._getExtraPath(METADATA_MASK_FILE))
NvoxelsOriginalMask = float(mData.getValue(md.MDL_COUNT, mData.firstObject()))
NvoxelsOutputMask = float(mData.getValue(md.MDL_COUNT2, mData.firstObject()))
nvox = int(round(((NvoxelsOriginalMask-NvoxelsOutputMask)/NvoxelsOriginalMask)*100))
return nvox
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
def createOutputStep(self):
volume_path = self._getExtraPath(OUTPUT_RESOLUTION_FILE)
self.volumesSet = self._createSetOfVolumes('resolutionVol')
if (self.halfVolumes):
self.volumesSet.setSamplingRate(self.inputVolume.get().getSamplingRate())
else:
self.volumesSet.setSamplingRate(self.inputVolumes.get().getSamplingRate())
readSetOfVolumes(volume_path, self.volumesSet)
self._defineOutputs(outputVolume=self.volumesSet)
if (self.halfVolumes):
self._defineSourceRelation(self.inputVolume, self.volumesSet)
else:
self._defineSourceRelation(self.inputVolumes, self.volumesSet)
#Setting the min max for the summary
imageFile = self._getExtraPath(OUTPUT_RESOLUTION_FILE_CHIMERA)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_*100)/100)
self.max_res_init.set(round(max_*100)/100)
self._store(self.min_res_init)
self._store(self.max_res_init)
if self.filterInput.get():
print 'Saving filtered map'
volume_filtered_path = self._getExtraPath(FN_FILTERED_MAP)
self.volumesSet2 = self._createSetOfVolumes('filteredVol')
if (self.halfVolumes):
self.volumesSet2.setSamplingRate(self.inputVolume.get().getSamplingRate())
else:
self.volumesSet2.setSamplingRate(self.inputVolumes.get().getSamplingRate())
readSetOfVolumes(volume_filtered_path, self.volumesSet2)
self._defineOutputs(outputVolume_Filtered=self.volumesSet2)
if (self.halfVolumes):
self._defineSourceRelation(self.inputVolume, self.volumesSet2)
else:
self._defineSourceRelation(self.inputVolumes, self.volumesSet2)
# --------------------------- INFO functions ------------------------------
def _methods(self):
messages = []
if hasattr(self, 'outputVolume'):
messages.append(
'Information about the method/article in ' + MONORES_METHOD_URL)
return messages
def _summary(self):
summary = []
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" % (self.min_res_init,
self.max_res_init))
Nvox = self.readMetaDataOutput()
if (Nvox>10):
summary.append("The resolution of %i %% of the mask voxels could not be computed. Maybe the mask was"
"not correctly created, it is too wide or the resolution range does not cover the resolution at those voxels. "
"If it is not the problem, decrease the significance in the advaced parameters can be an alternative" % Nvox)
return summary
def _citations(self):
return ['Vilas2017']
def __init__(self, **args):
EMObject.__init__(self, **args)
self._angleY = Float()
self._angleY2 = Float()
self._angleTilt = Float()
class XmippProtMonoRes(ProtAnalysis3D):
"""
Given a map the protocol assigns local resolutions to each voxel of the map.
"""
_label = 'local MonoRes'
_lastUpdateVersion = VERSION_1_1
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
# --------------------------- DEFINE param functions ----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('halfVolumes', BooleanParam, default=False,
label="Would you like to use half volumes?",
help='The noise estimation for determining the '
'local resolution is performed via half volumes.')
form.addParam('inputVolumes', PointerParam, pointerClass='Volume',
label="Input Volume", important=True,
condition = 'not halfVolumes',
help='Select a volume for determining its '
'local resolution.')
form.addParam('inputVolume', PointerParam, pointerClass='Volume',
label="Volume Half 1", important=True,
condition = 'halfVolumes',
help='Select a volume for determining its '
'local resolution.')
form.addParam('inputVolume2', PointerParam, pointerClass='Volume',
label="Volume Half 2", important=True,
condition='halfVolumes',
help='Select a second volume for determining a '
'local resolution.')
form.addParam('Mask', PointerParam, pointerClass='VolumeMask',
condition='(halfVolumes) or (not halfVolumes)',
allowsNull=True,
label="Binary Mask",
help='The mask determines which points are specimen'
' and which are not')
group = form.addGroup('Extra parameters')
# group.addParam('symmetry', StringParam, default='c1',
# label="Symmetry",
# help='Symmetry group. By default = c1.'
# 'See [[http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry][Symmetry]]'
# 'for a description of the symmetry groups format,'
# 'If no symmetry is present, give c1.')
line = group.addLine('Resolution Range (Å)',
help='If the user knows the range of resolutions or'
'only a range of frequencies needs to be analysed.'
'If Low is empty MonoRes will try to estimate the range. '
'it should be better if a range is provided')
group.addParam('significance', FloatParam, default=0.95,
expertLevel=LEVEL_ADVANCED,
label="Significance",
help='Relution is computed using hipothesis tests, '
'this value determines the significance of that test')
group.addParam('maskthreshold', FloatParam, default=0.5,
expertLevel=LEVEL_ADVANCED,
label="Mask threshold",
help='If the provided mask is not binary. Then, MonoRes'
'will try to binarize it. Bmask values below the threshold'
'will be change to 0 and above the thresthol will be 1')
group.addParam('isPremasked', BooleanParam, default=False,
label="Is the original premasked?",
help='Sometimes the original volume is masked inside '
'a spherical mask. In this case please select yes')
form.addParam('noiseonlyinhalves', BooleanParam, expertLevel=LEVEL_ADVANCED,
default=True,
label="Use noise outside the mask?",
condition = 'halfVolumes',
help='Select yes if the volume present noise outside the mask.'
' Otherwise, select No.')
group.addParam('volumeRadius', FloatParam, default=-1,
label="Spherical mask radius (px)",
condition = 'isPremasked and not halfVolumes',
help='When the original volume is originally premasked,'
'the noise estimation ought to be performed inside'
'that premask, and out of the provieded mask asked in'
'the previus box. The radius value, determines the'
'radius of the spherical premask. '
'By default radius = -1 use the half of the '
'volume size as radius')
group.addParam('volumeRadiusHalf', FloatParam, default=-1,
label="Spherical mask radius (px)",
condition = 'halfVolumes and isPremasked',
help='When the origianl volume is originally premasked,'
'the noise estimation ought to be performed inside that'
'premask, and out of the provieded mask asked in the previus'
'box. The radius value, determines the radius in pixels of '
'the spherical premask. By default radius = -1 use the half'
'of the volume size as radius')
line.addParam('minRes', FloatParam, default=0, label='High')
line.addParam('maxRes', FloatParam, default=12, allowsNull=True, label='Low')
line.addParam('stepSize', FloatParam, allowsNull=True,
expertLevel=LEVEL_ADVANCED, label='Step')
group.addParam('filterInput', BooleanParam, default=False,
label="Filter input volume with local resolution?",
help='The input map is locally filtered at'
'the local resolution map.')
form.addParallelSection(threads = 4, mpi = 0)
# --------------------------- INSERT steps functions --------------------------------------------
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
FN_MEAN_VOL: self._getExtraPath('mean_volume.vol'),
OUTPUT_MASK_FILE: self._getExtraPath("output_Mask.vol"),
OUTPUT_RESOLUTION_FILE_CHIMERA: self._getExtraPath(CHIMERA_RESOLUTION_VOL),
FN_FILTERED_MAP: self._getExtraPath('filteredMap.vol'),
OUTPUT_RESOLUTION_FILE: self._getExtraPath('mgresolution.vol'),
METADATA_MASK_FILE: self._getExtraPath('mask_data.xmd'),
FN_METADATA_HISTOGRAM: self._getExtraPath('hist.xmd'),
BINARY_MASK: self._getExtraPath('binarized_mask.vol'),
FN_GAUSSIAN_MAP: self._getExtraPath('gaussianfilted.vol')
}
self._updateFilenamesDict(myDict)
def _insertAllSteps(self):
# Convert input into xmipp Metadata format
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep', )
self._insertFunctionStep('resolutionMonogenicSignalStep')
self._insertFunctionStep('createOutputStep')
self._insertFunctionStep("createHistrogram")
def convertInputStep(self):
""" Read the input volume.
"""
self.micsFn = self._getPath()
if self.halfVolumes:
self.vol1Fn = self.inputVolume.get().getFileName()
self.vol2Fn = self.inputVolume2.get().getFileName()
extVol1 = getExt(self.vol1Fn)
extVol2 = getExt(self.vol2Fn)
if (extVol1 == '.mrc') or (extVol1 == '.map'):
self.vol1Fn = self.vol1Fn + ':mrc'
if (extVol2 == '.mrc') or (extVol2 == '.map'):
self.vol2Fn = self.vol2Fn + ':mrc'
if not self.Mask.hasValue():
self.ifNomask(self.vol1Fn)
else:
self.maskFn = self.Mask.get().getFileName()
self.inputVolumes.set(None)
else:
self.vol0Fn = self.inputVolumes.get().getFileName()
extVol0 = getExt(self.vol0Fn)
if (extVol0 == '.mrc') or (extVol0 == '.map'):
self.vol0Fn = self.vol0Fn + ':mrc'
if not self.Mask.hasValue():
self.ifNomask(self.vol0Fn)
else:
self.maskFn = self.Mask.get().getFileName()
print self.maskFn
self.inputVolume.set(None)
self.inputVolume2.set(None)
extMask = getExt(self.maskFn)
if (extMask == '.mrc') or (extMask == '.map'):
self.maskFn = self.maskFn + ':mrc'
if self.Mask.hasValue():
params = ' -i %s' % self.maskFn
params += ' -o %s' % self._getFileName(BINARY_MASK)
params += ' --select below %f' % self.maskthreshold.get()
params += ' --substitute binarize'
self.runJob('xmipp_transform_threshold', params)
def ifNomask(self, fnVol):
if self.halfVolumes:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
params = ' -i %s' % fnVol
else:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
params = ' -i %s' % fnVol
params += ' -o %s' % self._getFileName(FN_GAUSSIAN_MAP)
setsize = 0.02*xdim
params += ' --fourier real_gaussian %f' % (setsize)
self.runJob('xmipp_transform_filter', params)
img = ImageHandler().read(self._getFileName(FN_GAUSSIAN_MAP))
imgData = img.getData()
max_val = np.amax(imgData)*0.05
params = ' -i %s' % self._getFileName(FN_GAUSSIAN_MAP)
params += ' --select below %f' % max_val
params += ' --substitute binarize'
params += ' -o %s' % self._getFileName(BINARY_MASK)
self.runJob('xmipp_transform_threshold', params)
self.maskFn = self._getFileName(BINARY_MASK)
def maskRadius(self, halfmaps, premaskedmap):
if halfmaps:
if premaskedmap:
if self.volumeRadiusHalf == -1:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim*0.5
else:
xdim = self.volumeRadiusHalf.get()
else:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim*0.5
else:
if premaskedmap:
if self.volumeRadius == -1:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim*0.5
else:
xdim = self.volumeRadius.get()
else:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim*0.5
return xdim
def resolutionMonogenicAuto(self):
freq_step = 1
xdim = self.maskRadius(self.halfVolumes, self.isPremasked)
if self.halfVolumes:
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getFileName(FN_MEAN_VOL)
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolume.get().getSamplingRate()
if (self.noiseonlyinhalves is False):
params += ' --noiseonlyinhalves'
else:
params = ' --vol %s' % self.vol0Fn
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolumes.get().getSamplingRate()
params += ' --automatic '
params += ' --step %f' % freq_step
params += ' --mask_out %s' % self._getFileName(OUTPUT_MASK_FILE)
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --volumeRadius %f' % xdim
params += ' --chimera_volume %s' % self._getFileName(
OUTPUT_RESOLUTION_FILE_CHIMERA)
params += ' --sym %s' % 'c1'#self.symmetry.get()
params += ' --significance %f' % self.significance.get()
params += ' --md_outputdata %s' % self._getFileName(METADATA_MASK_FILE)
params += ' --filtered_volume %s' % ''
params += ' --threads %i' % self.numberOfThreads.get()
self.runJob('xmipp_resolution_monogenic_signal', params)
def resolutionMonogenicSignalStep(self):
# Number of frequencies
if (not self.maxRes.hasValue()):
self.resolutionMonogenicAuto()
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE)
min_, max_ = self.getMinMax(imageFile)
else:
max_ = self.maxRes.get()
if self.stepSize.hasValue():
freq_step = self.stepSize.get()
else:
freq_step = 0.25
xdim = self.maskRadius(self.halfVolumes, self.isPremasked)
if self.halfVolumes:
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getFileName(FN_MEAN_VOL)
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolume.get().getSamplingRate()
if (self.noiseonlyinhalves is False):
params += ' --noiseonlyinhalves'
else:
params = ' --vol %s' % self.vol0Fn
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolumes.get().getSamplingRate()
params += ' --minRes %f' % self.minRes.get()
params += ' --maxRes %f' % max_
params += ' --step %f' % freq_step
params += ' --mask_out %s' % self._getFileName(OUTPUT_MASK_FILE)
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --volumeRadius %f' % xdim
params += ' --exact'
params += ' --chimera_volume %s' % self._getFileName(
OUTPUT_RESOLUTION_FILE_CHIMERA)
params += ' --sym %s' % 'c1'#self.symmetry.get()
params += ' --significance %f' % self.significance.get()
params += ' --md_outputdata %s' % self._getFileName(METADATA_MASK_FILE)
params += ' --threads %i' % self.numberOfThreads.get()
if self.filterInput.get():
params += ' --filtered_volume %s' % self._getFileName(FN_FILTERED_MAP)
else:
params += ' --filtered_volume %s' % ''
self.runJob('xmipp_resolution_monogenic_signal', params)
def createHistrogram(self):
M = float(self.max_res_init)
m = float(self.min_res_init)
range_res = round((M - m)*4.0)
params = ' -i %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --mask binary_file %s' % self._getFileName(OUTPUT_MASK_FILE)
params += ' --steps %f' % (range_res)
params += ' --range %f %f' % (self.min_res_init, self.max_res_init)
params += ' -o %s' % self._getFileName(FN_METADATA_HISTOGRAM)
self.runJob('xmipp_image_histogram', params)
def readMetaDataOutput(self):
mData = md.MetaData(self._getFileName(METADATA_MASK_FILE))
NvoxelsOriginalMask = float(mData.getValue(md.MDL_COUNT, mData.firstObject()))
NvoxelsOutputMask = float(mData.getValue(md.MDL_COUNT2, mData.firstObject()))
nvox = int(round(
((NvoxelsOriginalMask-NvoxelsOutputMask)/NvoxelsOriginalMask)*100))
return nvox
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
def createOutputStep(self):
volume=Volume()
volume.setFileName(self._getFileName(OUTPUT_RESOLUTION_FILE))
if (self.halfVolumes):
volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolume, volume)
else:
volume.setSamplingRate(self.inputVolumes.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolumes, volume)
#Setting the min max for the summary
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE_CHIMERA)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_*100)/100)
self.max_res_init.set(round(max_*100)/100)
self._store(self.min_res_init)
self._store(self.max_res_init)
if self.filterInput.get():
print 'Saving filtered map'
volume.setFileName(self._getFileName(FN_FILTERED_MAP))
if (self.halfVolumes):
volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
self._defineOutputs(outputVolume_Filtered=volume)
self._defineSourceRelation(self.inputVolume, volume)
else:
volume.setSamplingRate(self.inputVolumes.get().getSamplingRate())
self._defineOutputs(outputVolume_Filtered=volume)
self._defineSourceRelation(self.inputVolumes, volume)
# --------------------------- INFO functions ------------------------------
def _methods(self):
messages = []
if hasattr(self, 'resolution_Volume'):
messages.append(
'Information about the method/article in ' + MONORES_METHOD_URL)
return messages
def _summary(self):
summary = []
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" % (self.min_res_init,
self.max_res_init))
return summary
def _citations(self):
return ['Vilas2018']
def createAnalyzeFilesStep(self):
""" This method will create two sqlite files that
will be used in by the viewer in the 'Analyze result' action.
ctfSet: a table with average ctf values between all methods.
ctfSetPair: a table where each row is a pair between two methods
"""
ctfSetFn, ctfSetPairFn = self._getAnalyzeFiles()
cleanPath(ctfSetFn, ctfSetPairFn)
ctfSet = Set(filename=ctfSetFn)
ctfSetPair = em.SetOfCTF(filename=ctfSetPairFn)
#import pdb
#pdb.set_trace()
minimumResolution = {}
maximumResolution = {}
averageResolution = {}
averageDefocusU = {}
averageDefocusV = {}
averageDefocusAngle = {}
for ctf in self.setOfCTF:
micFileName = self._getMicName(ctf)
minimumResolution[micFileName] = 0.
maximumResolution[micFileName] = 0.
averageResolution[micFileName] = 0.
averageDefocusU[micFileName] = 0.
averageDefocusV[micFileName] = 0.
averageDefocusAngle[micFileName] = 0.
for method1, method2, ctfId in self._freqResol:
ctf = em.CTFModel()
ctf1 = self.inputCTFs[method1].get()[ctfId]
ctf2 = self.inputCTFs[method2].get()[ctfId]
ctf.setDefocusU((ctf1.getDefocusU() + ctf2.getDefocusU()) / 2.)
ctf.setDefocusV((ctf1.getDefocusV() + ctf2.getDefocusV()) / 2.)
ctf.setDefocusAngle(
(ctf1.getDefocusAngle() + ctf2.getDefocusAngle()) / 2.)
ctf.setMicrograph(ctf1.getMicrograph())
#Clean objId since we can have ctf from the same micrograph
# and by default it is set to micrograph id
ctf.cleanObjId()
resolution = self._freqResol[(method1, method2, ctfId)]
ctf.resolution = Float(resolution)
ctf.method1 = String(self.methodNames[method1])
ctf.method2 = String(self.methodNames[method2])
# save the values of defocus for each micrograph in a list
ctfSetPair.append(ctf)
micFileName = self._getMicName(ctf1)
if (not micFileName in minimumResolution
or not micFileName in maximumResolution):
pass
if resolution < minimumResolution[micFileName]:
minimumResolution[micFileName] = resolution
if resolution > maximumResolution[micFileName]:
maximumResolution[micFileName] = resolution
averageResolution[micFileName] += resolution
averageDefocusU[micFileName] += ctf.getDefocusU()
averageDefocusV[micFileName] += ctf.getDefocusV()
averageDefocusAngle[micFileName] += ctf.getDefocusAngle()
size = float(len(self.setOfCTF))
for ctf in self.setOfCTF:
ctfAvg = Object()
micFileName = self._getMicName(ctf)
ctfAvg._micObj = ctf.getMicrograph()
ctfAvg.averageDefocusU = Float(averageDefocusU[micFileName] / size)
ctfAvg.averageDefocusV = Float(averageDefocusV[micFileName] / size)
ctfAvg.averageDefocusAngle = Float(
averageDefocusAngle[micFileName] / size)
ctfAvg.averageResolution = Float(averageResolution[micFileName] /
size)
ctfSet.append(ctfAvg)
ctfSetPair.write()
ctfSet.write()
def __init__(self, imag=0., real=0., **args):
Object.__init__(self, **args)
self.imag = Float(imag)
self.real = Float(real)
class XmippProtDeepRes(ProtAnalysis3D):
"""
Given a map the protocol assigns local resolutions to each voxel of the map.
"""
_label = 'local deepRes'
_lastUpdateVersion = VERSION_2_0
#RESOLUTION RANGE
LOW_RESOL = 0
HIGH_RESOL = 1
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
self.median_res_init = Float()
# --------------------------- DEFINE param functions ----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addHidden(GPU_LIST,
StringParam,
default='0',
label="Choose GPU ID",
help="GPU may have several cores. Set it to zero"
" if you do not know what we are talking about."
" First core index is 0, second 1 and so on.")
form.addParam('inputVolume',
PointerParam,
pointerClass='Volume',
label="Input Volume",
help='Select a volume for determining its '
'local resolution.')
form.addParam('Mask',
PointerParam,
pointerClass='VolumeMask',
allowsNull=True,
label="Mask",
help='The mask determines which points are specimen'
' and which are not')
form.addParam('range',
EnumParam,
choices=[u'2.5Å - 13.0Å', u'1.5Å - 6.0Å'],
label="Expected resolutions range",
default=self.LOW_RESOL,
display=EnumParam.DISPLAY_HLIST,
help='The program uses a trained network to determine'
' resolutions between 2.5Å-13.0Å or'
' resolutions between 1.5Å-6.0Å')
# --------------------------- INSERT steps functions --------------------------------------------
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
MASK_DILATE:
self._getTmpPath('Mask_dilate.vol'),
OPERATE_VOL:
self._getTmpPath('operateVolume.vol'),
RESIZE_MASK:
self._getExtraPath('binaryMask.vol'),
RESIZE_VOL:
self._getExtraPath('originalVolume.vol'),
OUTPUT_RESOLUTION_FILE_CHIMERA:
self._getExtraPath('chimera_resolution.vol'),
# OUTPUT_RESOLUTION_FILE_CHIMERA: self._getExtraPath(CHIMERA_RESOLUTION_VOL),
OUTPUT_RESOLUTION_FILE:
self._getExtraPath('deepRes_resolution.vol'),
FN_METADATA_HISTOGRAM:
self._getExtraPath('hist.xmd')
}
self._updateFilenamesDict(myDict)
def _insertAllSteps(self):
# Convert input into xmipp Metadata format
updateEnviron(self.gpuList.get())
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('transformStep')
self._insertFunctionStep('resizeStep')
self._insertFunctionStep('resolutionStep')
self._insertFunctionStep('createOutputStep')
self._insertFunctionStep("createHistrogram")
def convertInputStep(self):
""" Read the input volume.
"""
self.micsFn = self._getPath()
self.volFn = self.inputVolume.get().getFileName()
self.maskFn = self.Mask.get().getFileName()
extVol = getExt(self.volFn)
if (extVol == '.mrc') or (extVol == '.map'):
self.volFn = self.volFn + ':mrc'
extMask = getExt(self.maskFn)
if (extMask == '.mrc') or (extMask == '.map'):
self.maskFn = self.maskFn + ':mrc'
def transformStep(self):
params = ' -i %s' % self.maskFn
params += ' -o %s' % self._getFileName(MASK_DILATE)
params += ' --binaryOperation dilation --size 1 '
self.runJob('xmipp_transform_morphology', params)
params2 = ' -i %s' % self.volFn
params2 += ' --mult %s' % self._getFileName(MASK_DILATE)
params2 += ' -o %s' % self._getFileName(OPERATE_VOL)
self.runJob('xmipp_image_operate', params2)
def resizeStep(self):
if self.range == self.LOW_RESOL:
sampling_new = 1.0
else:
sampling_new = 0.5
samplingFactor = float(
self.inputVolume.get().getSamplingRate()) / sampling_new
fourierValue = float(
self.inputVolume.get().getSamplingRate()) / (2 * sampling_new)
if self.inputVolume.get().getSamplingRate() > sampling_new:
#mask with sampling=1.0
paramsResizeMask = ' -i %s' % self.maskFn
paramsResizeMask += ' -o %s' % self._getFileName(RESIZE_MASK)
paramsResizeMask += ' --factor %s' % samplingFactor
self.runJob('xmipp_image_resize', paramsResizeMask)
#Original volume with sampling=1.0
paramsResizeVol = ' -i %s' % self._getFileName(OPERATE_VOL)
paramsResizeVol += ' -o %s' % self._getFileName(RESIZE_VOL)
paramsResizeVol += ' --factor %s' % samplingFactor
self.runJob('xmipp_image_resize', paramsResizeVol)
else:
#mask with sampling=1.0
paramsFilterMask = ' -i %s' % self.maskFn
paramsFilterMask += ' -o %s' % self._getFileName(RESIZE_MASK)
paramsFilterMask += ' --fourier low_pass %s' % fourierValue
paramsResizeMask = ' -i %s' % self._getFileName(RESIZE_MASK)
paramsResizeMask += ' -o %s' % self._getFileName(RESIZE_MASK)
paramsResizeMask += ' --factor %s' % samplingFactor
self.runJob('xmipp_transform_filter', paramsFilterMask)
self.runJob('xmipp_image_resize', paramsResizeMask)
#Original volume with sampling=1.0
paramsFilterVol = ' -i %s' % self._getFileName(OPERATE_VOL)
paramsFilterVol += ' -o %s' % self._getFileName(RESIZE_VOL)
paramsFilterVol += ' --fourier low_pass %s' % fourierValue
paramsResizeVol = ' -i %s' % self._getFileName(RESIZE_VOL)
paramsResizeVol += ' -o %s' % self._getFileName(RESIZE_VOL)
paramsResizeVol += ' --factor %s' % samplingFactor
self.runJob('xmipp_transform_filter', paramsFilterVol)
self.runJob('xmipp_image_resize', paramsResizeVol)
params = ' -i %s' % self._getFileName(RESIZE_MASK)
params += ' -o %s' % self._getFileName(RESIZE_MASK)
params += ' --select below %f' % 0.15
params += ' --substitute binarize'
self.runJob('xmipp_transform_threshold', params)
def resolutionStep(self):
if self.range == self.LOW_RESOL:
# sampling_new = 1.0
MODEL_DEEP_1 = xmipp3.Plugin.getModel("deepRes", "model_w13.h5")
params = ' -dl %s' % MODEL_DEEP_1
else:
# sampling_new = 0.5
MODEL_DEEP_2 = xmipp3.Plugin.getModel("deepRes", "model_w7.h5")
params = ' -dl %s' % MODEL_DEEP_2
params += ' -i %s' % self._getFileName(RESIZE_VOL)
params += ' -m %s' % self._getFileName(RESIZE_MASK)
params += ' -s %f' % self.inputVolume.get().getSamplingRate()
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
self.runJob("xmipp_deepRes_resolution", params)
def createHistrogram(self):
M = float(self.max_res_init)
m = float(self.min_res_init)
# M = 12.5
# m = 2.5
range_res = round((M - m) * 4.0)
params = ' -i %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --mask binary_file %s' % self._getFileName(RESIZE_MASK)
params += ' --steps %f' % (range_res)
params += ' --range %f %f' % (self.min_res_init, self.max_res_init)
params += ' -o %s' % self._getFileName(FN_METADATA_HISTOGRAM)
self.runJob('xmipp_image_histogram', params)
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
imgData = imgData[imgData != 0]
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
median_res = round(np.median(imgData) * 100) / 100
return min_res, max_res, median_res
def createChimeraOutput(self, vol, value):
Vx = xmipp3.Image(vol)
V = Vx.getData()
Zdim, Ydim, Xdim = V.shape
# Vout = V
for z in range(0, Zdim):
for y in range(0, Ydim):
for x in range(0, Xdim):
if V[z, y, x] == 0:
V[z, y, x] = value
Vx.setData(V)
# Vx.write(Vout)
return Vx
def createOutputStep(self):
if self.range == self.LOW_RESOL:
sampling_new = 1.0
else:
sampling_new = 0.5
volume = Volume()
volume.setFileName(self._getFileName(OUTPUT_RESOLUTION_FILE))
volume.setSamplingRate(sampling_new)
self._defineOutputs(resolution_Volume=volume)
self._defineTransformRelation(self.inputVolume, volume)
#Setting the min max and median for the summary
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE)
min_, max_, median_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_ * 100) / 100)
self.max_res_init.set(round(max_ * 100) / 100)
self.median_res_init.set(round(median_ * 100) / 100)
self._store(self.min_res_init)
self._store(self.max_res_init)
self._store(self.median_res_init)
#create Resolution Map to visialize in Chimera
#vol_chimera=Volume()
vol_chimera = self.createChimeraOutput(
self._getFileName(OUTPUT_RESOLUTION_FILE), self.median_res_init)
# self.createChimeraOutput(self._getFileName(OUTPUT_RESOLUTION_FILE),
# self.median_res_init,
# self._getFileName(OUTPUT_RESOLUTION_FILE_CHIMERA))
vol_chimera.write(self._getFileName(OUTPUT_RESOLUTION_FILE_CHIMERA))
# vol_chimera.setFileName(self._getFileName(OUTPUT_RESOLUTION_FILE_CHIMERA))
# self.vol_chimera.setSamplingRate(sampling_new)
# self._defineOutputs(resolution_Volume=vol_chimera)
# self._defineTransformRelation(self.inputVolume, vol_chimera)
# --------------------------- INFO functions ------------------------------
def _methods(self):
messages = []
if hasattr(self, 'resolution_Volume'):
messages.append('Information about the method/article in ' +
DEEPRES_METHOD_URL)
return messages
def _summary(self):
summary = []
summary.append("Median resolution %.2f Å." % (self.median_res_init))
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" %
(self.min_res_init, self.max_res_init))
return summary
def _validate(self):
""" Check if the installation of this protocol is correct.
Can't rely on package function since this is a "multi package" package
Returning an empty list means that the installation is correct
and there are not errors. If some errors are found, a list with
the error messages will be returned.
"""
error = validateDLtoolkit(model="deepRes")
return error
def _citations(self):
return ['Ramirez-Aportela-2019']
def __init__(self, **args):
EMObject.__init__(self, **args)
self._angleY = Float()
self._angleY2 = Float()
self._angleTilt = Float()
def testMergeAlternateColumn(self):
"""Test that the union operation works as expected.
Even if the order of the columns do not match.
That is, M1(a,b,c) U M2(a,c,b)"""
# Create two sets of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles1.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles2.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
# import them
protImport1 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set1',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set2',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
# create merge protocol
p_union = self.newProtocol(emprot.ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
# assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
counter += 1
def _updateItemCtfBeamTilt(self, particle, row):
particle.setCTF(convert.rowToCtfModel(row))
# check if beamtilt is available and save it
if row.hasLabel('rlnBeamTiltX'):
particle._rlnBeamTiltX = Float(row.getValue('rlnBeamTiltX', 0))
particle._rlnBeamTiltY = Float(row.getValue('rlnBeamTiltY', 0))
class BsoftProtBlocres(ProtAnalysis3D):
""" Wrapper around blocres program.
"""
_label = 'blocres'
_version = VERSION_1_1
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
self.halfVolumes = True
#--------------------------- DEFINE param functions -----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputVolume',
params.PointerParam,
pointerClass='Volume',
label="Half 1",
help="""Select first half volume to compute the
local resolution.
""")
form.addParam('inputVolume2',
params.PointerParam,
pointerClass='Volume',
label="Half 2",
help="""Select first half volume to compute the
local resolution.
""")
form.addParam('mask',
params.PointerParam,
allowsNull=True,
pointerClass='Volume',
label='Mask',
help="""Mask file to use for limiting the analysis to a
defined region and level value to use
(optional, default: all but zero).""")
form.addParam('method',
params.BooleanParam,
default=True,
label='Use Box',
help="""The local (box) and shell (shell) resolution
calculations are mutually exclusive.""")
form.addParam('box',
params.IntParam,
default=20,
condition='(method)',
label='Box',
help="""Kernel size for determining
local resolution (pixels/voxels)""")
form.addParam('shell',
params.IntParam,
default=20,
condition='(not method)',
label='Shell',
help="""Shell width for determining
radial resolution (pixels/voxels).""")
line = form.addLine('Resolution Criterion')
line.addParam('resolutionCriterion',
params.EnumParam,
choices=['FSC', 'DPR', 'SSNR', 'RAB'],
default=0,
help="""Resolution criterion:
FSC = Fourier Shell Correlation (default),
DPR = Differential Phase Residual,
SSNR = Spectral Signal-to-Noise Ratio,
RAB = R(A+B) figure of merit.""")
line.addParam('cutoff',
params.FloatParam,
default=0.5,
label='Cutoff',
help="""Resolution cutoff for the criterion chosen
(default: FSC: 0.5, DPR: 45,
SSNR: 1, RAB: 0.5).""")
form.addSection(label='Parameters')
form.addParam('step',
params.IntParam,
default=1,
label='Step',
help="""Interval between voxel samples or shells for
resolution analysis (pixels, default: 1)""")
form.addParam(
'maxresolution',
params.FloatParam,
default=2,
label='Maximum Resolution',
help="""Maximum frequency available in the data (angstrom)""")
form.addParam('fill',
params.IntParam,
allowsNull=True,
label='Fill',
help="""Value to fill the background
(non-masked regions; default 0).
""")
form.addParam('pad',
params.EnumParam,
choices=['None', 'Box', 'Shell'],
default=1,
label='Padding Factor',
help="""Resolution box padding factor
(0 = none, default: 1 (box) and 0 (shell)).
""")
form.addParam('symmetry',
params.StringParam,
allowsNull=True,
default='',
label='Symmetry',
help="""Point group symmetry.""")
form.addParam('smooth',
params.BooleanParam,
default=True,
label='Smooth',
help="""Smooth the shell edge.""")
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
FN_HALF1: self._getTmpPath("half1.map"),
FN_HALF2: self._getTmpPath("half2.map"),
FN_MASKVOL: self._getTmpPath("mask.map"),
FN_RESOLMAP: self._getExtraPath("resolutionMap.map")
}
self._updateFilenamesDict(myDict)
#--------------------------- INSERT steps functions -----------------------
def _insertAllSteps(self):
# Insert processing steps
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('resolutionStep')
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions -------------------------------
def convertInputStep(self):
# blocres works with .map
vol1Fn = self.inputVolume.get().getFileName()
vol2Fn = self.inputVolume2.get().getFileName()
if self.mask.get().getFileName() != '':
maskFn = self.mask.get().getFileName()
self.fnvol1 = self._getFileName("half1")
self.fnvol2 = self._getFileName("half2")
ImageHandler().convert(vol1Fn, self.fnvol1)
ImageHandler().convert(vol2Fn, self.fnvol2)
if self.mask.get().getFileName() != '':
self.fnmask = self._getFileName("maskvol")
ImageHandler().convert(maskFn, self.fnmask)
else:
self.fnmask = self.maks.get().getFileName()
def resolutionStep(self):
""" blocres parameters. """
sampling = self.inputVolume.get().getSamplingRate()
#Actions
params = ' -v 1' # No Verbose
params += ' -criterion %s' % self.getEnumText("resolutionCriterion")
if (self.method):
params += ' -box %i' % self.box.get()
else:
params += ' -shell %i' % self.shell.get()
#Parameters
params += ' -sampling %f,%f,%f' % (sampling, sampling, sampling)
params += ' -step %f' % self.step.get()
params += ' -maxresolution %f' % self.maxresolution.get()
params += ' -cutoff %f' % self.cutoff.get()
if self.fill.get() != '':
params += ' -fill %i' % self.fill.get()
#Parameters for local resolution
params += ' -pad %f' % self.pad.get()
if self.symmetry.get() != '':
params += ' -symmetry %s' % self.symmetry.get()
if self.smooth.get():
params += ' -smooth'
if self.mask.get().getFileName() != '':
params += ' -Mask %s' % self.fnmask
# Input
# input halves and output map
params += ' %s %s %s' % (self.fnvol1, self.fnvol2,
self._getFileName(FN_RESOLMAP))
self.runJob('blocres', params)
def createOutputStep(self):
volume = Volume()
volume.setFileName(self._getFileName(FN_RESOLMAP))
volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolume, volume)
imageFile = self._getFileName(FN_RESOLMAP)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_ * 100) / 100)
self.max_res_init.set(round(max_ * 100) / 100)
self._store(self.min_res_init)
self._store(self.max_res_init)
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
#--------------------------- INFO functions -----------------------------------
def _validate(self):
errors = []
return errors
def _citations(self):
cites = ['Cardone2013']
return cites
def _summary(self):
summary = []
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" %
(self.min_res_init, self.max_res_init))
def _methods(self):
methods = []
return methods
class XmippProtScreenParticles(ProtProcessParticles):
""" Attach different merit values to every particle in order to
prune the set.
zScore evaluates the similarity of a particles with an average
(lower zScore -> high similarity).
SSNR evaluates the signal/noise ration in the Fourier space.
Variance evaluates the varaince on the micrographs context where
the particle was picked.
"""
_label = 'screen particles'
# Automatic Particle rejection enum
ZSCORE_CHOICES = ['None', 'MaxZscore', 'Percentage']
SSNR_CHOICES = ['None', 'Percentage']
VAR_CHOICES = ['None', 'Variance', 'Var. and Gini']
REJ_NONE = 0
REJ_MAXZSCORE = 1
REJ_PERCENTAGE = 2
REJ_PERCENTAGE_SSNR = 1
REJ_VARIANCE = 1
REJ_VARGINI = 2
# --------------------------- DEFINE param functions ---------------------
def _defineProcessParams(self, form):
# --- zScore rejection ---
form.addParam('autoParRejection', EnumParam,
choices=self.ZSCORE_CHOICES,
label="Automatic rejection by Zscore",
default=self.REJ_NONE,
display=EnumParam.DISPLAY_COMBO,
help='zScore evaluates the similarity of a particles '
'with an average. The rejection can be:\n'
' None (no rejection)\n'
' MaxZscore (reject a particle if its zScore '
'is larger than this value).\n '
' Percentage (reject a given percentage for '
'this criteria).')
form.addParam('maxZscore', FloatParam, default=3, validators=[Positive],
condition='autoParRejection==1', label='zScore threshold',
help='Maximum Zscore.')
form.addParam('percentage', IntParam, default=5, label='Percentage (%)',
condition='autoParRejection==2',
help='The worse percentage of particles according to '
'metadata labels: ZScoreShape1, ZScoreShape2, '
'ZScoreSNR1, ZScoreSNR2, ZScoreHistogram are '
'automatically disabled.',
validators=[Range(0, 100, error="Percentage must be "
"between 0 and 100.")])
# --- SSNR rejection ---
form.addParam('autoParRejectionSSNR', EnumParam,
choices=self.SSNR_CHOICES,
label="Automatic rejection by SSNR",
default=self.REJ_NONE, display=EnumParam.DISPLAY_COMBO,
help='SSNR evaluates the signal/noise ration in the '
'Fourier space. The rejection can be:\n'
' None (no rejection)\n'
' Percentage (reject a given percentage of the '
'lowest SSNRs).')
form.addParam('percentageSSNR', IntParam, default=5,
condition='autoParRejectionSSNR==1',
label='Percentage (%)',
help='The worse percentage of particles according to '
'SSNR are automatically disabled.',
validators=[Range(0, 100, error="Percentage must be "
"between 0 and 100.")])
# --- Variance rejection ---
form.addParam('autoParRejectionVar', EnumParam, default=self.REJ_NONE,
choices=self.VAR_CHOICES,
label='Automatic rejection by Variance',
help='Variance evaluates the varaince on the micrographs '
'context where the particle was picked. '
'The rejection can be:\n'
' None (no rejection)\n'
' Variance (taking into account only the variance)\n'
' Var. and Gini (taking into account also the Gini '
'coeff.)')
# --- Add features ---
form.addParam('addFeatures', BooleanParam, default=False,
label='Add features', expertLevel=LEVEL_ADVANCED,
help='Add features used for the ranking to each one '
'of the input particles')
form.addParallelSection(threads=0, mpi=0)
def _getDefaultParallel(self):
"""This protocol doesn't have mpi version"""
return (0, 0)
#--------------------------- INSERT steps functions ----------------------
def _insertAllSteps(self):
self._initializeZscores()
self.outputSize = 0
self.inputSize = 0
self.check = None
self.fnInputMd = self._getExtraPath("input.xmd")
self.fnInputOldMd = self._getExtraPath("inputOld.xmd")
self.fnOutputMd = self._getExtraPath("output.xmd")
self.inputSize, self.streamClosed = self._loadInput()
partsSteps = self._insertNewPartsSteps()
self._insertFunctionStep('createOutputStep',
prerequisites=partsSteps, wait=True)
def _getFirstJoinStep(self):
for s in self._steps:
if s.funcName == self._getFirstJoinStepName():
return s
return None
def _getFirstJoinStepName(self):
# This function will be used for streaming, to check which is
# the first function that need to wait for all micrographs
# to have completed, this can be overriden in subclasses
# (e.g., in Xmipp 'sortPSDStep')
return 'createOutputStep'
def createOutputStep(self):
pass
def _insertNewPartsSteps(self):
deps = []
stepId = self._insertFunctionStep('sortImagesStep', prerequisites=[])
deps.append(stepId)
return deps
def _stepsCheck(self):
# Input particles set can be loaded or None when checked for new inputs
# If None, we load it
self._checkNewInput()
self._checkNewOutput()
def _checkNewInput(self):
# Check if there are new particles to process from the input set
partsFile = self.inputParticles.get().getFileName()
mTime = datetime.fromtimestamp(os.path.getmtime(partsFile))
# If the input movies.sqlite have not changed since our last check,
# it does not make sense to check for new input data
if self.lastCheck > mTime:
return None
self.inputSize, self.streamClosed = self._loadInput()
if not isEmpty(self.fnInputMd):
fDeps = self._insertNewPartsSteps()
outputStep = self._getFirstJoinStep()
if outputStep is not None:
outputStep.addPrerequisites(*fDeps)
self.updateSteps()
def _loadInput(self):
self.lastCheck = datetime.now()
partsFile = self.inputParticles.get().getFileName()
inPartsSet = SetOfParticles(filename=partsFile)
inPartsSet.loadAllProperties()
check = None
for p in inPartsSet.iterItems(orderBy='creation', direction='DESC'):
check = p.getObjCreation()
break
if self.check is None:
writeSetOfParticles(inPartsSet, self.fnInputMd,
alignType=ALIGN_NONE, orderBy='creation')
else:
writeSetOfParticles(inPartsSet, self.fnInputMd,
alignType=ALIGN_NONE, orderBy='creation',
where='creation>"' + str(self.check) + '"')
writeSetOfParticles(inPartsSet, self.fnInputOldMd,
alignType=ALIGN_NONE, orderBy='creation',
where='creation<"' + str(self.check) + '"')
self.check = check
streamClosed = inPartsSet.isStreamClosed()
inputSize = inPartsSet.getSize()
inPartsSet.close()
return inputSize, streamClosed
def _checkNewOutput(self):
if getattr(self, 'finished', False):
return
self.finished = self.streamClosed and \
self.outputSize == self.inputSize
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
newData = os.path.exists(self.fnOutputMd)
lastToClose = self.finished and hasattr(self, 'outputParticles')
if newData or lastToClose:
outSet = self._loadOutputSet(SetOfParticles, 'outputParticles.sqlite')
if newData:
partsSet = self._createSetOfParticles()
readSetOfParticles(self.fnOutputMd, partsSet)
outSet.copyItems(partsSet)
for item in partsSet:
self._calculateSummaryValues(item)
self._store()
writeSetOfParticles(outSet.iterItems(orderBy='_xmipp_zScore'),
self._getPath("images.xmd"),
alignType=ALIGN_NONE)
cleanPath(self.fnOutputMd)
self._updateOutputSet('outputParticles', outSet, streamMode)
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(cons.STATUS_NEW)
def _loadOutputSet(self, SetClass, baseName):
setFile = self._getPath(baseName)
if os.path.exists(setFile):
outputSet = SetClass(filename=setFile)
outputSet.loadAllProperties()
outputSet.enableAppend()
else:
outputSet = SetClass(filename=setFile)
outputSet.setStreamState(outputSet.STREAM_OPEN)
self._store(outputSet)
self._defineTransformRelation(self.inputParticles, outputSet)
outputSet.copyInfo(self.inputParticles.get())
return outputSet
#--------------------------- STEP functions -----------------------------
def sortImagesStep(self):
args = "-i Particles@%s -o %s --addToInput " % (self.fnInputMd,
self.fnOutputMd)
if os.path.exists(self.fnInputOldMd):
args += "-t Particles@%s " % self.fnInputOldMd
if self.autoParRejection == self.REJ_MAXZSCORE:
args += "--zcut " + str(self.maxZscore.get())
elif self.autoParRejection == self.REJ_PERCENTAGE:
args += "--percent " + str(self.percentage.get())
if self.addFeatures:
args += "--addFeatures "
self.runJob("xmipp_image_sort_by_statistics", args)
args = "-i Particles@%s -o %s" % (self.fnInputMd, self.fnOutputMd)
if self.autoParRejectionSSNR == self.REJ_PERCENTAGE_SSNR:
args += " --ssnrpercent " + str(self.percentageSSNR.get())
self.runJob("xmipp_image_ssnr", args)
if self.autoParRejectionVar != self.REJ_NONE:
print('Rejecting by variance:')
if self.outputSize == 0:
varList = []
giniList = []
print(' - Reading metadata')
mdata = emlib.MetaData(self.fnInputMd)
for objId in mdata:
varList.append(mdata.getValue(emlib.MDL_SCORE_BY_VAR, objId))
giniList.append(mdata.getValue(emlib.MDL_SCORE_BY_GINI, objId))
if self.autoParRejectionVar == self.REJ_VARIANCE:
valuesList = varList
self.mdLabels = [emlib.MDL_SCORE_BY_VAR]
else: # not working pretty well
valuesList = [var*(1-gini) for var, gini in zip(varList, giniList)]
self.mdLabels = [emlib.MDL_SCORE_BY_VAR, emlib.MDL_SCORE_BY_GINI]
self.varThreshold.set(histThresholding(valuesList))
print(' - Variance threshold: %f' % self.varThreshold)
rejectByVariance(self.fnInputMd, self.fnOutputMd, self.varThreshold,
self.autoParRejectionVar)
# update the processed particles
self.outputSize += getSize(self.fnInputMd)
def _initializeZscores(self):
# Store the set for later access , ;-(
self.minZScore = Float()
self.maxZScore = Float()
self.sumZScore = Float()
self.varThreshold = Float()
self._store()
def _calculateSummaryValues(self, particle):
zScore = particle._xmipp_zScore.get()
self.minZScore.set(min(zScore, self.minZScore.get(1000)))
self.maxZScore.set(max(zScore, self.maxZScore.get(0)))
self.sumZScore.set(self.sumZScore.get(0) + zScore)
# -------------------------- INFO functions ------------------------------
def _summary(self):
summary = []
sumRejMet = {} # A dict with the form choices
if self.autoParRejection is not None:
metStr = self.ZSCORE_CHOICES[self.autoParRejection.get()]
if self.autoParRejection.get() == self.REJ_MAXZSCORE:
metStr += " = %.2f" % self.maxZscore.get()
elif self.autoParRejection.get() == self.REJ_PERCENTAGE:
metStr += " = %.2f" % self.percentage.get()
sumRejMet['Zscore'] = ("Zscore rejection method: " + metStr)
if self.autoParRejectionSSNR is not None:
metStr = self.SSNR_CHOICES[self.autoParRejectionSSNR.get()]
if self.autoParRejectionSSNR.get() == self.REJ_PERCENTAGE_SSNR:
metStr += " = %.2f" % self.percentageSSNR.get()
sumRejMet['SSNR'] = ("SSNR rejection method: " + metStr)
if self.autoParRejectionVar is not None:
sumRejMet['Var'] = ("Variance rejection method: " +
self.VAR_CHOICES[self.autoParRejectionVar.get()])
# If no output yet, just the form choices are shown plus a no-ready text
if not hasattr(self, 'outputParticles'):
summary += sumRejMet.values()
summary.append("Output particles not ready yet.")
else:
summary.append(sumRejMet['Zscore'])
if hasattr(self, 'sumZScore'):
summary.append(" - The minimum ZScore is %.2f" % self.minZScore)
summary.append(" - The maximum ZScore is %.2f" % self.maxZScore)
meanZScore = self.sumZScore.get() * 1.0 / len(self.outputParticles)
summary.append(" - The mean ZScore is %.2f" % meanZScore)
else:
summary.append(
"Summary values not calculated during processing.")
summary.append(sumRejMet['SSNR'])
summary.append(sumRejMet['Var'])
if self.autoParRejectionVar != self.REJ_NONE:
if hasattr(self, 'varThreshold'):
summary.append(" - Variance threshold: %.2f" % self.varThreshold)
else:
summary.append(" - Variance threshold not calculed yet.")
return summary
def _validate(self):
validateMsgs = []
if self.autoParRejectionVar != self.REJ_NONE:
part = self.inputParticles.get().getFirstItem()
if not part.hasAttribute('_xmipp_scoreByVariance'):
validateMsgs.append('The auto-rejection by Variance can not be '
'done because the particles have not the '
'scoreByVariance attribute. Use Xmipp to '
'extract the particles.')
return validateMsgs
def _citations(self):
return ['Vargas2013b']
def _methods(self):
methods = []
if hasattr(self, 'outputParticles'):
outParticles = (len(self.outputParticles) if self.outputParticles
is not None else None)
particlesRejected = (len(self.inputParticles.get())-outParticles
if outParticles is not None else None)
particlesRejectedText = (' ('+str(particlesRejected)+')' if
particlesRejected is not None else '')
rejectionText = ['', # REJ_NONE
' and removing those not reaching %s%s'
% (str(self.maxZscore.get()),
particlesRejectedText), # REJ_MAXZSCORE
' and removing worst %s percent %s'
% (str(self.percentage.get()),
particlesRejectedText) # REJ_PERCENTAGE
]
methods.append('Input dataset %s of %s particles was sorted by'
' its ZScore using xmipp_image_sort_by_statistics'
' program%s. '
% (self.getObjectTag('inputParticles'),
len(self.inputParticles.get()),
rejectionText[self.autoParRejection.get()]))
methods.append('Output set is %s.'
% self.getObjectTag('outputParticles'))
return methods
def _checkNewOutput(self):
""" Check for already selected CTF and update the output set. """
# Load previously done items (from text file)
doneListDiscarded = self._readDoneListDiscarded()
doneListAccepted = self._readDoneListAccepted()
# Check for newly done items
ctfListIdAccepted = self._readtCtfId(True)
ctfListIdDiscarded = self._readtCtfId(False)
newDoneAccepted = [ctfId for ctfId in ctfListIdAccepted
if ctfId not in doneListAccepted]
newDoneDiscarded = [ctfId for ctfId in ctfListIdDiscarded
if ctfId not in doneListDiscarded]
firstTimeAccepted = len(doneListAccepted) == 0
firstTimeDiscarded = len(doneListDiscarded) == 0
allDone = len(doneListAccepted) + len(doneListDiscarded) +\
len(newDoneAccepted) + len(newDoneDiscarded)
# We have finished when there is not more input ctf (stream closed)
# and the number of processed ctf is equal to the number of inputs
self.finished = (self.isStreamClosed and allDone == len(self.allCtf1))
streamMode = Set.STREAM_CLOSED if self.finished else Set.STREAM_OPEN
# reading the outputs
if (len(doneListAccepted) > 0 or len(newDoneAccepted) > 0):
ctfSet = self._loadOutputSet(em.SetOfCTF, 'ctfs.sqlite')
micSet = self._loadOutputSet(em.SetOfMicrographs,
'micrographs.sqlite')
# AJ new subsets with discarded ctfs
if (len(doneListDiscarded) > 0 or len(newDoneDiscarded) > 0):
ctfSetDiscarded = \
self._loadOutputSet(em.SetOfCTF, 'ctfsDiscarded.sqlite')
micSetDiscarded = \
self._loadOutputSet(em.SetOfMicrographs,
'micrographsDiscarded.sqlite')
if newDoneAccepted:
inputCtfSet = self._loadInputCtfSet()
for ctfId in newDoneAccepted:
ctf = inputCtfSet[ctfId].clone()
mic = ctf.getMicrograph().clone()
ctf.setEnabled(self._getEnable(ctfId))
mic.setEnabled(self._getEnable(ctfId))
if self.calculateConsensus:
setattr(ctf, prefixAttribute('consensus_resolution'),
Float(self._freqResol[ctfId]))
setattr(ctf, prefixAttribute('discrepancy_astigmatism'),
Float(ctf.getDefocusU() - ctf.getDefocusV()))
ctfSet.append(ctf)
micSet.append(mic)
self._writeDoneListAccepted(ctfId)
inputCtfSet.close()
if newDoneDiscarded:
inputCtfSet = self._loadInputCtfSet()
for ctfId in newDoneDiscarded:
ctf = inputCtfSet[ctfId].clone()
if self.calculateConsensus:
setattr(ctf, prefixAttribute('consensus_resolution'),
Float(self._freqResol[ctfId]))
setattr(ctf, prefixAttribute('discrepancy_astigmatism'),
Float(ctf.getDefocusU() - ctf.getDefocusV()))
mic = ctf.getMicrograph().clone()
micSetDiscarded.append(mic)
ctfSetDiscarded.append(ctf)
self._writeDoneListDiscarded(ctfId)
inputCtfSet.close()
if not self.finished and not newDoneDiscarded and not newDoneAccepted:
# If we are not finished and no new output have been produced
# it does not make sense to proceed and updated the outputs
# so we exit from the function here
return
if (os.path.exists(self._getPath('ctfs.sqlite'))):
self._updateOutputSet('outputCTF', ctfSet, streamMode)
self._updateOutputSet('outputMicrographs', micSet, streamMode)
# AJ new subsets with discarded ctfs
if (os.path.exists(self._getPath('ctfsDiscarded.sqlite'))):
self._updateOutputSet('outputCTFDiscarded',
ctfSetDiscarded, streamMode)
self._updateOutputSet('outputMicrographsDiscarded',
micSetDiscarded, streamMode)
if (os.path.exists(self._getPath('ctfs.sqlite'))):
if firstTimeAccepted:
# define relation just once
self._defineTransformRelation(
self.inputCTF.get().getMicrographs(), micSet)
self._defineTransformRelation(ctfSet, micSet)
self._defineTransformRelation(self.inputCTF, ctfSet)
self._defineCtfRelation(micSet, ctfSet)
else:
ctfSet.close()
micSet.close()
# AJ new subsets with discarded ctfs
if (os.path.exists(self._getPath('ctfsDiscarded.sqlite'))):
if firstTimeDiscarded:
self._defineTransformRelation(
self.inputCTF.get().getMicrographs(), micSetDiscarded)
self._defineTransformRelation(ctfSetDiscarded, micSetDiscarded)
self._defineTransformRelation(self.inputCTF, ctfSetDiscarded)
self._defineCtfRelation(micSetDiscarded, ctfSetDiscarded)
else:
micSetDiscarded.close()
ctfSetDiscarded.close()
if self.finished: # Unlock createOutputStep if finished all jobs
outputStep = self._getFirstJoinStep()
if outputStep and outputStep.isWaiting():
outputStep.setStatus(STATUS_NEW)
if (os.path.exists(self._getPath('ctfs.sqlite'))):
ctfSet.close()
micSet.close()
# AJ new subsets with discarded ctfs
if (os.path.exists(self._getPath('ctfsDiscarded.sqlite'))):
micSetDiscarded.close()
ctfSetDiscarded.close()
def _setWeight(self, item, row):
item._xmipp_weightClusterability = Float(
row.getValue(md.MDL_VOLUME_SCORE1))
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
# create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
# Start ids 4
img1 = Particle(objId=4)
img2 = Particle(objId=4)
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
# Test the join handles different attributes at a second level
ctf1 = CTFModel(defocusU=1000, defocusV=1000, defocusAngle=0)
ctf2 = CTFModel(defocusU=2000, defocusV=2000, defocusAngle=0)
ctf2._myOwnQuality = Float(1.)
img1.setCTF(ctf1)
img2.setCTF(ctf2)
for i in range(1, 3):
# Increment Id
img1.setObjId(img1.getObjId() + 1)
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
# Increment Id
img2.setObjId(img2.getObjId() + 1)
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
# import them
protImport1 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set1',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
emprot.ProtImportParticles,
objLabel='import set2',
importFrom=emprot.ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
# create merge protocol
p_union = self.newProtocol(emprot.ProtUnionSet,
objLabel='join different attrs',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
self.assertFalse(hasattr(img, '_attrb3'),
"join should not have attrb3")
self.assertTrue(hasattr(img, '_attrb2'), "join should have attrb2")
ctf = img.getCTF()
self.assertIsNotNone(ctf, "Image should have CTF after join")
self.assertFalse(hasattr(ctf, '_myOwnQuality'),
"CTF should not have non common attributes")
# Assert ids
self.assertEqual(counter + 5, img.getObjId(),
"Object id's not kept.")
counter += 1
class BsoftProtBlocres(ProtAnalysis3D):
"""
Bsoft program: blocres
It calculates the local resolution map from to half maps.
The method is based on a local measurement inside a mobile window.
"""
_label = 'blocres'
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
self.halfVolumes = True
# --------------------------- DEFINE param functions -----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputVolume',
params.PointerParam,
pointerClass='Volume',
label="Half 1",
help="Select first half volume to compute the "
"local resolution.")
form.addParam('inputVolume2',
params.PointerParam,
pointerClass='Volume',
label="Half 2",
help="Select first half volume to compute the "
"local resolution.")
form.addParam('mask',
params.PointerParam,
allowsNull=True,
pointerClass='Volume',
label='Mask',
help="Mask file to use for limiting the analysis to a "
"defined region and level value to use "
"(optional, default: all but zero).")
form.addParam('method',
params.BooleanParam,
default=True,
label='Use Box',
help="The local (box) and shell (shell) resolution "
"calculations are mutually exclusive.")
form.addParam('box',
params.IntParam,
default=20,
condition='method',
label='Box',
help="Kernel size for determining "
"local resolution (pixels/voxels).")
form.addParam('shell',
params.IntParam,
default=20,
condition='not method',
label='Shell',
help="Shell width for determining "
"radial resolution (pixels/voxels).")
form.addParam('cutoff',
params.FloatParam,
default=0.5,
label='Cutoff',
help="Resolution cutoff for FSC."
"(default: 0.5).")
form.addSection(label='Parameters')
form.addParam('step',
params.IntParam,
default=1,
label='Step',
help="Interval between voxel samples or shells for "
"resolution analysis (pixels, default: 1)")
form.addParam(
'maxresolution',
params.FloatParam,
default=2,
label='Maximum Resolution',
help="Maximum frequency available in the data (angstrom).")
form.addParam('fill',
params.IntParam,
allowsNull=True,
label='Fill',
help="Value to fill the background "
"(non-masked regions; default 0).")
form.addParam('pad',
params.EnumParam,
choices=['None', 'Box', 'Shell'],
default=1,
label='Padding Factor',
help="Resolution box padding factor "
"(0 = none, default: 1 (box) and 0 (shell)).")
form.addParam('symmetry',
params.StringParam,
allowsNull=True,
default='',
label='Symmetry',
help="Point group symmetry.")
form.addParam('smooth',
params.BooleanParam,
default=True,
label='Smooth',
help="Smooth the shell edge.")
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
FN_HALF1: self._getTmpPath("half1.map"),
FN_HALF2: self._getTmpPath("half2.map"),
FN_MASKVOL: self._getTmpPath("mask.map"),
FN_RESOLMAP: self._getExtraPath("resolutionMap.map")
}
self._updateFilenamesDict(myDict)
# --------------------------- INSERT steps functions -----------------------
def _insertAllSteps(self):
# Insert processing steps
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('resolutionStep')
self._insertFunctionStep('createOutputStep')
# --------------------------- STEPS functions ------------------------------
def convertInputStep(self):
# blocres works with .map
vol1Fn = self.inputVolume.get().getFileName()
vol2Fn = self.inputVolume2.get().getFileName()
if self.mask.get().getFileName() != '':
maskFn = self.mask.get().getFileName()
self.fnvol1 = self._getFileName("half1")
self.fnvol2 = self._getFileName("half2")
ImageHandler().convert(vol1Fn, self.fnvol1)
ImageHandler().convert(vol2Fn, self.fnvol2)
if self.mask.get().getFileName() != '':
self.fnmask = self._getFileName("maskvol")
ImageHandler().convert(maskFn, self.fnmask)
else:
self.fnmask = self.maks.get().getFileName()
def resolutionStep(self):
""" blocres parameters. """
sampling = self.inputVolume.get().getSamplingRate()
# Actions
params = ' -v 1' # No Verbose
if self.method:
params += ' -box %i' % self.box.get()
else:
params += ' -shell %i' % self.shell.get()
# Parameters
params += ' -sampling %f,%f,%f' % (sampling, sampling, sampling)
params += ' -step %f' % self.step.get()
params += ' -maxresolution %f' % self.maxresolution.get()
params += ' -cutoff %f' % self.cutoff.get()
if self.fill.get() != '':
params += ' -fill %i' % self.fill.get()
# Parameters for local resolution
params += ' -pad %f' % self.pad.get()
if self.symmetry.get() != '':
params += ' -symmetry %s' % self.symmetry.get()
if self.smooth.get():
params += ' -smooth'
if self.mask.get().getFileName() != '':
params += ' -Mask %s' % self.fnmask
# input halves and output map
params += ' %s %s %s' % (self.fnvol1, self.fnvol2,
self._getFileName(FN_RESOLMAP))
self.runJob(bsoft.Plugin.getProgram('blocres'),
params,
env=bsoft.Plugin.getEnviron())
def createOutputStep(self):
volume = Volume()
volume.setFileName(self._getFileName(FN_RESOLMAP))
volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolume, volume)
imageFile = self._getFileName(FN_RESOLMAP)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_ * 100) / 100)
self.max_res_init.set(round(max_ * 100) / 100)
self._store(self.min_res_init)
self._store(self.max_res_init)
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
# Remove 0's
imgData = imgData[np.nonzero(imgData)]
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
# --------------------------- INFO functions -----------------------------------
def _validate(self):
errors = []
return errors
def _citations(self):
cites = ['Cardone2013']
return cites
def _summary(self):
summary = []
summary.append("Highest resolution %.2f A, "
"Lowest resolution %.2f A." %
(self.min_res_init.get(), self.max_res_init.get()))
def _methods(self):
methods = []
return methods
class XmippProtMonoRes(ProtAnalysis3D):
"""
Given a map the protocol assigns local resolutions to each voxel of the map.
"""
_label = 'local MonoRes'
_lastUpdateVersion = VERSION_1_1
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
# --------------------------- DEFINE param functions ----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('halfVolumes',
BooleanParam,
default=False,
label="Would you like to use half volumes?",
help='The noise estimation for determining the '
'local resolution is performed via half volumes.')
form.addParam('inputVolumes',
PointerParam,
pointerClass='Volume',
label="Input Volume",
important=True,
condition='not halfVolumes',
help='Select a volume for determining its '
'local resolution.')
form.addParam('inputVolume',
PointerParam,
pointerClass='Volume',
label="Volume Half 1",
important=True,
condition='halfVolumes',
help='Select a volume for determining its '
'local resolution.')
form.addParam('inputVolume2',
PointerParam,
pointerClass='Volume',
label="Volume Half 2",
important=True,
condition='halfVolumes',
help='Select a second volume for determining a '
'local resolution.')
form.addParam('Mask',
PointerParam,
pointerClass='VolumeMask',
condition='(halfVolumes) or (not halfVolumes)',
allowsNull=True,
label="Binary Mask",
help='The mask determines which points are specimen'
' and which are not')
group = form.addGroup('Extra parameters')
# group.addParam('symmetry', StringParam, default='c1',
# label="Symmetry",
# help='Symmetry group. By default = c1.'
# 'See [[http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry][Symmetry]]'
# 'for a description of the symmetry groups format,'
# 'If no symmetry is present, give c1.')
line = group.addLine(
'Resolution Range (Å)',
help='If the user knows the range of resolutions or'
'only a range of frequencies needs to be analysed.'
'If Low is empty MonoRes will try to estimate the range. '
'it should be better if a range is provided')
group.addParam('significance',
FloatParam,
default=0.95,
expertLevel=LEVEL_ADVANCED,
label="Significance",
help='Relution is computed using hipothesis tests, '
'this value determines the significance of that test')
group.addParam(
'maskthreshold',
FloatParam,
default=0.5,
expertLevel=LEVEL_ADVANCED,
label="Mask threshold",
help='If the provided mask is not binary. Then, MonoRes'
'will try to binarize it. Bmask values below the threshold'
'will be change to 0 and above the thresthol will be 1')
group.addParam('isPremasked',
BooleanParam,
default=False,
label="Is the original premasked?",
help='Sometimes the original volume is masked inside '
'a spherical mask. In this case please select yes')
form.addParam(
'noiseonlyinhalves',
BooleanParam,
expertLevel=LEVEL_ADVANCED,
default=True,
label="Use noise outside the mask?",
condition='halfVolumes',
help='Select yes if the volume present noise outside the mask.'
' Otherwise, select No.')
group.addParam('volumeRadius',
FloatParam,
default=-1,
label="Spherical mask radius (px)",
condition='isPremasked and not halfVolumes',
help='When the original volume is originally premasked,'
'the noise estimation ought to be performed inside'
'that premask, and out of the provieded mask asked in'
'the previus box. The radius value, determines the'
'radius of the spherical premask. '
'By default radius = -1 use the half of the '
'volume size as radius')
group.addParam(
'volumeRadiusHalf',
FloatParam,
default=-1,
label="Spherical mask radius (px)",
condition='halfVolumes and isPremasked',
help='When the origianl volume is originally premasked,'
'the noise estimation ought to be performed inside that'
'premask, and out of the provieded mask asked in the previus'
'box. The radius value, determines the radius in pixels of '
'the spherical premask. By default radius = -1 use the half'
'of the volume size as radius')
line.addParam('minRes', FloatParam, default=0, label='High')
line.addParam('maxRes',
FloatParam,
default=12,
allowsNull=True,
label='Low')
line.addParam('stepSize',
FloatParam,
allowsNull=True,
expertLevel=LEVEL_ADVANCED,
label='Step')
group.addParam('filterInput',
BooleanParam,
default=False,
label="Filter input volume with local resolution?",
help='The input map is locally filtered at'
'the local resolution map.')
form.addParallelSection(threads=4, mpi=0)
# --------------------------- INSERT steps functions --------------------------------------------
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
FN_MEAN_VOL:
self._getExtraPath('mean_volume.vol'),
OUTPUT_MASK_FILE:
self._getExtraPath("output_Mask.vol"),
OUTPUT_RESOLUTION_FILE_CHIMERA:
self._getExtraPath(CHIMERA_RESOLUTION_VOL),
FN_FILTERED_MAP:
self._getExtraPath('filteredMap.vol'),
OUTPUT_RESOLUTION_FILE:
self._getExtraPath('mgresolution.vol'),
METADATA_MASK_FILE:
self._getExtraPath('mask_data.xmd'),
FN_METADATA_HISTOGRAM:
self._getExtraPath('hist.xmd'),
BINARY_MASK:
self._getExtraPath('binarized_mask.vol'),
FN_GAUSSIAN_MAP:
self._getExtraPath('gaussianfilted.vol')
}
self._updateFilenamesDict(myDict)
def _insertAllSteps(self):
# Convert input into xmipp Metadata format
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep', )
self._insertFunctionStep('resolutionMonogenicSignalStep')
self._insertFunctionStep('createOutputStep')
self._insertFunctionStep("createHistrogram")
def convertInputStep(self):
""" Read the input volume.
"""
self.micsFn = self._getPath()
if self.halfVolumes:
self.vol1Fn = self.inputVolume.get().getFileName()
self.vol2Fn = self.inputVolume2.get().getFileName()
extVol1 = getExt(self.vol1Fn)
extVol2 = getExt(self.vol2Fn)
if (extVol1 == '.mrc') or (extVol1 == '.map'):
self.vol1Fn = self.vol1Fn + ':mrc'
if (extVol2 == '.mrc') or (extVol2 == '.map'):
self.vol2Fn = self.vol2Fn + ':mrc'
if not self.Mask.hasValue():
self.ifNomask(self.vol1Fn)
else:
self.maskFn = self.Mask.get().getFileName()
self.inputVolumes.set(None)
else:
self.vol0Fn = self.inputVolumes.get().getFileName()
extVol0 = getExt(self.vol0Fn)
if (extVol0 == '.mrc') or (extVol0 == '.map'):
self.vol0Fn = self.vol0Fn + ':mrc'
if not self.Mask.hasValue():
self.ifNomask(self.vol0Fn)
else:
self.maskFn = self.Mask.get().getFileName()
print self.maskFn
self.inputVolume.set(None)
self.inputVolume2.set(None)
extMask = getExt(self.maskFn)
if (extMask == '.mrc') or (extMask == '.map'):
self.maskFn = self.maskFn + ':mrc'
if self.Mask.hasValue():
params = ' -i %s' % self.maskFn
params += ' -o %s' % self._getFileName(BINARY_MASK)
params += ' --select below %f' % self.maskthreshold.get()
params += ' --substitute binarize'
self.runJob('xmipp_transform_threshold', params)
def ifNomask(self, fnVol):
if self.halfVolumes:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
params = ' -i %s' % fnVol
else:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
params = ' -i %s' % fnVol
params += ' -o %s' % self._getFileName(FN_GAUSSIAN_MAP)
setsize = 0.02 * xdim
params += ' --fourier real_gaussian %f' % (setsize)
self.runJob('xmipp_transform_filter', params)
img = ImageHandler().read(self._getFileName(FN_GAUSSIAN_MAP))
imgData = img.getData()
max_val = np.amax(imgData) * 0.05
params = ' -i %s' % self._getFileName(FN_GAUSSIAN_MAP)
params += ' --select below %f' % max_val
params += ' --substitute binarize'
params += ' -o %s' % self._getFileName(BINARY_MASK)
self.runJob('xmipp_transform_threshold', params)
self.maskFn = self._getFileName(BINARY_MASK)
def maskRadius(self, halfmaps, premaskedmap):
if halfmaps:
if premaskedmap:
if self.volumeRadiusHalf == -1:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim * 0.5
else:
xdim = self.volumeRadiusHalf.get()
else:
xdim, _ydim, _zdim = self.inputVolume.get().getDim()
xdim = xdim * 0.5
else:
if premaskedmap:
if self.volumeRadius == -1:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim * 0.5
else:
xdim = self.volumeRadius.get()
else:
xdim, _ydim, _zdim = self.inputVolumes.get().getDim()
xdim = xdim * 0.5
return xdim
def resolutionMonogenicAuto(self):
freq_step = 1
xdim = self.maskRadius(self.halfVolumes, self.isPremasked)
if self.halfVolumes:
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getFileName(FN_MEAN_VOL)
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolume.get(
).getSamplingRate()
if (self.noiseonlyinhalves is False):
params += ' --noiseonlyinhalves'
else:
params = ' --vol %s' % self.vol0Fn
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolumes.get(
).getSamplingRate()
params += ' --automatic '
params += ' --step %f' % freq_step
params += ' --mask_out %s' % self._getFileName(OUTPUT_MASK_FILE)
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --volumeRadius %f' % xdim
params += ' --chimera_volume %s' % self._getFileName(
OUTPUT_RESOLUTION_FILE_CHIMERA)
params += ' --sym %s' % 'c1' #self.symmetry.get()
params += ' --significance %f' % self.significance.get()
params += ' --md_outputdata %s' % self._getFileName(METADATA_MASK_FILE)
params += ' --filtered_volume %s' % ''
params += ' --threads %i' % self.numberOfThreads.get()
self.runJob('xmipp_resolution_monogenic_signal', params)
def resolutionMonogenicSignalStep(self):
# Number of frequencies
if (not self.maxRes.hasValue()):
self.resolutionMonogenicAuto()
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE)
min_, max_ = self.getMinMax(imageFile)
else:
max_ = self.maxRes.get()
if self.stepSize.hasValue():
freq_step = self.stepSize.get()
else:
freq_step = 0.25
xdim = self.maskRadius(self.halfVolumes, self.isPremasked)
if self.halfVolumes:
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getFileName(FN_MEAN_VOL)
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolume.get(
).getSamplingRate()
if (self.noiseonlyinhalves is False):
params += ' --noiseonlyinhalves'
else:
params = ' --vol %s' % self.vol0Fn
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.inputVolumes.get(
).getSamplingRate()
params += ' --minRes %f' % self.minRes.get()
params += ' --maxRes %f' % max_
params += ' --step %f' % freq_step
params += ' --mask_out %s' % self._getFileName(OUTPUT_MASK_FILE)
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --volumeRadius %f' % xdim
params += ' --exact'
params += ' --chimera_volume %s' % self._getFileName(
OUTPUT_RESOLUTION_FILE_CHIMERA)
params += ' --sym %s' % 'c1' #self.symmetry.get()
params += ' --significance %f' % self.significance.get()
params += ' --md_outputdata %s' % self._getFileName(METADATA_MASK_FILE)
params += ' --threads %i' % self.numberOfThreads.get()
if self.filterInput.get():
params += ' --filtered_volume %s' % self._getFileName(
FN_FILTERED_MAP)
else:
params += ' --filtered_volume %s' % ''
self.runJob('xmipp_resolution_monogenic_signal', params)
def createHistrogram(self):
M = float(self.max_res_init)
m = float(self.min_res_init)
range_res = round((M - m) * 4.0)
params = ' -i %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --mask binary_file %s' % self._getFileName(
OUTPUT_MASK_FILE)
params += ' --steps %f' % (range_res)
params += ' --range %f %f' % (self.min_res_init, self.max_res_init)
params += ' -o %s' % self._getFileName(FN_METADATA_HISTOGRAM)
self.runJob('xmipp_image_histogram', params)
def readMetaDataOutput(self):
mData = md.MetaData(self._getFileName(METADATA_MASK_FILE))
NvoxelsOriginalMask = float(
mData.getValue(md.MDL_COUNT, mData.firstObject()))
NvoxelsOutputMask = float(
mData.getValue(md.MDL_COUNT2, mData.firstObject()))
nvox = int(
round(((NvoxelsOriginalMask - NvoxelsOutputMask) /
NvoxelsOriginalMask) * 100))
return nvox
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
def createOutputStep(self):
volume = Volume()
volume.setFileName(self._getFileName(OUTPUT_RESOLUTION_FILE))
if (self.halfVolumes):
volume.setSamplingRate(self.inputVolume.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolume, volume)
else:
volume.setSamplingRate(self.inputVolumes.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.inputVolumes, volume)
#Setting the min max for the summary
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE_CHIMERA)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_ * 100) / 100)
self.max_res_init.set(round(max_ * 100) / 100)
self._store(self.min_res_init)
self._store(self.max_res_init)
if self.filterInput.get():
print 'Saving filtered map'
volume.setFileName(self._getFileName(FN_FILTERED_MAP))
if (self.halfVolumes):
volume.setSamplingRate(
self.inputVolume.get().getSamplingRate())
self._defineOutputs(outputVolume_Filtered=volume)
self._defineSourceRelation(self.inputVolume, volume)
else:
volume.setSamplingRate(
self.inputVolumes.get().getSamplingRate())
self._defineOutputs(outputVolume_Filtered=volume)
self._defineSourceRelation(self.inputVolumes, volume)
# --------------------------- INFO functions ------------------------------
def _methods(self):
messages = []
if hasattr(self, 'resolution_Volume'):
messages.append('Information about the method/article in ' +
MONORES_METHOD_URL)
return messages
def _summary(self):
summary = []
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" %
(self.min_res_init, self.max_res_init))
return summary
def _citations(self):
return ['Vilas2018']
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
self.halfVolumes = True
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
self.median_res_init = Float()
class XmippProtMonoTomo(ProtAnalysis3D):
"""
Given a tomogram the protocol assigns local resolutions to each voxel of the tomogram.
"""
_label = 'local Resolution MonoTomo'
_lastUpdateVersion = VERSION_2_0
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
# --------------------------- DEFINE param functions ----------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('half1', PointerParam, pointerClass='Tomogram',
label="Odd tomogram", important=True,
help='Select a volume for determining its '
'local resolution.')
form.addParam('half2', PointerParam, pointerClass='Tomogram',
label="Even Tomogram", important=True,
help='Select a second volume for determining a '
'local resolution.')
form.addParam('useMask', BooleanParam, default=False,
label="Use mask?",
help='The mask determines which points are specimen'
' and which are not.')
form.addParam('Mask', PointerParam, pointerClass='VolumeMask',
condition='useMask', allowsNull=True,
label="Binary Mask",
help='The mask determines which points are specimen'
' and which are not')
group = form.addGroup('Extra parameters')
line = group.addLine('Resolution Range (Å)',
help="Resolution range (and step in expert mode) "
"to evaluate the local resolution.")
group.addParam('significance', FloatParam, default=0.95,
expertLevel=LEVEL_ADVANCED,
label="Significance",
help='Relution is computed using hipothesis tests, '
'this value determines the significance of that test')
line.addParam('minRes', FloatParam, default=0, label='High')
line.addParam('maxRes', FloatParam, allowsNull=True, label='Low')
line.addParam('stepSize', FloatParam, allowsNull=True, default=0.25,
expertLevel=LEVEL_ADVANCED, label='Step')
form.addParallelSection(threads=4, mpi=0)
# --------------------------- INSERT steps functions --------------------------------------------
def _createFilenameTemplates(self):
""" Centralize how files are called """
myDict = {
FN_MEAN_VOL: self._getExtraPath('mean_Tomogram.mrc'),
OUTPUT_RESOLUTION_FILE: self._getExtraPath('TomogramLocalResolution.mrc'),
METADATA_MASK_FILE: self._getExtraPath('mask_data.xmd'),
FN_METADATA_HISTOGRAM: self._getExtraPath('hist.xmd'),
BINARY_MASK: self._getExtraPath('binarized_mask.mrc'),
FN_GAUSSIAN_MAP: self._getExtraPath('gaussianfilted.mrc'),
}
self._updateFilenamesDict(myDict)
def _insertAllSteps(self):
# Convert input into xmipp Metadata format
self._createFilenameTemplates()
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('resolutionMonoTomoStep')
self._insertFunctionStep('createOutputStep')
self._insertFunctionStep("createHistrogram")
def convertInputStep(self):
""" Read the input volume.
"""
self.vol1Fn = self.half1.get().getFileName()
self.vol2Fn = self.half2.get().getFileName()
extVol1 = getExt(self.vol1Fn)
extVol2 = getExt(self.vol2Fn)
if (extVol1 == '.mrc') or (extVol1 == '.map'):
self.vol1Fn = self.vol1Fn + ':mrc'
if (extVol2 == '.mrc') or (extVol2 == '.map'):
self.vol2Fn = self.vol2Fn + ':mrc'
if self.useMask.get():
if (not self.Mask.hasValue()):
self.ifNomask(self.vol1Fn)
else:
self.maskFn = self.Mask.get().getFileName()
extMask = getExt(self.maskFn)
if (extMask == '.mrc') or (extMask == '.map'):
self.maskFn = self.maskFn + ':mrc'
if self.Mask.hasValue():
params = ' -i %s' % self.maskFn
params += ' -o %s' % self._getFileName(BINARY_MASK)
params += ' --select below %f' % 0.5 # Mask threshold = 0.5 self.maskthreshold.get()
params += ' --substitute binarize'
self.runJob('xmipp_transform_threshold', params)
def ifNomask(self, fnVol):
xdim, _ydim, _zdim = self.half1.get().getDim()
params = ' -i %s' % fnVol
params += ' -o %s' % self._getFileName(FN_GAUSSIAN_MAP)
setsize = 0.02 * xdim
params += ' --fourier real_gaussian %f' % (setsize)
self.runJob('xmipp_transform_filter', params)
img = ImageHandler().read(self._getFileName(FN_GAUSSIAN_MAP))
imgData = img.getData()
max_val = np.amax(imgData) * 0.05
params = ' -i %s' % self._getFileName(FN_GAUSSIAN_MAP)
params += ' --select below %f' % max_val
params += ' --substitute binarize'
params += ' -o %s' % self._getFileName(BINARY_MASK)
self.runJob('xmipp_transform_threshold', params)
self.maskFn = self._getFileName(BINARY_MASK)
def maskRadius(self):
xdim, _ydim, _zdim = self.half1.get().getDim()
xdim = xdim * 0.5
return xdim
def resolutionMonoTomoStep(self):
# Number of frequencies
max_ = self.maxRes.get()
if self.stepSize.hasValue():
freq_step = self.stepSize.get()
else:
freq_step = 0.25
xdim = self.maskRadius()
params = ' --vol %s' % self.vol1Fn
params += ' --vol2 %s' % self.vol2Fn
params += ' --meanVol %s' % self._getFileName(FN_MEAN_VOL)
if self.useMask.get():
params += ' --mask %s' % self._getFileName(BINARY_MASK)
params += ' --sampling_rate %f' % self.half1.get().getSamplingRate()
params += ' --minRes %f' % self.minRes.get()
params += ' --maxRes %f' % max_
params += ' --step %f' % freq_step
params += ' -o %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
params += ' --significance %f' % self.significance.get()
self.runJob('xmipp_resolution_monotomo', params)
def createHistrogram(self):
freq_step = self.stepSize.get() if self.stepSize.hasValue() else 10
M = float(self.max_res_init)
m = float(self.min_res_init)
range_res = round((M - m) / freq_step)
params = ' -i %s' % self._getFileName(OUTPUT_RESOLUTION_FILE)
if self.useMask.get():
params += ' --mask binary_file %s' % self._getFileName(BINARY_MASK)
params += ' --steps %f' % range_res
params += ' --range %f %f' % (self.min_res_init.get(),
(float(self.max_res_init.get()) - float(freq_step)))
params += ' -o %s' % self._getFileName(FN_METADATA_HISTOGRAM)
self.runJob('xmipp_image_histogram', params)
def readMetaDataOutput(self):
mData = md.MetaData(self._getFileName(METADATA_MASK_FILE))
NvoxelsOriginalMask = float(mData.getValue(md.MDL_COUNT, mData.firstObject()))
NvoxelsOutputMask = float(mData.getValue(md.MDL_COUNT2, mData.firstObject()))
nvox = int(round(
((NvoxelsOriginalMask - NvoxelsOutputMask) / NvoxelsOriginalMask) * 100))
return nvox
def getMinMax(self, imageFile):
img = ImageHandler().read(imageFile)
imgData = img.getData()
min_res = round(np.amin(imgData) * 100) / 100
max_res = round(np.amax(imgData) * 100) / 100
return min_res, max_res
def createOutputStep(self):
volume = Tomogram()
volume.setFileName(self._getFileName(OUTPUT_RESOLUTION_FILE))
volume.setSamplingRate(self.half1.get().getSamplingRate())
self._defineOutputs(resolution_Volume=volume)
self._defineSourceRelation(self.half1, volume)
# Setting the min max for the summary
imageFile = self._getFileName(OUTPUT_RESOLUTION_FILE)
min_, max_ = self.getMinMax(imageFile)
self.min_res_init.set(round(min_ * 100) / 100)
self.max_res_init.set(round(max_ * 100) / 100)
self._store(self.min_res_init)
self._store(self.max_res_init)
# --------------------------- INFO functions ------------------------------
def _methods(self):
messages = []
if hasattr(self, 'LocalResolution_Tomogram'):
messages.append(
'Information about the method/article in ' + MONOTOMO_METHOD_URL)
return messages
def _summary(self):
summary = []
summary.append("Highest resolution %.2f Å, "
"Lowest resolution %.2f Å. \n" % (self.min_res_init,
self.max_res_init))
return summary
def _citations(self):
return ['Vilas2020']
def __init__(self, **args):
ProtAnalysis3D.__init__(self, **args)
self.min_res_init = Float()
self.max_res_init = Float()
def testMergeDifferentAttrs(self):
""" Test merge from subsets with different attritubes.
That is, M1(a,b,c) U M2(a,b,c,d)"""
#create two set of particles
inFileNameMetadata1 = self.proj.getTmpPath('particles11.sqlite')
inFileNameMetadata2 = self.proj.getTmpPath('particles22.sqlite')
imgSet1 = SetOfParticles(filename=inFileNameMetadata1)
imgSet2 = SetOfParticles(filename=inFileNameMetadata2)
inFileNameData = self.proj.getTmpPath('particles.stk')
img1 = Particle()
img2 = Particle()
attrb1 = [11, 12, 13, 14]
attrb2 = [21, 22, 23, 24]
attrb3 = [31, 32]
counter = 0
for i in range(1, 3):
img1.cleanObjId()
img1.setLocation(i, inFileNameData)
img1.setMicId(i % 3)
img1.setClassId(i % 5)
img1.setSamplingRate(1.)
img1._attrb1 = Float(attrb1[counter])
img1._attrb2 = Float(attrb2[counter])
img1._attrb3 = Float(attrb3[counter])
imgSet1.append(img1)
counter += 1
for i in range(1, 3):
img2.cleanObjId()
img2.setLocation(i, inFileNameData)
img2.setClassId(i % 5)
img2.setMicId(i % 3)
img2.setSamplingRate(2.)
img2._attrb1 = Float(attrb1[counter])
img2._attrb2 = Float(attrb2[counter])
imgSet2.append(img2)
counter += 1
imgSet1.write()
imgSet2.write()
#import them
protImport1 = self.newProtocol(
ProtImportParticles,
objLabel='import set1',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata1,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport1)
protImport2 = self.newProtocol(
ProtImportParticles,
objLabel='import set2',
importFrom=ProtImportParticles.IMPORT_FROM_SCIPION,
sqliteFile=inFileNameMetadata2,
magnification=10000,
samplingRate=7.08,
haveDataBeenPhaseFlipped=True)
self.launchProtocol(protImport2)
#create merge protocol
p_union = self.newProtocol(ProtUnionSet,
objLabel='join diff column order',
ignoreExtraAttributes=True)
p_union.inputSets.append(protImport1.outputParticles)
p_union.inputSets.append(protImport2.outputParticles)
self.proj.launchProtocol(p_union, wait=True)
#assert
counter = 0
for img in p_union.outputSet:
self.assertAlmostEqual(attrb1[counter], img._attrb1, 4)
self.assertAlmostEqual(attrb2[counter], img._attrb2, 4)
if hasattr(img, '_attrb3'):
self.assertTrue(False, "join should not have attrb3")
if not hasattr(img, '_attrb2'):
self.assertTrue(False, "join should have attrb2")
counter += 1
