class PcaFile(EMObject):
""" This is a container of files produced by CA PCA Spider protocol.
It is possible to use the cas_IMC or cas_SEQ files.
"""
def __init__(self, **args):
EMObject.__init__(self, **args)
self.filename = String()
def getFileName(self):
return self.filename.get()
class XmippProtScreenParticles(ProtProcessParticles):
""" Classify particles according their similarity to the others in order
to detect outliers. """
_label = 'screen particles'
# Automatic Particle rejection enum
ZSCORE_CHOICES = ['None', 'MaxZscore', 'Percentage']
REJ_NONE = 0
REJ_MAXZSCORE = 1
REJ_PERCENTAGE = 2
REJ_PERCENTAGE_SSNR = 1
# --------------------------- DEFINE param functions -----------------------
def _defineProcessParams(self, form):
form.addParam(
'autoParRejection',
EnumParam,
choices=self.ZSCORE_CHOICES,
label="Automatic particle rejection based on Zscore",
default=self.REJ_NONE,
display=EnumParam.DISPLAY_COMBO,
expertLevel=LEVEL_ADVANCED,
help='How to automatically reject particles. It can be:\n'
' None (no rejection)\n'
' MaxZscore (reject a particle if its Zscore [a '
'similarity index] is larger than this value).\n '
' Percentage (reject a given percentage in each '
'one of the screening criteria).')
form.addParam('maxZscore',
FloatParam,
default=3,
condition='autoParRejection==1',
label='Maximum Zscore',
expertLevel=LEVEL_ADVANCED,
help='Maximum Zscore.',
validators=[Positive])
form.addParam('percentage',
IntParam,
default=5,
condition='autoParRejection==2',
label='Percentage (%)',
expertLevel=LEVEL_ADVANCED,
help='The worse percentage of particles according to '
'metadata labels: ZScoreShape1, ZScoreShape2, '
'ZScoreSNR1, ZScoreSNR2, ZScoreHistogram are '
'automatically disabled. Therefore, the total '
'number of disabled particles belongs to ['
'percetage, 5*percentage]',
validators=[
Range(0,
100,
error="Percentage must be "
"between 0 and 100.")
])
form.addParam('autoParRejectionSSNR',
EnumParam,
choices=['None', 'Percentage'],
label="Automatic particle rejection based on SSNR",
default=self.REJ_NONE,
display=EnumParam.DISPLAY_COMBO,
expertLevel=LEVEL_ADVANCED,
help='How to automatically reject particles based on '
'SSNR. It can be:\n'
' None (no rejection)\n'
'Percentage (reject a given percentage of the '
'lowest SSNRs).')
form.addParam('percentageSSNR',
IntParam,
default=5,
condition='autoParRejectionSSNR==1',
label='Percentage (%)',
expertLevel=LEVEL_ADVANCED,
help='The worse percentage of particles according to '
'SSNR are automatically disabled.',
validators=[
Range(0,
100,
error="Percentage must be "
"between 0 and 100.")
])
form.addParallelSection(threads=0, mpi=0)
# --------------------------- INSERT steps functions -----------------------
def _insertAllSteps(self):
""" Mainly prepare the command line for call the program"""
# Convert input images if necessary
partSetId = self.inputParticles.getObjId()
self._insertFunctionStep('sortImages', partSetId)
self._insertFunctionStep('sortImagesSSNR', partSetId)
self._insertFunctionStep('createOutputStep')
# --------------------------- STEPS functions ------------------------------
def sortImages(self, inputId):
imagesMd = self._getPath('images.xmd')
writeSetOfParticles(self.inputParticles.get(), imagesMd)
args = "-i Particles@%s --addToInput " % imagesMd
if self.autoParRejection == self.REJ_MAXZSCORE:
args += "--zcut " + str(self.maxZscore.get())
elif self.autoParRejection == self.REJ_PERCENTAGE:
args += "--percent " + str(self.percentage.get())
self.runJob("xmipp_image_sort_by_statistics", args)
self.outputMd = String(imagesMd)
def sortImagesSSNR(self, inputId):
imagesMd = self._getPath('images.xmd')
args = "-i Particles@%s " % imagesMd
if self.autoParRejectionSSNR == self.REJ_PERCENTAGE_SSNR:
args += "--ssnrpercent " + str(self.percentageSSNR.get())
self.runJob("xmipp_image_ssnr", args)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
self._initializeZscores()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateParticle,
itemDataIterator=md.iterRows(
self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
# Store Zcore summary values.
self._store()
def _calculateSummaryValues(self, particle):
zScore = particle._xmipp_zScore.get()
self.minZScore.set(min(zScore, self.minZScore.get(1000)))
self.maxZScore.set(max(zScore, self.maxZScore.get(0)))
self.sumZScore.set(self.sumZScore.get(0) + zScore)
# -------------------------- INFO functions --------------------------------
def _summary(self):
summary = []
if self.autoParRejection is not None:
summary.append("Rejection method: " +
self.ZSCORE_CHOICES[self.autoParRejection.get()])
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
if hasattr(self, 'sumZScore'):
summary.append("The minimum ZScore is %.2f" % self.minZScore)
summary.append("The maximum ZScore is %.2f" % self.maxZScore)
meanZScore = self.sumZScore.get() * 1.0 / len(
self.outputParticles)
summary.append("The mean ZScore is %.2f" % meanZScore)
else:
summary.append(
"Summary values not calculated during processing.")
return summary
def _validate(self):
pass
def _citations(self):
return ['Vargas2013b']
def _methods(self):
methods = []
if hasattr(self, 'outputParticles'):
outParticles = (len(self.outputParticles)
if self.outputParticles is not None else None)
particlesRejected = (len(self.inputParticles.get()) - outParticles
if outParticles is not None else None)
particlesRejectedText = (' (' + str(particlesRejected) + ')'
if particlesRejected is not None else '')
rejectionText = [
'', # REJ_NONE
' and removing those not reaching %s%s' %
(str(self.maxZscore.get()),
particlesRejectedText), # REJ_MAXZSCORE
' and removing worst %s percent %s' %
(str(self.percentage.get()), particlesRejectedText
) # REJ_PERCENTAGE
]
methods.append('Input dataset %s of %s particles was sorted by'
' its ZScore using xmipp_image_sort_by_statistics'
' program%s. ' %
(self.getObjectTag('inputParticles'),
len(self.inputParticles.get()),
rejectionText[self.autoParRejection.get()]))
methods.append('Output set is %s.' %
self.getObjectTag('outputParticles'))
return methods
# --------------------------- UTILS functions ------------------------------
def _updateParticle(self, item, row):
setXmippAttributes(item, row, md.MDL_ZSCORE, md.MDL_ZSCORE_SHAPE1,
md.MDL_ZSCORE_SHAPE2, md.MDL_ZSCORE_SNR1,
md.MDL_ZSCORE_SNR2, md.MDL_CUMULATIVE_SSNR)
if row.getValue(md.MDL_ENABLED) <= 0:
item._appendItem = False
else:
item._appendItem = True
self._calculateSummaryValues(item)
def _initializeZscores(self):
# Store the set for later access , ;-(
self.minZScore = Float()
self.maxZScore = Float()
self.sumZScore = Float()
class XmippProtParticlePickingPairs(ProtParticlePicking, XmippProtocol):
""" Picks particles in a set of untilted-tilted pairs of micrographs. """
_label = 'tilt pairs particle picking'
def __init__(self, **args):
ProtParticlePicking.__init__(self, **args)
# The following attribute is only for testing
self.importFolder = String(args.get('importFolder', None))
#--------------- DEFINE param functions ---------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMicrographsTiltedPair',
params.PointerParam,
pointerClass='MicrographsTiltPair',
label="Micrographs tilt pair",
help='Select the MicrographsTiltPair ')
form.addParam('memory',
params.FloatParam,
default=2,
label='Memory to use (In Gb)',
expertLevel=2)
#----------- INSERT steps functions ----------------------------------
def _insertAllSteps(self):
""" The Particle Picking process is realized for a pair
of set of micrographs
"""
self.micsFn = self._getPath('input_micrographs.xmd')
# Convert input into xmipp Metadata format
self._insertFunctionStep('convertInputStep')
# Launch Particle Picking GUI
if not self.importFolder.hasValue():
self._insertFunctionStep('launchParticlePickGUIStep',
interactive=True)
else: # This is only used for test purposes
self._insertFunctionStep('_importFromFolderStep')
#------------------- STEPS functions -----------------------------------
def convertInputStep(self):
micTiltPairs = self.inputMicrographsTiltedPair.get()
# Get the converted input micrographs in Xmipp format
convert.writeSetOfMicrographsPairs(micTiltPairs.getUntilted(),
micTiltPairs.getTilted(),
self.micsFn)
def launchParticlePickGUIStep(self):
process = launchTiltPairPickerGUI(self.micsFn,
self._getExtraPath(),
self,
memory='%dg' % self.memory.get())
process.wait()
def _importFromFolderStep(self):
""" This function will copy Xmipp .pos files for
simulating a particle picking run...this is only
for testing purposes.
"""
extraDir = self._getExtraPath()
for f in pwutils.getFiles(self.importFolder.get()):
pwutils.copyFile(f, extraDir)
self.registerCoords(extraDir, readFromExtra=True)
#--------------------------- INFO functions --------------------------------------------
def _citations(self):
return []
#--------------------------- UTILS functions -------------------------------------------
def __str__(self):
""" String representation of a Particle Picking Tilt run """
outputs = self.getOutputsSize()
if outputs == 0:
msg = "No particles picked yet."
elif outputs == 1:
picked = self.getCoords().getSize()
mics = self.inputMicrographsTiltedPair.get().getTilted().getSize()
msg = "Number of particles picked: %d " % picked
msg += "(from %d micrographs)" % mics
else:
msg = 'Number of outputs: %d' % outputs
return msg
def getInputMicrographs(self):
return self.inputMicrographsTiltedPair.get().getTilted()
def getCoords(self):
return self.getCoordsTiltPair()
def _summary(self):
summary = []
if self.getInputMicrographs() is not None:
summary.append("Number of input micrographs: %d" %
self.getInputMicrographs().getSize())
if self.getOutputsSize() >= 1:
for key, output in self.iterOutputAttributes(CoordinatesTiltPair):
summary.append("*%s:*" % key)
summary.append(" Particles pairs picked: %d" %
output.getSize())
summary.append(" Particle size: %d \n" % output.getBoxSize())
else:
summary.append("Output tilpairs not ready yet.")
return summary
def __getOutputSuffix(self):
maxCounter = -1
for attrName, _ in self.iterOutputAttributes(CoordinatesTiltPair):
suffix = attrName.replace('outputCoordinatesTiltPair', '')
try:
counter = int(suffix)
except:
counter = 1 # when there is not number assume 1
maxCounter = max(counter, maxCounter)
return str(maxCounter +
1) if maxCounter > 0 else '' # empty if not outputs
def _getBoxSize(self):
""" Redefine this function to set a specific box size to the output
coordinates untilted and tilted.
"""
return None
def _readCoordinates(self, coordsDir, suffix=''):
micTiltPairs = self.inputMicrographsTiltedPair.get()
uSuffix = 'Untilted' + suffix
tSuffix = 'Tilted' + suffix
uSet = micTiltPairs.getUntilted()
tSet = micTiltPairs.getTilted()
# Create Untilted and Tilted SetOfCoordinates
uCoordSet = self._createSetOfCoordinates(uSet, suffix=uSuffix)
convert.readSetOfCoordinates(coordsDir, uSet, uCoordSet)
uCoordSet.write()
tCoordSet = self._createSetOfCoordinates(tSet, suffix=tSuffix)
convert.readSetOfCoordinates(coordsDir, tSet, tCoordSet)
tCoordSet.write()
boxSize = self._getBoxSize()
if boxSize:
uCoordSet.setBoxSize(boxSize)
tCoordSet.setBoxSize(boxSize)
return uCoordSet, tCoordSet
def _readAngles(self, micsFn, suffix=''):
# Read Angles from input micrographs
anglesSet = self._createSetOfAngles(suffix=suffix)
convert.readAnglesFromMicrographs(micsFn, anglesSet)
anglesSet.write()
return anglesSet
def registerCoords(self, coordsDir, store=True, readFromExtra=False):
micTiltPairs = self.inputMicrographsTiltedPair.get()
suffix = self.__getOutputSuffix()
uCoordSet, tCoordSet = self._readCoordinates(coordsDir, suffix)
if readFromExtra:
micsFn = self._getExtraPath('input_micrographs.xmd')
else:
micsFn = self._getPath('input_micrographs.xmd')
anglesSet = self._readAngles(micsFn, suffix)
# Create CoordinatesTiltPair object
outputset = self._createCoordinatesTiltPair(micTiltPairs, uCoordSet,
tCoordSet, anglesSet,
suffix)
summary = self.getSummary(outputset)
outputset.setObjComment(summary)
outputName = 'outputCoordinatesTiltPair' + suffix
outputs = {outputName: outputset}
self._defineOutputs(**outputs)
self._defineSourceRelation(self.inputMicrographsTiltedPair, outputset)
if store:
self._store()
class XmippProtDenoiseParticles(ProtProcessParticles):
""" Remove particles noise by filtering them.
This filtering process is based on a projection over a basis created
from some averages (extracted from classes). This filtering is not
intended for processing particles. The huge filtering they will be
passed through is known to remove part of the signal with the noise.
However this is a good method for clearly see which particle are we
going to process before it's done.
"""
_label = 'denoise particles'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
# First we customize the inputParticles param to fit our needs in this protocol
form.getParam('inputParticles').pointerCondition = String(
'hasAlignment')
form.getParam('inputParticles').help = String(
'Input images you want to filter. It is important that the images have alignment information with '
'respect to the chosen set of classes. This is the standard situation '
'after CL2D or ML2D.')
form.addParam(
'inputClasses',
PointerParam,
label='Input Classes',
important=True,
pointerClass='SetOfClasses',
help=
'Select the input classes for the basis construction against images will be projected to.'
)
form.addSection(label='Basis construction')
form.addParam('maxClasses',
IntParam,
default=128,
label='Max. number of classes',
expertLevel=LEVEL_ADVANCED,
help='Maximum number of classes.')
form.addParam('maxPCABases',
IntParam,
default=200,
label='Number of PCA bases',
expertLevel=LEVEL_ADVANCED,
help='Number of PCA bases.')
form.addSection(label='Denoising')
form.addParam('PCABases2Project',
IntParam,
default=200,
label='Number of PCA bases on which to project',
expertLevel=LEVEL_ADVANCED,
help='Number of PCA bases on which to project.')
def _getDefaultParallel(self):
""" Return the default value for thread and MPI
for the parallel section definition.
"""
return (2, 4)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Insert every step of the protocol"""
# Convert input images if necessary
self._insertFunctionStep('denoiseImages',
self.inputParticles.getObjId(),
self.inputClasses.getObjId())
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def denoiseImages(self, inputId, inputClassesId):
# We start preparing writing those elements we're using as input to keep them untouched
imagesMd = self._getPath('images.xmd')
writeSetOfParticles(self.inputParticles.get(), imagesMd)
classesMd = self._getPath('classes.xmd')
writeSetOfClasses2D(self.inputClasses.get(), classesMd)
fnRoot = self._getExtraPath('pca')
fnRootDenoised = self._getExtraPath('imagesDenoised')
args = '-i Particles@%s --oroot %s --eigenvectors %d --maxImages %d' % (
imagesMd, fnRoot, self.maxPCABases.get(), self.maxClasses.get())
self.runJob("xmipp_image_rotational_pca", args)
N = min(self.maxPCABases.get(), self.PCABases2Project.get())
args='-i %s -o %s.stk --save_metadata_stack %s.xmd --basis %s.stk %d'\
% (imagesMd, fnRootDenoised, fnRootDenoised, fnRoot, N)
self.runJob("xmipp_transform_filter", args)
self.outputMd = String('%s.stk' % fnRootDenoised)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._updateLocation,
itemDataIterator=md.iterRows(
self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
summary.append('PCA basis created by using %d classes' %
len(self.inputClasses.get()))
summary.append(
'Max. number of classes defined for PCA basis creation: %d' %
self.maxClasses.get())
summary.append(
'Max. number of PCA bases defined for PCA basis creation: %d' %
self.maxPCABases.get())
summary.append('PCA basis on which to project for denoising: %d' %
self.PCABases2Project.get())
return summary
def _validate(self):
pass
def _citations(self):
return ['zhao2013', 'ponce2011']
def _methods(self):
methods = []
if not hasattr(self, 'outputParticles'):
methods.append("Output particles not ready yet.")
else:
methods.append('An input dataset of %d particles was filtered creating a PCA basis (%d components) with '
'xmipp_image_rotational_pca and projecting the dataset into that base with xmipp_transform_filter.'\
% (len(self.inputParticles.get()), len(self.inputClasses.get())))
return methods
#--------------------------- UTILS functions --------------------------------------------
def _updateLocation(self, item, row):
index, filename = xmippToLocation(row.getValue(md.MDL_IMAGE))
item.setLocation(index, filename)
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
INPUT_HELP = """ Input volume(s) for ResMap.
Required volume properties:
1. The particle must be centered in the volume.
2. The background must not been masked out.
Desired volume properties:
1. The volume has not been filtered in any way (low-pass filtering, etc.)
2. The volume has a realistic noise spectrum.
This is sometimes obtained by so-called amplitude correction.
While a similar effect is often obtained by B-factor sharpening,
please make sure that the spectrum does not blow up near Nyquist.
"""
@classmethod
def validateInstallation(cls):
""" This function will be used to check if package is properly installed."""
missingPaths = ["%s: %s" % (var, os.environ[var])
for var in [RESMAP_HOME]
if not os.path.exists(os.environ[var])]
if missingPaths:
return ["Missing variables:"] + missingPaths
else:
return [] # No errors
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
self.plotData = String() # store some values for later plot
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('useSplitVolume', params.BooleanParam, default=False,
label="Use half volumes?",
help='Use split volumes for gold-standard FSC.')
form.addParam('inputVolume', params.PointerParam,
pointerClass='Volume', condition="not useSplitVolume",
label="Input volume", important=True,
help=self.INPUT_HELP)
form.addParam('volumeHalf1', params.PointerParam,
label="Volume half 1", important=True,
pointerClass='Volume', condition="useSplitVolume",
help=self.INPUT_HELP)
form.addParam('volumeHalf2', params.PointerParam,
pointerClass='Volume', condition="useSplitVolume",
label="Volume half 2", important=True,
help=self.INPUT_HELP)
form.addParam('applyMask', params.BooleanParam, default=False,
label="Mask input volume?",
help="It is not necessary to provide ResMap with a mask "
"volume. The algorithm will attempt to estimate a "
"mask volume by low-pass filtering the input volume "
"and thresholding it using a heuristic procedure.\n"
"If the automated procedure does not work well for "
"your particle, you may provide a mask volume that "
"matches the input volume in size and format. "
"The mask volume should be a binary volume with zero "
"(0) denoting the background/solvent and some positive"
"value (0+) enveloping the particle.")
form.addParam('maskVolume', params.PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
form.addParam('whiteningLabel', params.LabelParam, important=True,
label="It is strongly recommended to use the "
"pre-whitening wizard.")
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', params.FloatParam, default=10,
label="Angstroms")
line.addParam('prewhitenRamp', params.FloatParam, default=1,
label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', params.FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help="Default (0): algorithm will start a just above\n"
" 2*voxelSize until 4*voxelSize. \n"
"These fields are provided to accelerate computation "
"if you are only interested in analyzing a specific "
"resolution range. It is usually a good idea to provide "
"a maximum resolution value to save time. Another way to "
"save computation is to provide a larger step size.")
line.addParam('minRes', params.FloatParam, default=0, label='Min')
line.addParam('maxRes', params.FloatParam, default=0, label='Max')
group.addParam('pVal', params.FloatParam, default=0.05,
label='Confidence level:',
help="P-value, usually between [0.01, 0.05].\n\n"
"This is the p-value of the statistical hypothesis test "
"on which ResMap is based on. It is customarily set to "
"0.05 although you are welcome to reduce it (e.g. 0.01) "
"if you would like to obtain a more conservative result. "
"Empirically, ResMap results are not much affected by the p-value.")
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
if self.useSplitVolume:
inputs = [self.volumeHalf1, self.volumeHalf2]
self.inputVolume.set(None)
else:
inputs = [self.inputVolume]
self.volumeHalf1.set(None)
self.volumeHalf2.set(None)
locations = [i.get().getLocation() for i in inputs]
self._insertFunctionStep('convertInputStep', *locations)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropriate parameters. """
results = self.runResmap(self._getPath())
self.histogramData.set(dumps(results['resHisto']))
plotDict = {'minRes': results['minRes'],
'maxRes': results['maxRes'],
'orig_n': results['orig_n'],
'n': results['n'],
'currentRes': results['currentRes']
}
self.plotData.set(dumps(plotDict))
self._store(self.histogramData, self.plotData)
self.savePlots(results)
def savePlots(self, results=None):
""" Store png images of the plots to be used as images, """
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
plot = self._plotVolumeSlices()
plot.savefig(self._getExtraPath('volume1.map.png'))
plot.close()
plot = self._plotResMapSlices(results['resTOTALma'])
plot.savefig(self._getExtraPath('volume1_resmap.map.png'))
plot.close()
plot = self._plotHistogram()
plot.savefig(self._getExtraPath('histogram.png'))
plot.close()
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
if exists(self._getExtraPath('histogram.png')):
results = self._parseOutput()
summary.append('Mean resolution: %0.2f A' % results[0])
summary.append('Median resolution: %0.2f A' % results[1])
else:
summary.append("Output is not ready yet.")
return summary
def _validate(self):
errors = []
if self.useSplitVolume:
half1 = self.volumeHalf1.get()
half2 = self.volumeHalf2.get()
if half1.getSamplingRate() != half2.getSamplingRate():
errors.append('The selected half volumes have not the same pixel size.')
if half1.getXDim() != half2.getXDim():
errors.append('The selected half volumes have not the same dimensions.')
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ[RESMAP_HOME])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'vxSize': self.volumeHalf1.get().getSamplingRate(),
'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'vxSize': self.inputVolume.get().getSamplingRate(),
'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self, **kwargs):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'), **kwargs)
return Plotter(figure=fig)
def _plotResMapSlices(self, data=None, **kwargs):
from ResMap_visualization import plotResMapVolume
plotDict = loads(self.plotData.get())
if data is None:
data = self._getVolumeMatrix('volume1_resmap.map')
data = np.ma.masked_where(data > plotDict['currentRes'], data, copy=True)
kwargs.update(plotDict)
fig = plotResMapVolume(data, **kwargs)
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
def _parseOutput(self):
meanRes, medianRes = 0, 0
f = open(self.getLogPaths()[0], 'r')
for line in f.readlines():
if 'MEAN RESOLUTION in MASK' in line:
meanRes = line.strip().split('=')[1]
elif 'MEDIAN RESOLUTION in MASK' in line:
medianRes = line.strip().split('=')[1]
f.close()
return tuple(map(float, (meanRes, medianRes)))
class XmippProtAngBreakSymmetry(ProtProcessParticles):
"""
Given an input set of particles with angular assignment, find an
equivalent angular assignment for a given symmetry.
Be aware that input symmetry values follows Xmipp conventions as described in:
http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry
"""
_label = 'break symmetry'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
form.addParam('symmetryGroup', StringParam, default="c1",
label='Symmetry group',
help="See http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry"
" for a description of the symmetry groups format in Xmipp.\n"
"If no symmetry is present, use _c1_.")
def _getDefaultParallel(self):
"""This protocol doesn't have mpi version"""
return (0, 0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Mainly prepare the command line for call brak symmetry program"""
# Create a metadata with the geometrical information
# as expected by Xmipp
imgsFn = self._getPath('input_particles.xmd')
self._insertFunctionStep('convertInputStep', imgsFn)
self._insertFunctionStep('breakSymmetryStep', imgsFn)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, outputFn):
""" Create a metadata with the images and geometrical information. """
writeSetOfParticles(self.inputParticles.get(), outputFn)
#--------------------------- STEPS functions --------------------------------------------
def breakSymmetryStep(self, imgsFn):
outImagesMd = self._getPath('images.xmd')
args = "-i Particles@%s --sym %s -o %s" % (imgsFn,
self.symmetryGroup.get(),
outImagesMd )
self.runJob("xmipp_angular_break_symmetry", args)
self.outputMd = String(outImagesMd)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
import os
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
summary.append("Symmetry: %s"% self.symmetryGroup.get())
return summary
def _validate(self):
pass
def _citations(self):
return []#['Vargas2013b']
def _methods(self):
methods = []
# if hasattr(self, 'outputParticles'):
# outParticles = len(self.outputParticles) if self.outputParticles is not None else None
# particlesRejected = len(self.inputParticles.get())-outParticles if outParticles is not None else None
# particlesRejectedText = ' ('+str(particlesRejected)+')' if particlesRejected is not None else ''
# rejectionText = [
# '',# REJ_NONE
# ' and removing those not reaching %s%s' % (str(self.maxZscore.get()), particlesRejectedText),# REJ_MAXZSCORE
# ' and removing worst %s percent%s' % (str(self.percentage.get()), particlesRejectedText)# REJ_PERCENTAGE
# ]
# methods.append('Input dataset %s of %s particles was sorted by'
# ' its ZScore using xmipp_image_sort_by_statistics'
# ' program%s. ' % (self.getObjectTag('inputParticles'), len(self.inputParticles.get()), rejectionText[self.autoParRejection.get()]))
# methods.append('Output set is %s.'%self.getObjectTag('outputParticles'))
return methods
#--------------------------- Utils functions --------------------------------------------
def _createItemMatrix(self, item, row):
from pyworkflow.em.packages.xmipp3.convert import createItemMatrix
import pyworkflow.em as em
createItemMatrix(item, row, align=em.ALIGN_PROJ)
class XmippProtParticlePicking(ProtParticlePicking, XmippProtocol):
""" Picks particles in a set of micrographs
either manually or in a supervised mode.
"""
_label = 'manual-picking (step 1)'
def __init__(self, **args):
ProtParticlePicking.__init__(self, **args)
# The following attribute is only for testing
self.importFolder = String(args.get('importFolder', None))
#--------------------------- DEFINE param functions ------------------------
def _defineParams(self, form):
ProtParticlePicking._defineParams(self, form)
form.addParam('saveDiscarded', BooleanParam, default=False,
label='Save discarded particles',
help='Generates an output with '
'the manually discarded particles.')
form.addParam('doInteractive', BooleanParam, default=True,
label='Run in interactive mode',
expertLevel=LEVEL_ADVANCED,
help='If YES, you can pick particles in differents sessions.\n'
'If NO, once an outputCoordinates is created, '
'the protocol finishes. \n'
'(the last can be useful when other protocol '
'waits until this finish -internal scheduled-)')
#--------------------------- INSERT steps functions ------------------------
def _insertAllSteps(self):
"""The Particle Picking process is realized for a set of micrographs"""
# Get pointer to input micrographs
self.inputMics = self.inputMicrographs.get()
micFn = self.inputMics.getFileName()
# Launch Particle Picking GUI
if not self.importFolder.hasValue():
self._insertFunctionStep('launchParticlePickGUIStep', micFn,
interactive=self.doInteractive)
else: # This is only used for test purposes
self._insertFunctionStep('_importFromFolderStep')
# Insert step to create output objects
self._insertFunctionStep('createOutputStep')
def launchParticlePickGUIStep(self, micFn):
# Launch the particle picking GUI
extraDir = self._getExtraPath()
process = launchSupervisedPickerGUI(micFn, extraDir, self)
process.wait()
# generate the discarded output only if there is a good output
if self.saveDiscarded and exists(self._getPath('coordinates.sqlite')):
self.createDiscardedStep()
coordSet = self.getCoords()
if coordSet:
boxSize = Integer(coordSet.getBoxSize())
self._defineOutputs(boxsize=boxSize)
self._defineSourceRelation(self.inputMicrographs.get(), boxSize)
def _importFromFolderStep(self):
""" This function will copy Xmipp .pos files for
simulating a particle picking run...this is only
for testing purposes.
"""
for f in getFiles(self.importFolder.get()):
copyFile(f, self._getExtraPath())
def createOutputStep(self):
posDir = self._getExtraPath()
coordSet = self._createSetOfCoordinates(self.inputMics)
readSetOfCoordinates(posDir, self.inputMics, coordSet)
self._defineOutputs(outputCoordinates=coordSet)
self._defineSourceRelation(self.inputMicrographs, coordSet)
boxSize = Integer(coordSet.getBoxSize())
self._defineOutputs(boxsize=boxSize)
self._defineSourceRelation(self.inputMicrographs.get(), boxSize)
def createDiscardedStep(self):
posDir = self._getExtraPath()
suffixRoot = self._ProtParticlePicking__getOutputSuffix()
suffix = '' if suffixRoot=='2' or suffixRoot=='' \
else str(int(suffixRoot)-1)
coordSetDisc = self._createSetOfCoordinates(self.inputMics,
suffix='Discarded'+suffix)
readSetOfCoordinates(posDir, self.inputMics, coordSetDisc,
readDiscarded=True)
if coordSetDisc.getSize()>0:
outputName = 'outputDiscardedCoordinates' + suffix
outputs = {outputName: coordSetDisc}
self._defineOutputs(**outputs)
self._defineSourceRelation(self.inputMicrographs, coordSetDisc)
#--------------------------- INFO functions --------------------------------
def _citations(self):
return ['Abrishami2013']
#--------------------------- UTILS functions -------------------------------
def __str__(self):
""" String representation of a Supervised Picking run """
if not hasattr(self, 'outputCoordinates'):
msg = "No particles picked yet."
else:
picked = 0
# Get the number of picked particles of the last coordinates set
for key, output in self.iterOutputAttributes(EMObject):
picked = output.getSize()
msg = "%d particles picked (from %d micrographs)" % \
(picked, self.inputMicrographs.get().getSize())
return msg
def _methods(self):
if self.getOutputsSize() > 0:
return ProtParticlePicking._methods(self)
else:
return [self._getTmpMethods()]
def _getTmpMethods(self):
""" Return the message when there is not output generated yet.
We will read the Xmipp .pos files and other configuration files.
"""
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
msg = ''
if existsConfig:
md = emlib.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(emlib.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(emlib.MDL_PICKING_PARTICLE_SIZE, configobj)
isAutopick = pickingState != "Manual"
manualParts = md.getValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParts = md.getValue(emlib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj)
if manualParts is None:
manualParts = 0
if autoParts is None:
autoParts = 0
msg = 'User picked %d particles ' % (autoParts + manualParts)
msg += 'with a particle size of %d.' % particleSize
if isAutopick:
msg += "Automatic picking was used ([Abrishami2013]). "
msg += "%d particles were picked automatically " % autoParts
msg += "and %d manually." % manualParts
return msg
def _summary(self):
if self.getOutputsSize() > 0:
return ProtParticlePicking._summary(self)
else:
return [self._getTmpSummary()]
def _getTmpSummary(self):
summary = []
configfile = join(self._getExtraPath(), 'config.xmd')
existsConfig = exists(configfile)
if existsConfig:
md = emlib.MetaData('properties@' + configfile)
configobj = md.firstObject()
pickingState = md.getValue(emlib.MDL_PICKING_STATE, configobj)
particleSize = md.getValue(emlib.MDL_PICKING_PARTICLE_SIZE, configobj)
activeMic = md.getValue(emlib.MDL_MICROGRAPH, configobj)
isAutopick = pickingState != "Manual"
manualParticlesSize = md.getValue(emlib.MDL_PICKING_MANUALPARTICLES_SIZE, configobj)
autoParticlesSize = md.getValue(emlib.MDL_PICKING_AUTOPARTICLES_SIZE, configobj)
summary.append("Manual particles picked: %d"%manualParticlesSize)
summary.append("Particle size:%d" %(particleSize))
autopick = "Yes" if isAutopick else "No"
summary.append("Autopick: " + autopick)
if isAutopick:
summary.append("Automatic particles picked: %d"%autoParticlesSize)
summary.append("Last micrograph: " + activeMic)
return "\n".join(summary)
def getCoordsDir(self):
return self._getExtraPath()
class EmpiarDepositor(EMProtocol):
"""
Deposit image sets to empiar
"""
_label = 'Empiar deposition'
_ih = emlib.image.ImageHandler()
_imageSetCategories = {
"SetOfMicrographs": "T1",
"SetOfMovies": 'T2',
# 'T3' : 'micrographs - focal pairs - unprocessed',
# 'T4' : 'micrographs - focal pairs - contrast inverted',
"SetOfMovieParticles":
'T5', # : 'picked particles - single frame - unprocessed',
# 'T6' : 'picked particles - multiframe - unprocessed',
"SetOfParticles":
'T7', # 'picked particles - single frame - processed',
# "SetOfMovieParticles": 'T8', # : 'picked particles - multiframe - processed',
"TiltPairSet": 'T9', # : 'tilt series',
"SetOfAverages": 'T10', # 'class averages',
# 'OT' : 'other, in this case please specify the category in the second element.'
}
_imageSetFormats = {
'mrc': 'T1',
'mrcs': 'T2',
'tiff': 'T3',
'img': 'T4', # imagic
'dm3': 'T5',
'dm4': 'T6',
'spi': 'T7', # spider
}
_experimentTypes = ['1', '2', '3', '4', '5', '6', '7', '8', '9']
_releaseDateTypes = ["RE", "EP", "HP", "HO"]
_countryCodes = [
'AD', 'AE', 'AF', 'AG', 'AI', 'AL', 'AM', 'AO', 'AQ', 'AR', 'AS', 'AT',
'AU', 'AW', 'AZ', 'BA', 'BB', 'BD', 'BE', 'BF', 'BG', 'BH', 'BI', 'BJ',
'BL', 'BM', 'BN', 'BO', 'BR', 'BS', 'BT', 'BV', 'BW', 'BY', 'BZ', 'CA',
'CC', 'CD', 'CF', 'CG', 'CH', 'CI', 'CK', 'CL', 'CM', 'CN', 'CO', 'CR',
'CU', 'CV', 'CW', 'CX', 'CY', 'CZ', 'DE', 'DJ', 'DK', 'DM', 'DO', 'DZ',
'EC', 'EE', 'EG', 'EH', 'ER', 'ES', 'ET', 'FI', 'FJ', 'FK', 'FM', 'FO',
'FR', 'FX', 'GA', 'GB', 'GD', 'GE', 'GF', 'GG', 'GH', 'GI', 'GL', 'GM',
'GN', 'GP', 'GQ', 'GR', 'GS', 'GT', 'GU', 'GW', 'GY', 'HK', 'HM', 'HN',
'HR', 'HT', 'HU', 'ID', 'IE', 'IL', 'IM', 'IN', 'IO', 'IQ', 'IR', 'IS',
'IT', 'JE', 'JM', 'JO', 'JP', 'KE', 'KG', 'KH', 'KI', 'KM', 'KN', 'KP',
'KR', 'KW', 'KY', 'KZ', 'LA', 'LB', 'LC', 'LI', 'LK', 'LR', 'LS', 'LT',
'LU', 'LV', 'LY', 'MA', 'MC', 'MD', 'ME', 'MF', 'MG', 'MH', 'MK', 'ML',
'MM', 'MN', 'MO', 'MP', 'MQ', 'MR', 'MS', 'MT', 'MU', 'MV', 'MW', 'MX',
'MY', 'MZ', 'NA', 'NC', 'NE', 'NF', 'NG', 'NI', 'NL', 'NO', 'NP', 'NR',
'NU', 'NZ', 'OM', 'PA', 'PE', 'PF', 'PG', 'PH', 'PK', 'PL', 'PM', 'PN',
'PR', 'PS', 'PT', 'PW', 'PY', 'QA', 'RE', 'RO', 'RS', 'RU', 'RW', 'SA',
'SB', 'SC', 'SD', 'SE', 'SG', 'SH', 'SI', 'SJ', 'SK', 'SL', 'SM', 'SN',
'SO', 'SR', 'SS', 'ST', 'SV', 'SX', 'SY', 'SZ', 'TC', 'TD', 'TF', 'TG',
'TH', 'TJ', 'TK', 'TL', 'TM', 'TN', 'TO', 'TR', 'TT', 'TV', 'TW', 'TZ',
'UA', 'UG', 'UM', 'US', 'UY', 'UZ', 'VA', 'VC', 'VE', 'VG', 'VI', 'VN',
'VU', 'WF', 'WS', 'XK', 'YE', 'YT', 'ZA', 'ZM', 'ZW'
]
_voxelTypes = {
emlib.DT_UCHAR: 'T1', # 'UNSIGNED BYTE'
emlib.DT_SCHAR: 'T2', # 'SIGNED BYTE'
emlib.DT_USHORT: 'T3', # 'UNSIGNED 16 BIT INTEGER'
emlib.DT_SHORT: 'T4', # 'SIGNED 16 BIT INTEGER'
emlib.DT_UINT: 'T5', # 'UNSIGNED 32 BIT INTEGER'
emlib.DT_INT: 'T6', # 'SIGNED 32 BIT INTEGER'
emlib.DT_FLOAT: 'T7' # '32 BIT FLOAT'
}
OUTPUT_DEPO_JSON = 'deposition.json'
OUTPUT_WORKFLOW = 'workflow.json'
IMGSET_KEY = 'imagesets'
IMGSET_NAME = "name"
IMGSET_DIR = "directory"
IMGSET_CAT = "category"
IMGSET_HEADER_FORMAT = "header_format"
IMGSET_DATA_FORMAT = "data_format"
IMGSET_SIZE = "num_images_or_tilt_series"
IMGSET_FRAMES = "frames_per_image"
IMGSET_FRAME_MIN = "frame_range_min"
IMGSET_FRAME_MAX = "frame_range_max"
IMGSET_VOXEL_TYPE = "voxel_type"
IMGSET_PIXEL_WIDTH = "pixel_width"
IMGSET_PIXEL_HEIGHT = "pixel_height"
IMGSET_DETAILS = "details"
IMGSET_WIDTH = "image_width"
IMGSET_HEIGHT = "image_height"
_imageSetTemplate = {
IMGSET_NAME: "",
IMGSET_DIR: "/data/%s",
IMGSET_CAT: "('%s', '%s')",
IMGSET_HEADER_FORMAT: "('%s', '%s')",
IMGSET_DATA_FORMAT: "('%s', '%s')",
IMGSET_SIZE: 0,
IMGSET_FRAMES: 0,
IMGSET_FRAME_MIN: None,
IMGSET_FRAME_MAX: None,
IMGSET_VOXEL_TYPE: "('%s', '%s')",
IMGSET_PIXEL_WIDTH: None,
IMGSET_PIXEL_HEIGHT: None,
IMGSET_DETAILS: "",
IMGSET_WIDTH: 0,
IMGSET_HEIGHT: 0
}
def __init__(self, **kwargs):
EMProtocol.__init__(self, **kwargs)
self.workflowDicts = []
self.entryAuthorStr = ""
self.workflowPath = String()
self.depositionJsonPath = String()
# --------------- DEFINE param functions ---------------
def _defineParams(self, form):
form.addSection(label='Entry')
# form.addParam('workflowJson', params.PathParam,
# label='Workflow json', allowsNull=True,
# help='Path to the workflow json (obtained using the export option (right click on'
# 'one of your selected protocols). Will generate json of all protocols if not provided.')
form.addParam(
"submit",
params.BooleanParam,
label="Submit deposition",
default=True,
help="Set to false to avoid submitting the deposition to empiar "
"(it will just be created locally).")
form.addParam("resume",
params.BooleanParam,
label="Resume upload",
default=False,
condition='submit',
help="Is this a continuation of a previous upload?")
form.addParam(
'entryID',
params.StringParam,
label="Entry ID",
condition="resume",
important=True,
help="EMPIAR entry ID - use if you wanna resume an upload")
form.addParam('uniqueDir',
params.StringParam,
important=True,
label="Unique directory",
condition="resume",
help="EMPIAR directory assigned to this deposition ID")
form.addParam(
'depositionJson',
params.PathParam,
important=True,
label="Deposition json",
condition="resume",
help=
"Path to the json file of the deposition we're about to resume.")
form.addParam(
'jsonTemplate',
params.PathParam,
condition='not resume',
label="Custom json (Optional)",
allowsNull=True,
help=
"Path to a customized template of the EMPIAR submission json, if you don't want to use the "
"default one.")
form.addParam(
'entryTopLevel',
params.StringParam,
label="Top level folder",
validators=[params.NonEmpty],
important=True,
help=
"How you want to name the top level folder of the empiar entry. \n This should be a "
"simple and descriptive name without special characters (:,?, spaces, etc). \n"
"If you're resuming an upload, this should be the same name you used to create the folder."
)
form.addParam(
'entryTitle',
params.StringParam,
label="Entry title",
important=True,
condition="not resume",
help=
"EMPIAR entry title. This should not be empty if not using a custom template."
)
form.addParam(
'entryAuthor',
params.StringParam,
label="Entry author",
important=True,
condition="not resume",
help=
'EMPIAR entry author in the form "LastName, Initials" e.g. Smith, JW\n'
'This should not be empty if not using a custom template.')
form.addParam(
'experimentType',
params.EnumParam,
label="Experiment type",
condition="not resume",
choices=self._experimentTypes,
default=2,
important=True,
help="EMPIAR experiment type:\n"
"1 - image data collected using soft x-ray tomography\n"
"2 - simulated data, for instance, created using InSilicoTEM\n"
" (note: simulated data accepted in special circumstances such\n"
" as test/training sets for validation challenges: you need to\n"
" ask for and be granted permission PRIOR to deposition otherwise\n"
" the dataset will be rejected by EMPIAR)\n"
"3 - raw image data relating to structures deposited to the Electron Microscopy Data Bank\n"
"4 - image data collected using serial block-face scanning electron microscopy \n"
" (like the Gatan 3View system)\n"
"5 - image data collected using focused ion beam scanning electron microscopy\n"
"6 - integrative hybrid modelling data\n"
"7 - correlative light-electron microscopy\n"
"8 - correlative light X-ray microscopy\n"
"9 - microcrystal electron diffraction")
form.addParam(
'releaseDate',
params.EnumParam,
label="Release date",
condition="not resume",
choices=self._releaseDateTypes,
default=0,
important=True,
help="EMPIAR release date:\n"
"Options for releasing entry to the public: \n"
"RE - directly after the submission has been processed\n"
"EP - after the related EMDB entry has been released\n"
"HP - after the related primary citation has been published\n"
"HO - delay release of entry by one year from the date of deposition"
)
form.addSection(label='Image sets')
self.inputSetsParam = form.addParam(
'inputSets',
params.MultiPointerParam,
label="Input set",
important=True,
condition="not resume",
pointerClass=','.join(self._imageSetCategories.keys()),
minNumObjects=1,
help='Select one set (of micrographs, particles,'
' volumes, etc.) to be deposited to EMPIAR.')
# form.addParam('micSet', params.PointerParam, pointerClass='SetOfMicrographs,SetOfMovies,SetOfParticles',
# label='Image set', important=False,
# help='Image set to be uploaded to EMPIAR\n')
form.addSection(label="Principal investigator")
form.addParam(
'piFirstName',
params.StringParam,
label='First name',
condition="not resume",
help=
"PI first name e.g. Juan- this should not be empty if not using a custom template."
)
form.addParam(
'piLastName',
params.StringParam,
label='Last name',
condition="not resume",
help=
'PI Last name e.g. Perez - this should not be empty if not using a custom template.'
)
form.addParam(
'piOrg',
params.StringParam,
label='Organization',
condition="not resume",
help=
"The name of the organization e.g. Biocomputing Unit, CNB-CSIC \n"
"This should not be empty if not using a custom template.")
form.addParam(
'piEmail',
params.StringParam,
label="Email",
condition="not resume",
help='PI Email address e.g. [email protected] - '
'this should not be empty if not using a custom template.')
form.addParam(
'piPost',
params.StringParam,
label="Post or zip",
condition="not resume",
help=
"Post or ZIP code. This should not be empty if not using a custom template."
)
form.addParam(
'piTown',
params.StringParam,
label="Town or city",
condition="not resume",
help=
"Town or city name. This should not be empty if not using a custom template."
)
form.addParam(
'piCountry',
params.StringParam,
label="Country",
condition="not resume",
help=
"Two letter country code eg. ES. This should not be empty if not using a custom template."
"\nValid country codes are %s" % " ".join(self._countryCodes))
form.addSection(label="Corresponding Author")
form.addParam(
'caFirstName',
params.StringParam,
label='First name',
condition="not resume",
help="Corresponding author's first name e.g. Juan. "
"This should not be empty if not using a custom template. ")
form.addParam(
'caLastName',
params.StringParam,
label='Last name',
condition="not resume",
help="Corresponding author's Last name e.g. Perez. "
"This should not be empty if not using a custom template.")
form.addParam(
'caOrg',
params.StringParam,
label='Organization',
condition="not resume",
help="The name of the organization e.g. Biocomputing Unit, CNB-CSIC."
"This should not be empty if not using a custom template.")
form.addParam(
'caEmail',
params.StringParam,
label="Email",
condition="not resume",
help="Corresponding author's Email address e.g. [email protected]. "
"This should not be empty if not using a custom template.")
form.addParam(
'caPost',
params.StringParam,
label="Post or zip",
condition="not resume",
help=
"Post or ZIP code. This should not be empty if not using a custom template."
)
form.addParam(
'caTown',
params.StringParam,
label="Town or city",
condition="not resume",
help=
"Town or city name. This should not be empty if not using a custom template."
)
form.addParam(
'caCountry',
params.StringParam,
label="Country",
condition="not resume",
help=
"Two letter country code e.g. ES. This should not be empty if not using a custom template."
"\nValid country codes are %s" % " ".join(self._countryCodes))
# --------------- INSERT steps functions ----------------
def _insertAllSteps(self):
self._insertFunctionStep('createDepositionStep')
if self.submit:
self._insertFunctionStep('submitDepositionStep')
# --------------- STEPS functions -----------------------
def createDepositionStep(self):
# make folder in extra
if not self.resume:
pwutils.makePath(self._getExtraPath(self.entryTopLevel.get()))
# export workflow json
self.exportWorkflow()
# create deposition json
jsonTemplatePath = self.jsonTemplate.get(
'').strip() or DEPOSITION_TEMPLATE
entryAuthorStr = self.entryAuthor.get().split(',')
self.entryAuthorStr = "'%s', '%s'" % (entryAuthorStr[0].strip(),
entryAuthorStr[1].strip())
self.releaseDate = self.getEnumText('releaseDate')
self.experimentType = self.experimentType.get() + 1
jsonStr = open(jsonTemplatePath, 'rb').read().decode('utf-8')
jsonStr = jsonStr % self.__dict__
depoDict = json.loads(jsonStr)
imageSets = self.processImageSets()
depoDict[self.IMGSET_KEY] = imageSets
depoJson = self.getTopLevelPath(self.OUTPUT_DEPO_JSON)
with open(depoJson, 'w') as f:
# f.write(jsonStr.encode('utf-8'))
json.dump(depoDict, f, indent=4)
# self.depositionJsonPath = depoJson
self.depositionJsonPath.set(depoJson)
else:
self.depositionJsonPath.set(self.depositionJson.get())
with open(self.depositionJson.get()) as f:
depoDict = json.load(f)
self._store()
self.validateDepoJson(depoDict)
def submitDepositionStep(self):
depositorCall = '%(resume)s -a %(ascp)s %(token)s %(depoJson)s %(dataDir)s'
args = {
'resume':
'-r %s %s' % (self.entryID, self.uniqueDir) if self.resume else "",
'token':
os.environ[EMPIAR_TOKEN],
'depoJson':
os.path.abspath(self.depositionJsonPath.get()),
'ascp':
os.environ[ASCP_PATH],
'dataDir':
os.path.abspath(self.getTopLevelPath())
}
depositorCall = depositorCall % args
print("Empiar depositor call: %s" % depositorCall)
empiar_depositor.main(depositorCall.split())
# --------------- INFO functions -------------------------
def _validate(self):
errors = []
if self.submit:
if EMPIAR_TOKEN not in os.environ:
errors.append(
"Environment variable %s not set. Please set your %s in ~/.config/scipion/scipion.conf "
"or in your environment." % (EMPIAR_TOKEN, EMPIAR_TOKEN))
if ASPERA_PASS not in os.environ:
errors.append(
"Environment variable %s not set. Please set your %s in ~/.config/scipion/scipion.conf "
"or in your environment." % (ASPERA_PASS, ASPERA_PASS))
return errors
def _citations(self):
return ['Iudin2016']
def _summary(self):
summary = []
if self.depositionJsonPath.get():
summary.append('Generated deposition files:')
summary.append('- [[%s][Scipion workflow]]' % self.workflowPath)
summary.append('- [[%s][Deposition json]]' %
self.depositionJsonPath)
else:
summary.append('No deposition files generated yet')
return summary
def _methods(self):
return []
# -------------------- UTILS functions -------------------------
def getTopLevelPath(self, *paths):
return os.path.join(self._getExtraPath(self.entryTopLevel.get()),
*paths)
def exportWorkflow(self):
project = self.getProject()
workflowProts = [p for p in project.getRuns()
] # workflow prots are all prots if no json provided
workflowJsonPath = os.path.join(
project.path, self.getTopLevelPath(self.OUTPUT_WORKFLOW))
protDicts = project.getProtocolsDict(workflowProts)
for inputSetPointer in self.inputSets:
inputSet = inputSetPointer.get()
setName = inputSet.getObjName()
setParentId = inputSet.getObjParentId()
setParentObj = project.getObject(setParentId)
protDicts[setParentId]['filesPath'] = os.path.join('.', setName)
pwutils.createLink(setParentObj._getExtraPath(),
self.getTopLevelPath(setName))
with open(workflowJsonPath, 'w') as f:
f.write(
json.dumps(list(protDicts.values()),
indent=4,
separators=(',', ': ')))
self.workflowPath.set(workflowJsonPath)
return workflowJsonPath
def validateDepoJson(self, depoDict):
with open(DEPOSITION_SCHEMA) as f:
schema = json.load(f)
valid = jsonschema.validate(depoDict,
schema) # raises exception if not valid
return True
# --------------- imageSet utils -------------------------
def getEmpiarCategory(self, imageSet):
className = imageSet.getClassName()
category = self._imageSetCategories.get(className, None)
if category is None:
raise EmpiarMappingError(
'Could not assign an EMPIAR category to image set %s' %
imageSet.getObjName())
else:
return category, ''
def getEmpiarFormat(self, imagePath):
ext = pwutils.getExt(imagePath).lower().strip('.')
imgFormat = self._imageSetFormats.get(ext, None)
if imgFormat is None:
raise EmpiarMappingError('Image format not recognized: %s' % ext)
else:
return imgFormat, ''
def getVoxelType(self, imageObj):
dataType = self._ih.getDataType(imageObj)
empiarType = self._voxelTypes.get(dataType, None)
if empiarType is None:
raise EmpiarMappingError('Could not map voxel type for image %s' %
imageObj.getFilename())
else:
return empiarType, ''
def getImageSetDict(self, imageSet):
firstImg = imageSet.getFirstItem()
firstFileName = firstImg.getFileName()
dims = imageSet.getDimensions()
micSetDict = copy.deepcopy(self._imageSetTemplate)
micSetDict[self.IMGSET_NAME] = imageSet.getObjName()
micSetDict[self.IMGSET_DIR] = "/data/%s" % imageSet.getObjName()
micSetDict[self.IMGSET_CAT] = "('%s', '%s')" % self.getEmpiarCategory(
imageSet)
micSetDict[
self.IMGSET_HEADER_FORMAT] = "('%s', '%s')" % self.getEmpiarFormat(
firstFileName)
micSetDict[
self.IMGSET_DATA_FORMAT] = "('%s', '%s')" % self.getEmpiarFormat(
firstFileName)
micSetDict[self.IMGSET_SIZE] = len(imageSet)
micSetDict[self.IMGSET_FRAMES] = dims[2]
micSetDict[
self.
IMGSET_VOXEL_TYPE] = "('%s', '%s')" % self.getVoxelType(firstImg)
micSetDict[self.IMGSET_PIXEL_WIDTH] = imageSet.getSamplingRate()
micSetDict[self.IMGSET_PIXEL_HEIGHT] = imageSet.getSamplingRate()
micSetDict[self.IMGSET_DETAILS] = "/data/%s" % os.path.basename(
self.workflowPath.get())
micSetDict[self.IMGSET_WIDTH] = dims[0]
micSetDict[self.IMGSET_HEIGHT] = dims[1]
return micSetDict
def processImageSets(self):
inputSets = [s.get() for s in self.inputSets]
imgSetDicts = []
for imgSet in inputSets:
imgSetDict = self.getImageSetDict(imgSet)
imgSetDicts.append(imgSetDict)
return imgSetDicts
class QueueSystemConfig(Object):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.name = String()
# Number of cores from which the queue is mandatory
# 0 means no mandatory at all
# 1 will force to launch all jobs through the queue
self.mandatory = Integer()
self.queues = None # List for queue configurations
self.submitCommand = String()
# Allow to change the prefix of submission scripts
# we used by default the ID.job, but in some clusters
# the job script should start by a letter
self.submitPrefix = String()
self.checkCommand = String()
self.cancelCommand = String()
self.submitTemplate = String()
self.jobDoneRegex = String()
def hasName(self):
return self.name.hasValue()
def hasValue(self):
return self.hasName() and len(self.queues)
def getName(self):
return self.name.get()
def getMandatory(self):
return self.mandatory.get()
def getSubmitTemplate(self):
return self.submitTemplate.get()
def getSubmitCommand(self):
return self.submitCommand.get()
def getCheckCommand(self):
return self.checkCommand.get()
def getCancelCommand(self):
return self.cancelCommand.get()
def getQueues(self):
return self.queues
def setName(self, name):
self.name.set(name)
def setMandatory(self, mandatory):
# This condition is to be backward compatible
# when mandatory was a boolean
# now it should use the number of CPU
# that should force to use the queue
if mandatory in ['False', 'false']:
mandatory = 0
elif mandatory in ['True', 'true']:
mandatory = 1
self.mandatory.set(mandatory)
def setSubmitTemplate(self, submitTemplate):
self.submitTemplate.set(submitTemplate)
def setSubmitCommand(self, submitCommand):
self.submitCommand.set(submitCommand)
def setCheckCommand(self, checkCommand):
self.checkCommand.set(checkCommand)
def setCancelCommand(self, cancelCommand):
self.cancelCommand.set(cancelCommand)
def setJobDoneRegex(self, jobDoneRegex):
self.jobDoneRegex.set(jobDoneRegex)
def setQueues(self, queues):
self.queues = queues
def getQueueConfig(self, objId):
if objId is not None and self.queues is not None:
for queueConfig in self.queues:
if objId == queueConfig.getObjId():
return queueConfig
return None
class FlexProtDimredNMA(ProtAnalysis3D):
""" This protocol will take the images with NMA deformations
as points in a N-dimensional space (where N is the number
of computed normal modes) and will project them in a reduced
spaced (usually with less dimensions).
"""
_label = 'nma dimred'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.mappingFile = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputNMA',
PointerParam,
pointerClass='FlexProtAlignmentNMA',
label="Conformational distribution",
help='Select a previous run of the NMA alignment.')
form.addParam(
'dimredMethod',
EnumParam,
default=DIMRED_PCA,
choices=[
'Principal Component Analysis (PCA)',
'Local Tangent Space Alignment', 'Diffusion map',
'Linear Local Tangent Space Alignment',
'Linearity Preserving Projection', 'Kernel PCA',
'Probabilistic PCA', 'Laplacian Eigenmap',
'Hessian Locally Linear Embedding',
'Stochastic Proximity Embedding',
'Neighborhood Preserving Embedding'
],
label='Dimensionality reduction method',
help=""" Choose among the following dimensionality reduction methods:
PCA
Principal Component Analysis
LTSA <k=12>
Local Tangent Space Alignment, k=number of nearest neighbours
DM <s=1> <t=1>
Diffusion map, t=Markov random walk, s=kernel sigma
LLTSA <k=12>
Linear Local Tangent Space Alignment, k=number of nearest neighbours
LPP <k=12> <s=1>
Linearity Preserving Projection, k=number of nearest neighbours, s=kernel sigma
kPCA <s=1>
Kernel PCA, s=kernel sigma
pPCA <n=200>
Probabilistic PCA, n=number of iterations
LE <k=7> <s=1>
Laplacian Eigenmap, k=number of nearest neighbours, s=kernel sigma
HLLE <k=12>
Hessian Locally Linear Embedding, k=number of nearest neighbours
SPE <k=12> <global=1>
Stochastic Proximity Embedding, k=number of nearest neighbours, global embedding or not
NPE <k=12>
Neighborhood Preserving Embedding, k=number of nearest neighbours
""")
form.addParam('extraParams',
StringParam,
level=LEVEL_ADVANCED,
label="Extra params",
help='This parameters will be passed to the program.')
form.addParam('reducedDim',
IntParam,
default=2,
label='Reduced dimension')
form.addParallelSection(threads=0, mpi=0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Take deforamtions text file and the number of images and modes
inputSet = self.getInputParticles()
rows = inputSet.getSize()
reducedDim = self.reducedDim.get()
method = self.dimredMethod.get()
extraParams = self.extraParams.get('')
deformationsFile = self.getDeformationFile()
self._insertFunctionStep('convertInputStep', deformationsFile,
inputSet.getObjId())
self._insertFunctionStep('performDimredStep', deformationsFile, method,
extraParams, rows, reducedDim)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, deformationFile, inputId):
""" Iterate through the images and write the
plain deformation.txt file that will serve as
input for dimensionality reduction.
"""
inputSet = self.getInputParticles()
f = open(deformationFile, 'w')
for particle in inputSet:
f.write(' '.join(particle._xmipp_nmaDisplacements))
f.write('\n')
f.close()
def performDimredStep(self, deformationsFile, method, extraParams, rows,
reducedDim):
outputMatrix = self.getOutputMatrixFile()
methodName = DIMRED_VALUES[method]
# Get number of columes in deformation files
# it can be a subset of inputModes
f = open(deformationsFile)
columns = len(
f.readline().split()) # count number of values in first line
f.close()
args = "-i %(deformationsFile)s -o %(outputMatrix)s -m %(methodName)s %(extraParams)s"
args += "--din %(columns)d --samples %(rows)d --dout %(reducedDim)d"
if method in DIMRED_MAPPINGS:
mappingFile = self._getExtraPath('projector.txt')
args += " --saveMapping %(mappingFile)s"
self.mappingFile.set(mappingFile)
self.runJob("xmipp_matrix_dimred", args % locals())
def createOutputStep(self):
pass
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
def _citations(self):
return []
def _methods(self):
return []
#--------------------------- UTILS functions --------------------------------------------
def getInputParticles(self):
""" Get the output particles of the input NMA protocol. """
return self.inputNMA.get().outputParticles
def getInputPdb(self):
return self.inputNMA.get().getInputPdb()
def getOutputMatrixFile(self):
return self._getExtraPath('output_matrix.txt')
def getDeformationFile(self):
return self._getExtraPath('deformations.txt')
def getProjectorFile(self):
return self.mappingFile.get()
def getMethodName(self):
return DIMRED_VALUES[self.dimredMethod.get()]
class XmippProtAngBreakSymmetry(ProtProcessParticles):
""" Classify particles according their similarity to the others in order to detect outliers. """
_label = 'break symmetry'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
form.addParam('symmetryGroup', StringParam, default="c1",
label='Symmetry group',
help="See http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry"
" for a description of the symmetry groups format in Xmipp.\n"
"If no symmetry is present, use _c1_.")
def _getDefaultParallel(self):
"""This protocol doesn't have mpi version"""
return (0, 0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Mainly prepare the command line for call brak symmetry program"""
# Create a metadata with the geometrical information
# as expected by Xmipp
imgsFn = self._getPath('input_particles.xmd')
self._insertFunctionStep('convertInputStep', imgsFn)
self._insertFunctionStep('breakSymmetryStep', imgsFn)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, outputFn):
""" Create a metadata with the images and geometrical information. """
writeSetOfParticles(self.inputParticles.get(), outputFn)
#--------------------------- STEPS functions --------------------------------------------
def breakSymmetryStep(self, imgsFn):
outImagesMd = self._getPath('images.xmd')
args = "-i Particles@%s --sym %s -o %s" % (imgsFn,
self.symmetryGroup.get(),
outImagesMd )
self.runJob("xmipp_angular_break_symmetry", args)
self.outputMd = String(outImagesMd)
def createOutputStep(self):
imgSet = self._createSetOfParticles()
imgSet.copyInfo(self.inputParticles.get())
readSetOfParticles(self.outputMd.get(), imgSet)
self._defineOutputs(outputParticles=imgSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
import os
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
summary.append("Symmetry: %s"% self.symmetryGroup.get())
return summary
def _validate(self):
pass
def _citations(self):
return []#['Vargas2013b']
def _methods(self):
methods = []
# if hasattr(self, 'outputParticles'):
# outParticles = len(self.outputParticles) if self.outputParticles is not None else None
# particlesRejected = len(self.inputParticles.get())-outParticles if outParticles is not None else None
# particlesRejectedText = ' ('+str(particlesRejected)+')' if particlesRejected is not None else ''
# rejectionText = [
# '',# REJ_NONE
# ' and removing those not reaching %s%s' % (str(self.maxZscore.get()), particlesRejectedText),# REJ_MAXZSCORE
# ' and removing worst %s percent%s' % (str(self.percentage.get()), particlesRejectedText)# REJ_PERCENTAGE
# ]
# methods.append('Input dataset %s of %s particles was sorted by'
# ' its ZScore using xmipp_image_sort_by_statistics'
# ' program%s. ' % (self.getObjectTag('inputParticles'), len(self.inputParticles.get()), rejectionText[self.autoParRejection.get()]))
# methods.append('Output set is %s.'%self.getObjectTag('outputParticles'))
return methods
class XmippProtDimredNMA(ProtAnalysis3D):
""" This protocol will take the images with NMA deformations
as points in a N-dimensional space (where N is the number
of computed normal modes) and will project them in a reduced
spaced (usually with less dimensions).
"""
_label = 'nma dimred'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.mappingFile = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputNMA', PointerParam, pointerClass='XmippProtAlignmentNMA',
label="Conformational distribution",
help='Select a previous run of the NMA alignment.')
form.addParam('dimredMethod', EnumParam, default=DIMRED_PCA,
choices=['Principal Component Analysis (PCA)',
'Local Tangent Space Alignment',
'Diffusion map',
'Linear Local Tangent Space Alignment',
'Linearity Preserving Projection',
'Kernel PCA',
'Probabilistic PCA',
'Laplacian Eigenmap',
'Hessian Locally Linear Embedding',
'Stochastic Proximity Embedding',
'Neighborhood Preserving Embedding'],
label='Dim-Red method',
help=""" Dimensionality Reduction method.
PCA
Principal Component Analysis
LTSA <k=12>
Local Tangent Space Alignment, k=number of nearest neighbours
DM <s=1> <t=1>
Diffusion map, t=Markov random walk, s=kernel sigma
LLTSA <k=12>
Linear Local Tangent Space Alignment, k=number of nearest neighbours
LPP <k=12> <s=1>
Linearity Preserving Projection, k=number of nearest neighbours, s=kernel sigma
kPCA <s=1>
Kernel PCA, s=kernel sigma
pPCA <n=200>
Probabilistic PCA, n=number of iterations
LE <k=7> <s=1>
Laplacian Eigenmap, k=number of nearest neighbours, s=kernel sigma
HLLE <k=12>
Hessian Locally Linear Embedding, k=number of nearest neighbours
SPE <k=12> <global=1>
Stochastic Proximity Embedding, k=number of nearest neighbours, global embedding or not
NPE <k=12>
Neighborhood Preserving Embedding, k=number of nearest neighbours
""")
form.addParam('extraParams', StringParam, level=LEVEL_ADVANCED,
label="Extra params",
help='This parameters will be passed to the program.')
form.addParam('reducedDim', IntParam, default=2,
label='Reduced dimension')
form.addParallelSection(threads=0, mpi=0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Take deforamtions text file and the number of images and modes
inputSet = self.getInputParticles()
rows = inputSet.getSize()
reducedDim = self.reducedDim.get()
method = self.dimredMethod.get()
extraParams = self.extraParams.get('')
deformationsFile = self.getDeformationFile()
self._insertFunctionStep('convertInputStep',
deformationsFile, inputSet.getObjId())
self._insertFunctionStep('performDimredStep',
deformationsFile, method, extraParams,
rows, reducedDim)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, deformationFile, inputId):
""" Iterate throught the images and write the
plain deformation.txt file that will serve as
input for dimensionality reduction.
"""
inputSet = self.getInputParticles()
f = open(deformationFile, 'w')
for particle in inputSet:
f.write(' '.join(particle._xmipp_nmaDisplacements))
f.write('\n')
f.close()
def performDimredStep(self, deformationsFile, method, extraParams,
rows, reducedDim):
outputMatrix = self.getOutputMatrixFile()
methodName = DIMRED_VALUES[method]
# Get number of columes in deformation files
# it can be a subset of inputModes
f = open(deformationsFile)
columns = len(f.readline().split()) # count number of values in first line
f.close()
args = "-i %(deformationsFile)s -o %(outputMatrix)s -m %(methodName)s %(extraParams)s"
args += "--din %(columns)d --samples %(rows)d --dout %(reducedDim)d"
if method in DIMRED_MAPPINGS:
mappingFile = self._getExtraPath('projector.txt')
args += " --saveMapping %(mappingFile)s"
self.mappingFile.set(mappingFile)
self.runJob("xmipp_matrix_dimred", args % locals())
def createOutputStep(self):
pass
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
def _citations(self):
return []
def _methods(self):
return []
#--------------------------- UTILS functions --------------------------------------------
def getInputParticles(self):
""" Get the output particles of the input NMA protocol. """
return self.inputNMA.get().outputParticles
def getInputPdb(self):
return self.inputNMA.get().getInputPdb()
def getOutputMatrixFile(self):
return self._getExtraPath('output_matrix.txt')
def getDeformationFile(self):
return self._getExtraPath('deformations.txt')
def getProjectorFile(self):
return self.mappingFile.get()
def getMethodName(self):
return DIMRED_VALUES[self.dimredMethod.get()]
class XmippProtParticlePickingPairs(ProtParticlePicking, XmippProtocol):
""" Picks particles in a set of untilted-tilted pairs of micrographs. """
_label = 'tilt pairs particle picking'
def __init__(self, **args):
ProtParticlePicking.__init__(self, **args)
# The following attribute is only for testing
self.importFolder = String(args.get('importFolder', None))
#--------------- DEFINE param functions ---------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputMicrographsTiltedPair', params.PointerParam,
pointerClass='MicrographsTiltPair',
label="Micrographs tilt pair",
help='Select the MicrographsTiltPair ')
#----------- INSERT steps functions ----------------------------------
def _insertAllSteps(self):
""" The Particle Picking process is realized for a pair
of set of micrographs
"""
self.micsFn = self._getPath('input_micrographs.xmd')
# Convert input into xmipp Metadata format
self._insertFunctionStep('convertInputStep')
# Launch Particle Picking GUI
if not self.importFolder.hasValue():
self._insertFunctionStep('launchParticlePickGUIStep', interactive=True)
else: # This is only used for test purposes
self._insertFunctionStep('_importFromFolderStep')
#------------------- STEPS functions -----------------------------------
def convertInputStep(self):
micTiltPairs = self.inputMicrographsTiltedPair.get()
# Get the converted input micrographs in Xmipp format
convert.writeSetOfMicrographsPairs(micTiltPairs.getUntilted(),
micTiltPairs.getTilted(),
self.micsFn)
def launchParticlePickGUIStep(self):
process = launchTiltPairPickerGUI(self.micsFn, self._getExtraPath(),
self)
process.wait()
def _importFromFolderStep(self):
""" This function will copy Xmipp .pos files for
simulating a particle picking run...this is only
for testing purposes.
"""
extraDir = self._getExtraPath()
for f in pwutils.getFiles(self.importFolder.get()):
pwutils.copyFile(f, extraDir)
self.registerCoords(extraDir, readFromExtra=True)
#--------------------------- INFO functions --------------------------------------------
def _citations(self):
return []
#--------------------------- UTILS functions -------------------------------------------
def __str__(self):
""" String representation of a Particle Picking Tilt run """
outputs = self.getOutputsSize()
if outputs == 0:
msg = "No particles picked yet."
elif outputs == 1:
picked = self.getCoords().getSize()
mics = self.inputMicrographsTiltedPair.get().getTilted().getSize()
msg = "Number of particles picked: %d " % picked
msg += "(from %d micrographs)" % mics
else:
msg = 'Number of outputs: %d' % outputs
return msg
def getInputMicrographs(self):
return self.inputMicrographsTiltedPair.get().getTilted()
def getCoords(self):
return self.getCoordsTiltPair()
def _summary(self):
summary = []
if self.getInputMicrographs() is not None:
summary.append("Number of input micrographs: %d"
% self.getInputMicrographs().getSize())
if self.getOutputsSize() >= 1:
for key, output in self.iterOutputAttributes(CoordinatesTiltPair):
summary.append("*%s:*" % key)
summary.append(" Particles pairs picked: %d" % output.getSize())
summary.append(" Particle size: %d \n" % output.getBoxSize())
else:
summary.append("Output tilpairs not ready yet.")
return summary
def __getOutputSuffix(self):
maxCounter = -1
for attrName, _ in self.iterOutputAttributes(CoordinatesTiltPair):
suffix = attrName.replace('outputCoordinatesTiltPair', '')
try:
counter = int(suffix)
except:
counter = 1 # when there is not number assume 1
maxCounter = max(counter, maxCounter)
return str(maxCounter+1) if maxCounter > 0 else '' # empty if not outputs
def _getBoxSize(self):
""" Redefine this function to set a specific box size to the output
coordinates untilted and tilted.
"""
return None
def _readCoordinates(self, coordsDir, suffix=''):
micTiltPairs = self.inputMicrographsTiltedPair.get()
uSuffix = 'Untilted' + suffix
tSuffix = 'Tilted' + suffix
uSet = micTiltPairs.getUntilted()
tSet = micTiltPairs.getTilted()
# Create Untilted and Tilted SetOfCoordinates
uCoordSet = self._createSetOfCoordinates(uSet, suffix=uSuffix)
convert.readSetOfCoordinates(coordsDir, uSet, uCoordSet)
uCoordSet.write()
tCoordSet = self._createSetOfCoordinates(tSet, suffix=tSuffix)
convert.readSetOfCoordinates(coordsDir, tSet, tCoordSet)
tCoordSet.write()
boxSize = self._getBoxSize()
if boxSize:
uCoordSet.setBoxSize(boxSize)
tCoordSet.setBoxSize(boxSize)
return uCoordSet, tCoordSet
def _readAngles(self, micsFn, suffix=''):
# Read Angles from input micrographs
anglesSet = self._createSetOfAngles(suffix=suffix)
convert.readAnglesFromMicrographs(micsFn, anglesSet)
anglesSet.write()
return anglesSet
def registerCoords(self, coordsDir, store=True, readFromExtra=False):
micTiltPairs = self.inputMicrographsTiltedPair.get()
suffix = self.__getOutputSuffix()
uCoordSet, tCoordSet = self._readCoordinates(coordsDir, suffix)
if readFromExtra:
micsFn = self._getExtraPath('input_micrographs.xmd')
else:
micsFn = self._getPath('input_micrographs.xmd')
anglesSet = self._readAngles(micsFn, suffix)
# Create CoordinatesTiltPair object
outputset = self._createCoordinatesTiltPair(micTiltPairs,
uCoordSet, tCoordSet,
anglesSet, suffix)
summary = self.getSummary(outputset)
outputset.setObjComment(summary)
outputName = 'outputCoordinatesTiltPair' + suffix
outputs = {outputName: outputset}
self._defineOutputs(**outputs)
self._defineSourceRelation(self.inputMicrographsTiltedPair, outputset)
if store:
self._store()
class HostConfig(OrderedObject):
""" Main store the configuration for execution hosts. """
def __init__(self, **kwargs):
OrderedObject.__init__(self, **kwargs)
self.label = String(kwargs.get('label', None))
self.hostName = String(kwargs.get('hostName', None))
self.userName = String()
self.password = String()
self.hostPath = String()
self.mpiCommand = String()
self.scipionHome = String()
self.scipionConfig = String()
self.address = String()
self.queueSystem = QueueSystemConfig()
def getLabel(self):
return self.label.get()
def getHostName(self):
return self.hostName.get()
def getUserName(self):
return self.userName.get()
def getPassword(self):
return self.password.get()
def getHostPath(self):
return self.hostPath.get()
def getSubmitCommand(self):
return self.queueSystem.submitCommand.get()
def getSubmitPrefix(self):
return self.queueSystem.submitPrefix.get()
def getCheckCommand(self):
return self.queueSystem.checkCommand.get()
def getCancelCommand(self):
return self.queueSystem.cancelCommand.get()
def isQueueMandatory(self):
return self.queueSystem.mandatory.get()
def getSubmitTemplate(self):
return self.queueSystem.getSubmitTemplate()
def getQueuesDefault(self):
return self.queueSystem.queuesDefault
def getMpiCommand(self):
return self.mpiCommand.get()
def getQueueSystem(self):
return self.queueSystem
def getJobDoneRegex(self):
return self.queueSystem.jobDoneRegex.get()
def setLabel(self, label):
self.label.set(label)
def setHostName(self, hostName):
self.hostName.set(hostName)
def setUserName(self, userName):
self.userName.set(userName)
def setPassword(self, password):
self.password.set(password)
def setHostPath(self, hostPath):
self.hostPath.set(hostPath)
def setMpiCommand(self, mpiCommand):
self.mpiCommand.set(mpiCommand)
def setQueueSystem(self, queueSystem):
self.queueSystem = queueSystem
def getScipionHome(self):
""" Return the path where Scipion is installed in
the host. This is useful when launching remote jobs.
"""
return self.scipionHome.get()
def setScipionHome(self, newScipionHome):
self.scipionHome.set(newScipionHome)
def getScipionConfig(self):
""" From which file to read the configuration file in
this hosts. Useful for remote jobs.
"""
return self.scipionConfig.get()
def setScipionConfig(self, newConfig):
self.scipionConfig.set(newConfig)
def getAddress(self):
return self.address.get()
def setAddress(self, newAddress):
return self.address.set(newAddress)
@classmethod
def load(cls, hostsConf):
""" Load several hosts from a configuration file.
Return an dictionary with hostName -> hostConfig pairs.
"""
# Read from users' config file.
cp = ConfigParser()
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
hosts = OrderedDict()
try:
assert cp.read(hostsConf) != [], 'Missing file %s' % hostsConf
for hostName in cp.sections():
host = HostConfig(label=hostName, hostName=hostName)
host.setHostPath(pw.Config.SCIPION_USER_DATA)
# Helper functions (to write less)
def get(var, default=None):
if cp.has_option(hostName, var):
return cp.get(hostName, var).replace('%_(', '%(')
else:
return default
def getDict(var):
od = OrderedDict()
if cp.has_option(hostName, var):
for key, value in json.loads(get(var)).iteritems():
od[key] = value
return od
host.setScipionHome(get('SCIPION_HOME', pw.Config.SCIPION_HOME))
host.setScipionConfig(get('SCIPION_CONFIG'))
# Read the address of the remote hosts,
# using 'localhost' as default for backward compatibility
host.setAddress(get('ADDRESS', 'localhost'))
host.mpiCommand.set(get('PARALLEL_COMMAND'))
host.queueSystem = QueueSystemConfig()
hostQueue = host.queueSystem # shortcut
hostQueue.name.set(get('NAME'))
# If the NAME is not provided or empty
# do no try to parse the rest of Queue parameters
if hostQueue.hasName():
hostQueue.setMandatory(get('MANDATORY', 0))
hostQueue.submitPrefix.set(get('SUBMIT_PREFIX', ''))
hostQueue.submitCommand.set(get('SUBMIT_COMMAND'))
hostQueue.submitTemplate.set(get('SUBMIT_TEMPLATE'))
hostQueue.cancelCommand.set(get('CANCEL_COMMAND'))
hostQueue.checkCommand.set(get('CHECK_COMMAND'))
hostQueue.jobDoneRegex.set(get('JOB_DONE_REGEX'))
hostQueue.queues = getDict('QUEUES')
hostQueue.queuesDefault = getDict('QUEUES_DEFAULT')
hosts[hostName] = host
return hosts
except Exception as e:
sys.exit('Failed to read settings. The reported error was:\n %s\n'
'To solve it, delete %s and run again.' % (e, hostsConf))
class XmippProtScreenParticles(ProtProcessParticles):
""" Classify particles according their similarity to the others in order to detect outliers. """
_label = 'screen particles'
# Automatic Particle rejection enum
REJ_NONE = 0
REJ_MAXZSCORE = 1
REJ_PERCENTAGE =2
REJ_PERCENTAGE_SSNR =1
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
form.addParam('autoParRejection', EnumParam, choices=['None', 'MaxZscore', 'Percentage'],
label="Automatic particle rejection based on Zscore", default=self.REJ_NONE,
display=EnumParam.DISPLAY_COMBO, expertLevel=LEVEL_ADVANCED,
help='How to automatically reject particles. It can be none (no rejection), '
'maxZscore (reject a particle if its Zscore [a similarity index] is larger than this value), '
'Percentage (reject a given percentage in each one of the screening criteria). ')
form.addParam('maxZscore', FloatParam, default=3, condition='autoParRejection==1',
label='Maximum Zscore', expertLevel=LEVEL_ADVANCED,
help='Maximum Zscore.', validators=[Positive])
form.addParam('percentage', IntParam, default=5, condition='autoParRejection==2',
label='Percentage (%)', expertLevel=LEVEL_ADVANCED,
help='The worse percentage of particles according to metadata labels: ZScoreShape1, ZScoreShape2, ZScoreSNR1, ZScoreSNR2, ZScoreHistogram are automatically disabled. Therefore, the total number of disabled particles belongs to [percetage, 5*percentage]', validators=[Range(0, 100, error="Percentage must be between 0 and 100.")])
form.addParam('autoParRejectionSSNR', EnumParam, choices=['None', 'Percentage'],
label="Automatic particle rejection based on SSNR", default=self.REJ_NONE,
display=EnumParam.DISPLAY_COMBO, expertLevel=LEVEL_ADVANCED,
help='How to automatically reject particles. It can be none (no rejection), '
'Percentage (reject a given percentage of the lowest SSNRs). ')
form.addParam('percentageSSNR', IntParam, default=5, condition='autoParRejectionSSNR==1',
label='Percentage (%)', expertLevel=LEVEL_ADVANCED,
help='The worse percentage of particles according to SSNR are automatically disabled.',
validators=[Range(0, 100, error="Percentage must be between 0 and 100.")])
form.addParallelSection(threads=0, mpi=0)
def _getDefaultParallel(self):
"""This protocol doesn't have mpi version"""
return (0, 0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Mainly prepare the command line for call cl2d program"""
# Convert input images if necessary
self._insertFunctionStep('sortImages', self.inputParticles.getObjId())
self._insertFunctionStep('sortImagesSSNR', self.inputParticles.getObjId())
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def sortImages(self, inputId):
imagesMd = self._getPath('images.xmd')
writeSetOfParticles(self.inputParticles.get(), imagesMd)
args = "-i Particles@%s --addToInput " % imagesMd
if self.autoParRejection == self.REJ_MAXZSCORE:
args += "--zcut " + str(self.maxZscore.get())
elif self.autoParRejection == self.REJ_PERCENTAGE:
args += "--percent " + str(self.percentage.get())
self.runJob("xmipp_image_sort_by_statistics", args)
self.outputMd = String(imagesMd)
def sortImagesSSNR(self, inputId):
imagesMd = self._getPath('images.xmd')
args = "-i Particles@%s " % imagesMd
if self.autoParRejectionSSNR == self.REJ_PERCENTAGE_SSNR:
args += "--ssnrpercent " + str(self.percentageSSNR.get())
self.runJob("xmipp_image_ssnr", args)
def createOutputStep(self):
imgSet = self._createSetOfParticles()
imgSet.copyInfo(self.inputParticles.get())
readSetOfParticles(self.outputMd.get(), imgSet)
self._defineOutputs(outputParticles=imgSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
import os
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
zscores = [p._xmipp_zScore.get() for p in self.outputParticles]
summary.append("The minimum ZScore is %.2f" % min(zscores))
summary.append("The maximum ZScore is %.2f" % max(zscores))
summary.append("The mean ZScore is %.2f" % (sum(zscores)*1.0/len(self.outputParticles)))
return summary
def _validate(self):
pass
def _citations(self):
return ['Vargas2013b']
def _methods(self):
methods = []
if hasattr(self, 'outputParticles'):
outParticles = len(self.outputParticles) if self.outputParticles is not None else None
particlesRejected = len(self.inputParticles.get())-outParticles if outParticles is not None else None
particlesRejectedText = ' ('+str(particlesRejected)+')' if particlesRejected is not None else ''
rejectionText = [
'',# REJ_NONE
' and removing those not reaching %s%s' % (str(self.maxZscore.get()), particlesRejectedText),# REJ_MAXZSCORE
' and removing worst %s percent%s' % (str(self.percentage.get()), particlesRejectedText)# REJ_PERCENTAGE
]
methods.append('Input dataset %s of %s particles was sorted by'
' its ZScore using xmipp_image_sort_by_statistics'
' program%s. ' % (self.getObjectTag('inputParticles'), len(self.inputParticles.get()), rejectionText[self.autoParRejection.get()]))
methods.append('Output set is %s.'%self.getObjectTag('outputParticles'))
return methods
class HostConfig(Object):
""" Main store the configuration for execution hosts. """
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.label = String(kwargs.get('label', None))
self.hostName = String(kwargs.get('hostName', None))
self.userName = String()
self.password = String()
self.hostPath = String()
self.mpiCommand = String()
self.scipionHome = String()
self.scipionConfig = String()
self.address = String()
self.queueSystem = QueueSystemConfig()
def getLabel(self):
return self.label.get()
def getHostName(self):
return self.hostName.get()
def getUserName(self):
return self.userName.get()
def getPassword(self):
return self.password.get()
def getHostPath(self):
return self.hostPath.get()
def getSubmitCommand(self):
return self.queueSystem.submitCommand.get()
def getSubmitPrefix(self):
return self.queueSystem.submitPrefix.get()
def getCheckCommand(self):
return self.queueSystem.checkCommand.get()
def getCancelCommand(self):
return self.queueSystem.cancelCommand.get()
def isQueueMandatory(self):
return self.queueSystem.mandatory.get()
def getSubmitTemplate(self):
return self.queueSystem.getSubmitTemplate()
def getQueuesDefault(self):
return self.queueSystem.queuesDefault
def getMpiCommand(self):
return self.mpiCommand.get()
def getQueueSystem(self):
return self.queueSystem
def getJobDoneRegex(self):
return self.queueSystem.jobDoneRegex.get()
def setLabel(self, label):
self.label.set(label)
def setHostName(self, hostName):
self.hostName.set(hostName)
def setUserName(self, userName):
self.userName.set(userName)
def setPassword(self, password):
self.password.set(password)
def setHostPath(self, hostPath):
self.hostPath.set(hostPath)
def setMpiCommand(self, mpiCommand):
self.mpiCommand.set(mpiCommand)
def setQueueSystem(self, queueSystem):
self.queueSystem = queueSystem
def getScipionHome(self):
""" Return the path where Scipion is installed in
the host. This is useful when launching remote jobs.
"""
return self.scipionHome.get()
def setScipionHome(self, newScipionHome):
self.scipionHome.set(newScipionHome)
def getScipionConfig(self):
""" From which file to read the configuration file in
this hosts. Useful for remote jobs.
"""
return self.scipionConfig.get()
def setScipionConfig(self, newConfig):
self.scipionConfig.set(newConfig)
def getAddress(self):
return self.address.get()
def setAddress(self, newAddress):
return self.address.set(newAddress)
@classmethod
def writeBasic(cls, configFn):
""" Write a very basic Host configuration for testing purposes. """
with open(configFn, 'w') as f:
f.write('[localhost]\nPARALLEL_COMMAND = '
'mpirun -np %%(JOB_NODES)d --map-by node %%(COMMAND)s\n')
@classmethod
def load(cls, hostsConf):
""" Load several hosts from a configuration file.
Return an dictionary with hostName -> hostConfig pairs.
"""
# Read from users' config file. Raw to avoid interpolation of %: we expect %_
cp = RawConfigParser(comment_prefixes=";")
cp.optionxform = str # keep case (stackoverflow.com/questions/1611799)
hosts = OrderedDict()
try:
assert cp.read(hostsConf) != [], 'Missing file %s' % hostsConf
for hostName in cp.sections():
host = HostConfig(label=hostName, hostName=hostName)
host.setHostPath(pw.Config.SCIPION_USER_DATA)
# Helper functions (to write less)
def get(var, default=None):
if cp.has_option(hostName, var):
value = cp.get(hostName, var)
# Rescue python2.7 behaviour: ## at the beginning of a line, means a single #.
# https://github.com/scipion-em/scipion-pyworkflow/issues/70
value = value.replace("\n##", "\n#")
# Keep compatibility: %_ --> %%
value = value.replace('%_(', '%(')
return value
else:
return default
def getDict(var):
od = OrderedDict()
if cp.has_option(hostName, var):
for key, value in json.loads(get(var)).items():
od[key] = value
return od
host.setScipionHome(
get(pw.SCIPION_HOME_VAR, pw.Config.SCIPION_HOME))
host.setScipionConfig(pw.Config.SCIPION_CONFIG)
# Read the address of the remote hosts,
# using 'localhost' as default for backward compatibility
host.setAddress(get('ADDRESS', 'localhost'))
host.mpiCommand.set(get('PARALLEL_COMMAND'))
host.queueSystem = QueueSystemConfig()
hostQueue = host.queueSystem # shortcut
hostQueue.name.set(get('NAME'))
# If the NAME is not provided or empty
# do no try to parse the rest of Queue parameters
if hostQueue.hasName():
hostQueue.setMandatory(get('MANDATORY', 0))
hostQueue.submitPrefix.set(get('SUBMIT_PREFIX', ''))
hostQueue.submitCommand.set(get('SUBMIT_COMMAND'))
hostQueue.submitTemplate.set(get('SUBMIT_TEMPLATE'))
hostQueue.cancelCommand.set(get('CANCEL_COMMAND'))
hostQueue.checkCommand.set(get('CHECK_COMMAND'))
hostQueue.jobDoneRegex.set(get('JOB_DONE_REGEX'))
hostQueue.queues = getDict('QUEUES')
hostQueue.queuesDefault = getDict('QUEUES_DEFAULT')
hosts[hostName] = host
return hosts
except Exception as e:
sys.exit('Failed to read settings. The reported error was:\n %s\n'
'To solve it, delete %s and run again.' %
(e, os.path.abspath(hostsConf)))
class XmippProtDenoiseParticles(ProtProcessParticles):
""" Remove particles noise by filtering them. This filtering process is based on a projection over a basis created from some averages (extracted from classes). This filtering is not intended for processing particles. The huge filtering they will be passed through is known to remove part of the signal
with the noise. However this is a good method for clearly see which particle are we going to process before it's done. """
_label = 'denoise particles'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
# First we customize the inputParticles param to fit our needs in this protocol
form.getParam('inputParticles').pointerCondition = String('hasAlignment')
form.getParam('inputParticles').help = String('Input images you want to filter. It is important that the images have alignment information with '
'respect to the chosen set of classes. This is the standard situation '
'after CL2D or ML2D.')
form.addParam('inputClasses', PointerParam, label='Input Classes', important=True,
pointerClass='SetOfClasses',
help='Select the input classes for the basis construction against images will be projected to.')
form.addSection(label='Basis construction')
form.addParam('maxClasses', IntParam, default=128,
label='Max. number of classes', expertLevel=LEVEL_ADVANCED,
help='Maximum number of classes.')
form.addParam('maxPCABases', IntParam, default=200,
label='Number of PCA bases', expertLevel=LEVEL_ADVANCED,
help='Number of PCA bases.')
form.addSection(label='Denoising')
form.addParam('PCABases2Project', IntParam, default=200,
label='Number of PCA bases on which to project', expertLevel=LEVEL_ADVANCED,
help='Number of PCA bases on which to project.')
def _getDefaultParallel(self):
""" Return the default value for thread and MPI
for the parallel section definition.
"""
return (2, 4)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Insert every step of the protocol"""
# Convert input images if necessary
self._insertFunctionStep('denoiseImages', self.inputParticles.getObjId(), self.inputClasses.getObjId())
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def denoiseImages(self, inputId, inputClassesId):
# We start preparing writing those elements we're using as input to keep them untouched
imagesMd = self._getPath('images.xmd')
writeSetOfParticles(self.inputParticles.get(), imagesMd)
classesMd = self._getPath('classes.xmd')
writeSetOfClasses2D(self.inputClasses.get(), classesMd)
fnRoot = self._getExtraPath('pca')
fnRootDenoised = self._getExtraPath('imagesDenoised')
args = '-i Particles@%s --oroot %s --eigenvectors %d --maxImages %d' % (imagesMd, fnRoot, self.maxPCABases.get(), self.maxClasses.get())
self.runJob("xmipp_image_rotational_pca", args)
N=min(self.maxPCABases.get(), self.PCABases2Project.get())
args='-i %s -o %s.stk --save_metadata_stack %s.xmd --basis %s.stk %d'\
% (imagesMd, fnRootDenoised, fnRootDenoised, fnRoot, N)
self.runJob("xmipp_transform_filter", args)
self.outputMd = String('%s.stk' % fnRootDenoised)
def createOutputStep(self):
imgSet = self._createSetOfParticles()
imgSet.copyInfo(self.inputParticles.get())
readSetOfParticles(self.outputMd.get(), imgSet)
self._defineOutputs(outputParticles=imgSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
summary.append('PCA basis created by using %d classes' % len(self.inputClasses.get()))
summary.append('Max. number of classes defined for PCA basis creation: %d' % self.maxClasses.get())
summary.append('Max. number of PCA bases defined for PCA basis creation: %d' % self.maxPCABases.get())
summary.append('PCA basis on which to project for denoising: %d' % self.PCABases2Project.get())
return summary
def _validate(self):
pass
def _citations(self):
return ['zhao2013', 'ponce2011']
def _methods(self):
methods = []
if not hasattr(self, 'outputParticles'):
methods.append("Output particles not ready yet.")
else:
methods.append('An input dataset of %d particles was filtered creating a PCA basis (%d components) with '
'xmipp_image_rotational_pca and projecting the dataset into that base with xmipp_transform_filter.'\
% (len(self.inputParticles.get()), len(self.inputClasses.get())))
return methods
class XmippProtAngBreakSymmetry(ProtProcessParticles):
"""
Given an input set of particles with angular assignment, find an
equivalent angular assignment for a given symmetry.
Be aware that input symmetry values follows Xmipp conventions as described in:
http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry
"""
_label = 'break symmetry'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineProcessParams(self, form):
form.addParam(
'symmetryGroup',
StringParam,
default="c1",
label='Symmetry group',
help="See http://xmipp.cnb.csic.es/twiki/bin/view/Xmipp/Symmetry"
" for a description of the symmetry groups format in Xmipp.\n"
"If no symmetry is present, use _c1_.")
def _getDefaultParallel(self):
"""This protocol doesn't have mpi version"""
return (0, 0)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
""" Mainly prepare the command line for call brak symmetry program"""
# Create a metadata with the geometrical information
# as expected by Xmipp
imgsFn = self._getPath('input_particles.xmd')
self._insertFunctionStep('convertInputStep', imgsFn)
self._insertFunctionStep('breakSymmetryStep', imgsFn)
self._insertFunctionStep('createOutputStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, outputFn):
""" Create a metadata with the images and geometrical information. """
writeSetOfParticles(self.inputParticles.get(), outputFn)
#--------------------------- STEPS functions --------------------------------------------
def breakSymmetryStep(self, imgsFn):
outImagesMd = self._getPath('images.xmd')
args = "-i Particles@%s --sym %s -o %s" % (
imgsFn, self.symmetryGroup.get(), outImagesMd)
self.runJob("xmipp_angular_break_symmetry", args)
self.outputMd = String(outImagesMd)
def createOutputStep(self):
imgSet = self.inputParticles.get()
partSet = self._createSetOfParticles()
partSet.copyInfo(imgSet)
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
self.outputMd.get(), sortByLabel=md.MDL_ITEM_ID))
self._defineOutputs(outputParticles=partSet)
self._defineSourceRelation(imgSet, partSet)
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
import os
summary = []
if not hasattr(self, 'outputParticles'):
summary.append("Output particles not ready yet.")
else:
summary.append("Symmetry: %s" % self.symmetryGroup.get())
return summary
def _validate(self):
pass
def _citations(self):
return [] #['Vargas2013b']
def _methods(self):
methods = []
# if hasattr(self, 'outputParticles'):
# outParticles = len(self.outputParticles) if self.outputParticles is not None else None
# particlesRejected = len(self.inputParticles.get())-outParticles if outParticles is not None else None
# particlesRejectedText = ' ('+str(particlesRejected)+')' if particlesRejected is not None else ''
# rejectionText = [
# '',# REJ_NONE
# ' and removing those not reaching %s%s' % (str(self.maxZscore.get()), particlesRejectedText),# REJ_MAXZSCORE
# ' and removing worst %s percent%s' % (str(self.percentage.get()), particlesRejectedText)# REJ_PERCENTAGE
# ]
# methods.append('Input dataset %s of %s particles was sorted by'
# ' its ZScore using xmipp_image_sort_by_statistics'
# ' program%s. ' % (self.getObjectTag('inputParticles'), len(self.inputParticles.get()), rejectionText[self.autoParRejection.get()]))
# methods.append('Output set is %s.'%self.getObjectTag('outputParticles'))
return methods
#--------------------------- Utils functions --------------------------------------------
def _createItemMatrix(self, item, row):
from xmipp3.convert import createItemMatrix
createItemMatrix(item, row, align=ALIGN_PROJ)
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
INPUT_HELP = """ Input volume(s) for ResMap.
Required volume properties:
1. The particle must be centered in the volume.
2. The background must not been masked out.
Desired volume properties:
1. The volume has not been filtered in any way (low-pass filtering, etc.)
2. The volume has a realistic noise spectrum.
This is sometimes obtained by so-called amplitude correction.
While a similar effect is often obtained by B-factor sharpening,
please make sure that the spectrum does not blow up near Nyquist.
"""
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
self.plotData = String() # store some values for later plot
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('useSplitVolume', params.BooleanParam, default=False,
label="Use half volumes?",
help='Use to split volumes for gold-standard FSC.')
form.addParam('inputVolume', params.PointerParam,
pointerClass='Volume', condition="not useSplitVolume",
label="Input volume", important=True,
help=self.INPUT_HELP)
form.addParam('volumeHalf1', params.PointerParam,
label="Volume half 1", important=True,
pointerClass='Volume', condition="useSplitVolume",
help=self.INPUT_HELP)
form.addParam('volumeHalf2', params.PointerParam,
pointerClass='Volume', condition="useSplitVolume",
label="Volume half 2", important=True,
help=self.INPUT_HELP)
form.addParam('applyMask', params.BooleanParam, default=False,
label="Mask input volume?",
help="It is not necessary to provide ResMap with a mask "
"volume. The algorithm will attempt to estimate a "
"mask volume by low-pass filtering the input volume "
"and thresholding it using a heuristic procedure.\n"
"If the automated procedure does not work well for "
"your particle, you may provide a mask volume that "
"matches the input volume in size and format. "
"The mask volume should be a binary volume with zero "
"(0) denoting the background/solvent and some positive"
"value (0+) enveloping the particle.")
form.addParam('maskVolume', params.PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
form.addParam('whiteningLabel', params.LabelParam, important=True,
label="It is strongly recommended to use the "
"pre-whitening wizard.")
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', params.FloatParam, default=10,
label="Angstroms")
line.addParam('prewhitenRamp', params.FloatParam, default=1,
label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', params.FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help="Default (0): algorithm will start a just above\n"
" 2*voxelSize until 4*voxelSize. \n"
"These fields are provided to accelerate computation "
"if you are only interested in analyzing a specific "
"resolution range. It is usually a good idea to provide "
"a maximum resolution value to save time. Another way to "
"save computation is to provide a larger step size.")
line.addParam('minRes', params.FloatParam, default=0, label='Min')
line.addParam('maxRes', params.FloatParam, default=0, label='Max')
group.addParam('pVal', params.FloatParam, default=0.05,
label='Confidence level:',
help="P-value, usually between [0.01, 0.05].\n\n"
"This is the p-value of the statistical hypothesis test "
"on which ResMap is based on. It is customarily set to "
"0.05 although you are welcome to reduce it (e.g. 0.01) "
"if you would like to obtain a more conservative result. "
"Empirically, ResMap results are not much affected by the p-value.")
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
if self.useSplitVolume:
inputs = [self.volumeHalf1, self.volumeHalf2]
self.inputVolume.set(None)
else:
inputs = [self.inputVolume]
self.volumeHalf1.set(None)
self.volumeHalf2.set(None)
locations = [i.get().getLocation() for i in inputs]
self._insertFunctionStep('convertInputStep', *locations)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropriate parameters. """
results = self.runResmap(self._getPath())
self.histogramData.set(dumps(results['resHisto']))
plotDict = {'minRes': results['minRes'],
'maxRes': results['maxRes'],
'orig_n': results['orig_n'],
'n': results['n'],
'currentRes': results['currentRes']
}
self.plotData.set(dumps(plotDict))
self._store(self.histogramData, self.plotData)
self.savePlots(results)
def savePlots(self, results=None):
""" Store png images of the plots to be used as images, """
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
self._plotVolumeSlices().savefig(self._getExtraPath('volume1.map.png'))
plot = self._plotResMapSlices(results['resTOTALma'])
plot.savefig(self._getExtraPath('volume1_resmap.map.png'))
self._plotHistogram().savefig(self._getExtraPath('histogram.png'))
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
if self.useSplitVolume:
half1 = self.volumeHalf1.get()
half2 = self.volumeHalf2.get()
if half1.getSamplingRate() != half2.getSamplingRate():
errors.append('The selected half volumes have not the same pixel size.')
if half1.getXDim() != half2.getXDim():
errors.append('The selected half volumes have not the same dimensions.')
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'vxSize': self.volumeHalf1.get().getSamplingRate(),
'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'vxSize': self.inputVolume.get().getSamplingRate(),
'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self, **kwargs):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'), **kwargs)
return Plotter(figure=fig)
def _plotResMapSlices(self, data=None, **kwargs):
from ResMap_visualization import plotResMapVolume
plotDict = loads(self.plotData.get())
if data is None:
data = self._getVolumeMatrix('volume1_resmap.map')
data = np.ma.masked_where(data > plotDict['currentRes'], data, copy=True)
kwargs.update(plotDict)
fig = plotResMapVolume(data, **kwargs)
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
class ProtResMap(ProtAnalysis3D):
"""
ResMap is software tool for computing the local resolution of 3D
density maps studied in structural biology, primarily by cryo-electron
microscopy (cryo-EM).
Please find the manual at http://resmap.sourceforge.net
"""
_label = 'local resolution'
def __init__(self, **kwargs):
ProtAnalysis3D.__init__(self, **kwargs)
self.histogramData = String()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam('inputVolume', PointerParam, pointerClass='Volume',
label="Input volume", important=True,
help='Select the input volume.')
form.addParam('useSplitVolume', BooleanParam, default=False,
label="Use split volume?",
help='Use to split volumes for gold-standard FSC.')
form.addParam('splitVolume', PointerParam, label="Split volume", important=True,
pointerClass='Volume', condition="useSplitVolume",
help='Select the second split volume.')
form.addParam('applyMask', BooleanParam, default=False,
label="Mask input volume?",
help='If set to <No> ResMap will automatically compute a mask.')
form.addParam('maskVolume', PointerParam, label="Mask volume",
pointerClass='VolumeMask', condition="applyMask",
help='Select a volume to apply as a mask.')
line = form.addLine('Pre-whitening')
line.addParam('prewhitenAng', FloatParam, default=0, label="Angstroms")
line.addParam('prewhitenRamp', FloatParam, default=0, label='Ramp')
group = form.addGroup('Extra parameters')
#form.addSection(label='Optional')
group.addParam('stepRes', FloatParam, default=1,
label='Step size (Ang):',
help='in Angstroms (min 0.25, default 1.0)')
line = group.addLine('Resolution Range (A)',
help='Default: algorithm will start a just above 2*voxelSize')
line.addParam('minRes', FloatParam, default=0, label='Min')
line.addParam('maxRes', FloatParam, default=0, label='Max')
group.addParam('pVal', FloatParam, default=0.05,
label='Confidence level:',
help='usually between [0.01, 0.05]')
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
# Insert processing steps
inputs = [self.inputVolume.get().getLocation()]
if self.useSplitVolume:
inputs.append(self.splitVolume.get().getLocation())
self._insertFunctionStep('convertInputStep', *inputs)
self._insertFunctionStep('estimateResolutionStep',
self.pVal.get(),
self.minRes.get(),
self.maxRes.get(),
self.stepRes.get(),
self.prewhitenAng.get(),
self.prewhitenRamp.get())
self._insertFunctionStep('savePlotsStep')
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self, volLocation1, volLocation2=None):
""" Convert input volume to .mrc as expected by ResMap.
Params:
volLocation1: a tuple containing index and filename of the input volume.
volLocation2: if not None, a tuple like volLocation1 for the split volume.
"""
ih = ImageHandler()
ih.convert(volLocation1, self._getPath('volume1.map'))
if volLocation2 is not None:
ih.convert(volLocation2, self._getPath('volume2.map'))
def estimateResolutionStep(self, pValue, minRes, maxRes, stepRes, ang, rampWeight):
""" Call ResMap.py with the appropiate parameters. """
results = self.runResmap(self._getPath())
from cPickle import dumps
self.histogramData.set(dumps(results['resHisto']))
self._store(self.histogramData)
def savePlotsStep(self):
""" Store png images of the plots to be used
from web.
"""
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
# This is needed right now because we are having
# some memory problem with matplotlib plots right now in web
Plotter.setBackend('Agg')
self._plotVolumeSlices().savefig(self._getExtraPath('volume1.map.png'))
self._plotResMapSlices().savefig(self._getExtraPath('volume1_resmap.map.png'))
self._plotHistogram().savefig(self._getExtraPath('histogram.png'))
#--------------------------- INFO functions --------------------------------------------
def _summary(self):
summary = []
return summary
def _validate(self):
errors = []
return errors
#--------------------------- UTILS functions --------------------------------------------
def runResmap(self, workingDir, wizardMode=False):
""" Prepare the args dictionary to be used
and call the ResMap algorithm.
Params:
workingDir: where to run ResMap
wizardMode: some custom params to be used by the wizard
to display the pre-whitening GUI and only that.
with the """
self._enterDir(workingDir)
volumes = ['volume1.map', 'volume2.map']
# Add resmap libraries to the path
sys.path.append(os.environ['RESMAP_HOME'])
from ResMap_algorithm import ResMap_algorithm
from ResMap_fileIO import MRC_Data
# Always read the first volume as mrc data
data1 = MRC_Data(volumes[0],'ccp4')
prewhitenArgs = {'display': wizardMode,
'force-stop': wizardMode
}
if (self.prewhitenAng.hasValue() and
self.prewhitenRamp.hasValue()):
prewhitenArgs['newElbowAngstrom'] = self.prewhitenAng.get()
prewhitenArgs['newRampWeight'] = self.prewhitenRamp.get()
args = {'vxSize': self.inputVolume.get().getSamplingRate(),
'pValue': self.pVal.get(),
'minRes': self.minRes.get(),
'maxRes': self.maxRes.get(),
'stepRes': self.stepRes.get(),
'chimeraLaunch': False, # prevent ResMap to launch some graphical analysis
'graphicalOutput': False,
'scipionPrewhitenParams': prewhitenArgs
}
if self.useSplitVolume:
# Read the second splitted volume
data2 = MRC_Data(volumes[1],'ccp4')
args.update({'inputFileName1': 'volume1.map',
'inputFileName2': 'volume2.map',
'data1': data1,
'data2': data2,
})
else:
args.update({'inputFileName': 'volume1.map',
'data': data1,
})
results = ResMap_algorithm(**args)
self._leaveDir()
return results
#--------- Functions related to Plotting
def _getVolumeMatrix(self, volName):
from ResMap_fileIO import MRC_Data
volPath = self._getPath(volName)
return MRC_Data(volPath, 'ccp4').matrix
def _plotVolumeSlices(self):
from ResMap_visualization import plotOriginalVolume
fig = plotOriginalVolume(self._getVolumeMatrix('volume1.map'))
return Plotter(figure=fig)
def _plotResMapSlices(self):
from ResMap_visualization import plotResMapVolume
fig = plotResMapVolume(self._getVolumeMatrix('volume1_resmap.map'),
minRes=self.minRes.get(),
maxRes=self.maxRes.get())
return Plotter(figure=fig)
def _plotHistogram(self):
from ResMap_visualization import plotResolutionHistogram
from cPickle import loads
histogramData = loads(self.histogramData.get())
fig = plotResolutionHistogram(histogramData)
return Plotter(figure=fig)
