class ProtPKPDExponentialFit(ProtPKPDFitBase):
""" Fit a set of exponentials. The observed measurement is modelled as Y=sum_{i=1}^N c_i exp(-lambda_i * X).\n
Confidence intervals calculated by this fitting may be pessimistic because it assumes that all model parameters
are independent, which are not. Use Bootstrap estimates instead.\n
Protocol created by http://www.kinestatpharma.com\n"""
_label = 'fit exponentials'
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form, fullForm=True):
self._defineParams1(form,"t","Cp")
if fullForm:
form.addParam('fitType', params.EnumParam, choices=["Linear","Logarithmic","Relative"], label="Fit mode", default=1,
help='Linear: sum (Cobserved-Cpredicted)^2\nLogarithmic: sum(log10(Cobserved)-log10(Cpredicted))^2\n'\
"Relative: sum ((Cobserved-Cpredicted)/Cobserved)^2")
form.addParam('Nexp', params.IntParam, label="Number of exponentials", default=1,
help='Number of exponentials to fit')
else:
self.fitType=Integer()
self.fitType.set(1)
self.Nexp=Integer()
self.Nexp.set(1)
form.addParam('bounds', params.StringParam, label="Amplitude and time constant bounds", default="", expertLevel=LEVEL_ADVANCED,
help='Bounds for the c_i amplitudes and lambdas.\nExample 1: (0,10);(0,1e-2) -> c1 in (0,10), lambda1 in (0,1e-2)\n'\
'Example 2: (0,10);(0,1e-2);(0,1);(0,1e-1) -> c1 in (0,10), lambda1 in (0,1e-2), c2 in (0,1), lambda2 in (0,1e-1)')
form.addParam('confidenceInterval', params.FloatParam, label="Confidence interval=", default=95, expertLevel=LEVEL_ADVANCED,
help='Confidence interval for the fitted parameters')
if fullForm:
form.addParam('reportX', params.StringParam, label="Evaluate at X=", default="", expertLevel=LEVEL_ADVANCED,
help='Evaluate the model at these X values\nExample 1: [0,5,10,20,40,100]\nExample 2: 0:0.55:10, from 0 to 10 in steps of 0.5')
else:
self.reportX=String()
self.reportX.set("")
def getListOfFormDependencies(self):
return [self.predictor.get(), self.predicted.get(), self.fitType.get(), self.bounds.get()]
def createModel(self):
return PKPDExponentialModel()
def setupFromFormParameters(self):
self.model.Nexp=self.Nexp.get()
#--------------------------- INFO functions --------------------------------------------
def _warnings(self):
warnings = []
experiment = self.readExperiment(self.getInputExperiment().fnPKPD,show=False)
incorrectList = experiment.getNonBolusDoses()
if len(incorrectList)>0:
warnings.append("This protocol is meant only for intravenous bolus regimens. Check the doses for %s"%(','.join(incorrectList)))
return warnings
def _validate(self):
errors=ProtPKPDFitBase._validate(self)
if self.Nexp.get()<1:
errors.append("The number of exponentials has to be larger than 0")
return errors
class XmippProtProjMatch(ProtRefine3D, ProtClassify3D):
""" 3D reconstruction and classification using multireference projection matching"""
_label = 'projection matching'
FILENAMENUMBERLENGTH = 6
def __init__(self, **args):
ProtRefine3D.__init__(self, **args)
ProtClassify3D.__init__(self, **args)
self.numberOfCtfGroups = Integer(1)
self._lastIter = Integer(0)
def _initialize(self):
""" This function is mean to be called after the
working dir for the protocol have been set. (maybe after recovery from mapper)
"""
self._loadInputInfo()
# Setup the dictionary with filenames templates to
# be used by _getFileName
createFilenameTemplates(self)
# Load the values from several params generating a list
# of values per iteration or references
initializeLists(self)
def _loadInputInfo(self):
from pyworkflow.em.packages.xmipp3 import getImageLocation
reference = self.input3DReferences.get() # Input can be either a single volume or a set of volumes.
if isinstance(reference, Volume): # Treat the case of a single volume
self.referenceFileNames = [getImageLocation(reference)]
else:
self.referenceFileNames = [getImageLocation(vol) for vol in reference]
self.numberOfReferences = len(self.referenceFileNames)
self.resolSam = reference.getSamplingRate()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
""" Since the form definition is very very large,
we have do it in a separated function.
"""
_defineProjectionMatchingParams(self, form)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
self._initialize()
# Insert initial steps
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('executeCtfGroupsStep')
# insertExecuteCtfGroupsStep(self)
# insertInitAngularReferenceFileStep(self)
self._insertFunctionStep('initAngularReferenceFileStep')
# Steps per iteration
self._insertItersSteps()
# Final steps
self._insertFunctionStep('createOutputStep')
def _insertItersSteps(self):
""" Insert several steps needed per iteration. """
for iterN in self.allIters():
dirsStep = self._insertFunctionStep('createIterDirsStep', iterN)
# Insert some steps per reference volume
projMatchSteps = []
for refN in self.allRefs():
# Mask the references in the iteration
insertMaskReferenceStep(self, iterN, refN, prerequisites=[dirsStep])
# Create the library of projections
insertAngularProjectLibraryStep(self, iterN, refN)
# Projection matching steps
projMatchStep = self._insertProjectionMatchingStep(iterN, refN)
projMatchSteps.append(projMatchStep)
# Select the reference that best fits each image
self._insertFunctionStep('assignImagesToReferencesStep', iterN, prerequisites=projMatchSteps)
insertAngularClassAverageStep(self, iterN, refN)
# Reconstruct each reference with new averages
for refN in self.allRefs():
# Create new class averages with images assigned
insertReconstructionStep(self, iterN, refN)
if self.doComputeResolution and self._doSplitReferenceImages[iterN]:
# Reconstruct two halves of the data
insertReconstructionStep(self, iterN, refN, 'Split1')
insertReconstructionStep(self, iterN, refN, 'Split2')
# Compute the resolution
insertComputeResolutionStep(self, iterN, refN)
insertFilterVolumeStep(self, iterN, refN)
# Calculate both angles and shifts devitations for this iteration
self._insertFunctionStep('calculateDeviationsStep', iterN)
def _insertProjectionMatchingStep(self, iterN, refN):
args = getProjectionMatchingArgs(self, iterN)
return self._insertFunctionStep('projectionMatchingStep', iterN, refN, args)
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self):
""" Generated the input particles metadata expected
by projection matching. And copy the generated file to be
used as initial docfile for further iterations.
"""
from pyworkflow.em.packages.xmipp3 import writeSetOfParticles
writeSetOfParticles(self.inputParticles.get(), self.selFileName,
blockName=self.blockWithAllExpImages)
#copyFile(self.selFileName, self._getFileName('inputParticlesDoc'))
def createIterDirsStep(self, iterN):
""" Create the necessary directory for a given iteration. """
iterDirs = [self._getFileName(k, iter=iterN) for k in ['iterDir', 'projMatchDirs', 'libraryDirs']]
for d in iterDirs:
makePath(d)
return iterDirs
def volumeConvertStep(self, reconstructedFilteredVolume, maskedFileName):
runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName)
def executeCtfGroupsStep(self, **kwargs):
runExecuteCtfGroupsStep(self, **kwargs)
def transformMaskStep(self, args, **kwargs):
runTransformMaskStep(self, args, **kwargs)
def angularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs):
runAngularProjectLibraryStep(self, iterN, refN, args, stepParams, **kwargs)
def initAngularReferenceFileStep(self):
runInitAngularReferenceFileStep(self)
def projectionMatchingStep(self, iterN, refN, args):
runProjectionMatching(self, iterN, refN, args)
def assignImagesToReferencesStep(self, iterN):
runAssignImagesToReferences(self, iterN)
def cleanVolumeStep(self, vol1, vol2):
cleanPath(vol1, vol2)
def reconstructionStep(self, iterN, refN, program, method, args, suffix, mpi, threads, **kwargs):
runReconstructionStep(self, iterN, refN, program, method, args, suffix, mpi, threads, **kwargs)
def storeResolutionStep(self, resolIterMd, resolIterMaxMd, sampling):
runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling)
def calculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs):
runCalculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs)
def filterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs):
runFilterVolumeStep(self, iterN, refN, constantToAddToFiltration, **kwargs)
def createOutputStep(self):
runCreateOutputStep(self)
#--------------------------- INFO functions --------------------------------------------
def _validate(self):
errors = []
if self.doCTFCorrection and not self.doAutoCTFGroup and not exists(self.setOfDefocus.get()):
errors.append("Error: for non-automated ctf grouping, please provide a docfile!")
if self.numberOfMpi<=1:
errors.append("The number of MPI processes has to be larger than 1")
xDimImg = self.inputParticles.get().getXDim()
xDimVol, _, _ = self.input3DReferences.get().getDim()
if xDimImg != xDimVol:
errors.append("The dimensions of the volume(s) and particles must be equal!!!!")
return errors
def _citations(self):
cites = []
return cites
def _summary(self):
summary = []
return summary
def _methods(self):
return self._summary() # summary is quite explicit and serve as methods
#--------------------------- UTILS functions --------------------------------------------
def allIters(self):
""" Iterate over all iterations. """
for i in range(1, self.numberOfIterations.get()+1):
yield i
def allRefs(self):
""" Iterate over all references. """
for i in range(1, self.numberOfReferences+1):
yield i
def allCtfGroups(self):
""" Iterate over all CTF groups. """
for i in range(1, self.numberOfCtfGroups.get() + 1):
yield i
def itersFloatValues(self, attributeName, firstValue=-1):
""" Take the string of a given attribute and
create a list of floats that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getFloatListFromValues(valuesStr, length=self.numberOfIterations.get())
def itersBoolValues(self, attributeName, firstValue=False):
""" Take the string of a given attribute and
create a list of booleans that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getBoolListFromValues(valuesStr, length=self.numberOfIterations.get())
def itersStringValues(self, attributeName, firstValue='c1'):
""" Take the string of a given attribute and
create a list of strings that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getStringListFromValues(valuesStr, length=self.numberOfIterations.get())
def _getBlockFileName(self, blockName, blockNumber, filename, length=None):
l = length or self.FILENAMENUMBERLENGTH
return blockName + str(blockNumber).zfill(l) + '@' + filename
def _getExpImagesFileName(self, filename):
return self.blockWithAllExpImages + '@' + filename
def _getRefBlockFileName(self, ctfBlName, ctfBlNumber, refBlName, refBlNumber, filename, length=None):
l = length or self.FILENAMENUMBERLENGTH
return ctfBlName + str(ctfBlNumber).zfill(l) + '_' + refBlName + str(refBlNumber).zfill(l) + '@' + filename
def _getFourierMaxFrequencyOfInterest(self, iterN, refN):
""" Read the corresponding resolution metadata and return the
desired resolution.
"""
md = xmipp.MetaData(self._getFileName('resolutionXmdMax', iter=iterN, ref=refN))
return md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject())
def calculateDeviationsStep(self, it):
""" Calculate both angles and shifts devitations for all iterations
"""
SL = xmipp.SymList()
mdIter = xmipp.MetaData()
#for it in self.allIters():
mdIter.clear()
SL.readSymmetryFile(self._symmetry[it])
md1 = xmipp.MetaData(self.docFileInputAngles[it])
md2 = xmipp.MetaData(self.docFileInputAngles[it-1])
#ignore disabled,
md1.removeDisabled()
md2.removeDisabled()
#first metadata file may not have shiftx and shifty
if not md2.containsLabel(xmipp.MDL_SHIFT_X):
md2.addLabel(xmipp.MDL_SHIFT_X)
md2.addLabel(xmipp.MDL_SHIFT_Y)
md2.fillConstant(xmipp.MDL_SHIFT_X,0.)
md2.fillConstant(xmipp.MDL_SHIFT_Y,0.)
oldLabels=[xmipp.MDL_ANGLE_ROT,
xmipp.MDL_ANGLE_TILT,
xmipp.MDL_ANGLE_PSI,
xmipp.MDL_SHIFT_X,
xmipp.MDL_SHIFT_Y]
newLabels=[xmipp.MDL_ANGLE_ROT2,
xmipp.MDL_ANGLE_TILT2,
xmipp.MDL_ANGLE_PSI2,
xmipp.MDL_SHIFT_X2,
xmipp.MDL_SHIFT_Y2]
md2.renameColumn(oldLabels,newLabels)
md2.addLabel(xmipp.MDL_SHIFT_X_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_DIFF)
mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN)
SL.computeDistance(mdIter,False,False,False)
xmipp.activateMathExtensions()
#operate in sqlite
shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X)
shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2)
shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF)
shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y)
shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2)
shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF)
shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF)
#timeStr = str(dtBegin)
operateString = shiftXDiff+"="+shiftXLabel+"-"+shiftX2Label
operateString += "," + shiftYDiff+"="+shiftYLabel+"-"+shiftY2Label
mdIter.operate(operateString)
operateString = shiftDiff+"=sqrt(" \
+shiftXDiff+"*"+shiftXDiff+"+" \
+shiftYDiff+"*"+shiftYDiff+");"
mdIter.operate(operateString)
iterFile = self._mdDevitationsFn(it)
mdIter.write(iterFile,xmipp.MD_APPEND)
self._setLastIter(it)
def _mdDevitationsFn(self, it):
mdFn = self._getPath('deviations.xmd')
return "iter_%03d@" % it + mdFn
def _setLastIter(self, iterN):
self._lastIter.set(iterN)
self._store(self._lastIter)
def getLastIter(self):
return self._lastIter.get()
def _createItemMatrix(self, item, row):
from pyworkflow.em.packages.xmipp3.convert import createItemMatrix
import pyworkflow.em as em
createItemMatrix(item, row, align=em.ALIGN_PROJ)
class QueueSystemConfig(Object):
def __init__(self, **kwargs):
super().__init__(**kwargs)
self.name = String()
# Number of cores from which the queue is mandatory
# 0 means no mandatory at all
# 1 will force to launch all jobs through the queue
self.mandatory = Integer()
self.queues = None # List for queue configurations
self.submitCommand = String()
# Allow to change the prefix of submission scripts
# we used by default the ID.job, but in some clusters
# the job script should start by a letter
self.submitPrefix = String()
self.checkCommand = String()
self.cancelCommand = String()
self.submitTemplate = String()
self.jobDoneRegex = String()
def hasName(self):
return self.name.hasValue()
def hasValue(self):
return self.hasName() and len(self.queues)
def getName(self):
return self.name.get()
def getMandatory(self):
return self.mandatory.get()
def getSubmitTemplate(self):
return self.submitTemplate.get()
def getSubmitCommand(self):
return self.submitCommand.get()
def getCheckCommand(self):
return self.checkCommand.get()
def getCancelCommand(self):
return self.cancelCommand.get()
def getQueues(self):
return self.queues
def setName(self, name):
self.name.set(name)
def setMandatory(self, mandatory):
# This condition is to be backward compatible
# when mandatory was a boolean
# now it should use the number of CPU
# that should force to use the queue
if mandatory in ['False', 'false']:
mandatory = 0
elif mandatory in ['True', 'true']:
mandatory = 1
self.mandatory.set(mandatory)
def setSubmitTemplate(self, submitTemplate):
self.submitTemplate.set(submitTemplate)
def setSubmitCommand(self, submitCommand):
self.submitCommand.set(submitCommand)
def setCheckCommand(self, checkCommand):
self.checkCommand.set(checkCommand)
def setCancelCommand(self, cancelCommand):
self.cancelCommand.set(cancelCommand)
def setJobDoneRegex(self, jobDoneRegex):
self.jobDoneRegex.set(jobDoneRegex)
def setQueues(self, queues):
self.queues = queues
def getQueueConfig(self, objId):
if objId is not None and self.queues is not None:
for queueConfig in self.queues:
if objId == queueConfig.getObjId():
return queueConfig
return None
class XmippProtProjMatch(ProtRefine3D, ProtClassify3D):
""" 3D reconstruction and classification using multireference projection matching"""
_label = 'projection matching'
FILENAMENUMBERLENGTH = 6
def __init__(self, **args):
ProtRefine3D.__init__(self, **args)
ProtClassify3D.__init__(self, **args)
self.numberOfCtfGroups = Integer(1)
self._lastIter = Integer(0)
def _initialize(self):
""" This function is mean to be called after the
working dir for the protocol have been set. (maybe after recovery from mapper)
"""
self._loadInputInfo()
# Setup the dictionary with filenames templates to
# be used by _getFileName
createFilenameTemplates(self)
# Load the values from several params generating a list
# of values per iteration or references
initializeLists(self)
def _loadInputInfo(self):
from pyworkflow.em.packages.xmipp3 import getImageLocation
reference = self.input3DReferences.get(
) # Input can be either a single volume or a set of volumes.
if isinstance(reference, Volume): # Treat the case of a single volume
self.referenceFileNames = [getImageLocation(reference)]
else:
self.referenceFileNames = [
getImageLocation(vol) for vol in reference
]
self.numberOfReferences = len(self.referenceFileNames)
self.resolSam = reference.getSamplingRate()
#--------------------------- DEFINE param functions --------------------------------------------
def _defineParams(self, form):
""" Since the form definition is very very large,
we have do it in a separated function.
"""
_defineProjectionMatchingParams(self, form)
#--------------------------- INSERT steps functions --------------------------------------------
def _insertAllSteps(self):
self._initialize()
# Insert initial steps
self._insertFunctionStep('convertInputStep')
self._insertFunctionStep('executeCtfGroupsStep')
# insertExecuteCtfGroupsStep(self)
# insertInitAngularReferenceFileStep(self)
self._insertFunctionStep('initAngularReferenceFileStep')
# Steps per iteration
self._insertItersSteps()
# Final steps
self._insertFunctionStep('createOutputStep')
def _insertItersSteps(self):
""" Insert several steps needed per iteration. """
for iterN in self.allIters():
dirsStep = self._insertFunctionStep('createIterDirsStep', iterN)
# Insert some steps per reference volume
projMatchSteps = []
for refN in self.allRefs():
# Mask the references in the iteration
insertMaskReferenceStep(self,
iterN,
refN,
prerequisites=[dirsStep])
# Create the library of projections
insertAngularProjectLibraryStep(self, iterN, refN)
# Projection matching steps
projMatchStep = self._insertProjectionMatchingStep(iterN, refN)
projMatchSteps.append(projMatchStep)
# Select the reference that best fits each image
self._insertFunctionStep('assignImagesToReferencesStep',
iterN,
prerequisites=projMatchSteps)
insertAngularClassAverageStep(self, iterN, refN)
# Reconstruct each reference with new averages
for refN in self.allRefs():
# Create new class averages with images assigned
insertReconstructionStep(self, iterN, refN)
if self.doComputeResolution and self._doSplitReferenceImages[
iterN]:
# Reconstruct two halves of the data
insertReconstructionStep(self, iterN, refN, 'Split1')
insertReconstructionStep(self, iterN, refN, 'Split2')
# Compute the resolution
insertComputeResolutionStep(self, iterN, refN)
insertFilterVolumeStep(self, iterN, refN)
# Calculate both angles and shifts devitations for this iteration
self._insertFunctionStep('calculateDeviationsStep', iterN)
def _insertProjectionMatchingStep(self, iterN, refN):
args = getProjectionMatchingArgs(self, iterN)
return self._insertFunctionStep('projectionMatchingStep', iterN, refN,
args)
#--------------------------- STEPS functions --------------------------------------------
def convertInputStep(self):
""" Generated the input particles metadata expected
by projection matching. And copy the generated file to be
used as initial docfile for further iterations.
"""
from pyworkflow.em.packages.xmipp3 import writeSetOfParticles
writeSetOfParticles(self.inputParticles.get(),
self.selFileName,
blockName=self.blockWithAllExpImages)
#copyFile(self.selFileName, self._getFileName('inputParticlesDoc'))
def createIterDirsStep(self, iterN):
""" Create the necessary directory for a given iteration. """
iterDirs = [
self._getFileName(k, iter=iterN)
for k in ['iterDir', 'projMatchDirs', 'libraryDirs']
]
for d in iterDirs:
makePath(d)
return iterDirs
def volumeConvertStep(self, reconstructedFilteredVolume, maskedFileName):
runVolumeConvertStep(self, reconstructedFilteredVolume, maskedFileName)
def executeCtfGroupsStep(self, **kwargs):
runExecuteCtfGroupsStep(self, **kwargs)
def transformMaskStep(self, program, args, **kwargs):
runTransformMaskStep(self, program, args, **kwargs)
def angularProjectLibraryStep(self, iterN, refN, args, stepParams,
**kwargs):
runAngularProjectLibraryStep(self, iterN, refN, args, stepParams,
**kwargs)
def initAngularReferenceFileStep(self):
runInitAngularReferenceFileStep(self)
def projectionMatchingStep(self, iterN, refN, args):
runProjectionMatching(self, iterN, refN, args)
def assignImagesToReferencesStep(self, iterN):
runAssignImagesToReferences(self, iterN)
def cleanVolumeStep(self, vol1, vol2):
cleanPath(vol1, vol2)
def reconstructionStep(self, iterN, refN, program, method, args, suffix,
**kwargs):
runReconstructionStep(self, iterN, refN, program, method, args, suffix,
**kwargs)
def storeResolutionStep(self, resolIterMd, resolIterMaxMd, sampling):
runStoreResolutionStep(self, resolIterMd, resolIterMaxMd, sampling)
def calculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs):
runCalculateFscStep(self, iterN, refN, args, constantToAdd, **kwargs)
def filterVolumeStep(self, iterN, refN, constantToAddToFiltration,
**kwargs):
runFilterVolumeStep(self, iterN, refN, constantToAddToFiltration,
**kwargs)
def createOutputStep(self):
runCreateOutputStep(self)
#--------------------------- INFO functions --------------------------------------------
def _validate(self):
errors = []
if self.doCTFCorrection:
if not self.doAutoCTFGroup and not exists(self.setOfDefocus.get()):
errors.append("Error: for non-automated ctf grouping, "
"please provide a docfile!")
if not self.inputParticles.get().hasCTF():
errors.append("Error: for doing CTF correction the input "
"particles should have CTF information.")
if self.numberOfMpi <= 1:
errors.append(
"The number of MPI processes has to be larger than 1")
self._validateDim(self.inputParticles.get(),
self.input3DReferences.get(), errors,
'Input particles', 'Reference volume')
return errors
def _citations(self):
cites = []
return cites
def _summary(self):
summary = []
return summary
def _methods(self):
return self._summary(
) # summary is quite explicit and serve as methods
#--------------------------- UTILS functions --------------------------------------------
def allIters(self):
""" Iterate over all iterations. """
for i in range(1, self.numberOfIterations.get() + 1):
yield i
def allRefs(self):
""" Iterate over all references. """
for i in range(1, self.numberOfReferences + 1):
yield i
def allCtfGroups(self):
""" Iterate over all CTF groups. """
for i in range(1, self.numberOfCtfGroups.get() + 1):
yield i
def itersFloatValues(self, attributeName, firstValue=-1):
""" Take the string of a given attribute and
create a list of floats that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getFloatListFromValues(
valuesStr, length=self.numberOfIterations.get())
def itersBoolValues(self, attributeName, firstValue=False):
""" Take the string of a given attribute and
create a list of booleans that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getBoolListFromValues(
valuesStr, length=self.numberOfIterations.get())
def itersStringValues(self, attributeName, firstValue='c1'):
""" Take the string of a given attribute and
create a list of strings that will be used by
the iteratioins. An special first value will be
added to the list for iteration 0.
"""
valuesStr = self.getAttributeValue(attributeName)
if valuesStr is None:
raise Exception('None value for attribute: %s' % attributeName)
return [firstValue] + getStringListFromValues(
valuesStr, length=self.numberOfIterations.get())
def _getBlockFileName(self, blockName, blockNumber, filename, length=None):
l = length or self.FILENAMENUMBERLENGTH
return blockName + str(blockNumber).zfill(l) + '@' + filename
def _getExpImagesFileName(self, filename):
return self.blockWithAllExpImages + '@' + filename
def _getRefBlockFileName(self,
ctfBlName,
ctfBlNumber,
refBlName,
refBlNumber,
filename,
length=None):
l = length or self.FILENAMENUMBERLENGTH
return ctfBlName + str(ctfBlNumber).zfill(l) + '_' + refBlName + str(
refBlNumber).zfill(l) + '@' + filename
def _getFourierMaxFrequencyOfInterest(self, iterN, refN):
""" Read the corresponding resolution metadata and return the
desired resolution.
"""
md = xmipp.MetaData(
self._getFileName('resolutionXmdMax', iter=iterN, ref=refN))
return md.getValue(xmipp.MDL_RESOLUTION_FREQREAL, md.firstObject())
def calculateDeviationsStep(self, it):
""" Calculate both angles and shifts devitations for all iterations
"""
SL = xmipp.SymList()
mdIter = xmipp.MetaData()
#for it in self.allIters():
mdIter.clear()
SL.readSymmetryFile(self._symmetry[it])
md1 = xmipp.MetaData(self.docFileInputAngles[it])
md2 = xmipp.MetaData(self.docFileInputAngles[it - 1])
#ignore disabled,
md1.removeDisabled()
md2.removeDisabled()
#first metadata file may not have shiftx and shifty
if not md2.containsLabel(xmipp.MDL_SHIFT_X):
md2.addLabel(xmipp.MDL_SHIFT_X)
md2.addLabel(xmipp.MDL_SHIFT_Y)
md2.fillConstant(xmipp.MDL_SHIFT_X, 0.)
md2.fillConstant(xmipp.MDL_SHIFT_Y, 0.)
oldLabels = [
xmipp.MDL_ANGLE_ROT, xmipp.MDL_ANGLE_TILT, xmipp.MDL_ANGLE_PSI,
xmipp.MDL_SHIFT_X, xmipp.MDL_SHIFT_Y
]
newLabels = [
xmipp.MDL_ANGLE_ROT2, xmipp.MDL_ANGLE_TILT2, xmipp.MDL_ANGLE_PSI2,
xmipp.MDL_SHIFT_X2, xmipp.MDL_SHIFT_Y2
]
md2.renameColumn(oldLabels, newLabels)
md2.addLabel(xmipp.MDL_SHIFT_X_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_Y_DIFF)
md2.addLabel(xmipp.MDL_SHIFT_DIFF)
mdIter.join1(md1, md2, xmipp.MDL_IMAGE, xmipp.INNER_JOIN)
SL.computeDistance(mdIter, False, False, False)
xmipp.activateMathExtensions()
#operate in sqlite
shiftXLabel = xmipp.label2Str(xmipp.MDL_SHIFT_X)
shiftX2Label = xmipp.label2Str(xmipp.MDL_SHIFT_X2)
shiftXDiff = xmipp.label2Str(xmipp.MDL_SHIFT_X_DIFF)
shiftYLabel = xmipp.label2Str(xmipp.MDL_SHIFT_Y)
shiftY2Label = xmipp.label2Str(xmipp.MDL_SHIFT_Y2)
shiftYDiff = xmipp.label2Str(xmipp.MDL_SHIFT_Y_DIFF)
shiftDiff = xmipp.label2Str(xmipp.MDL_SHIFT_DIFF)
#timeStr = str(dtBegin)
operateString = shiftXDiff + "=" + shiftXLabel + "-" + shiftX2Label
operateString += "," + shiftYDiff + "=" + shiftYLabel + "-" + shiftY2Label
mdIter.operate(operateString)
operateString = shiftDiff+"=sqrt(" \
+shiftXDiff+"*"+shiftXDiff+"+" \
+shiftYDiff+"*"+shiftYDiff+");"
mdIter.operate(operateString)
iterFile = self._mdDevitationsFn(it)
mdIter.write(iterFile, xmipp.MD_APPEND)
self._setLastIter(it)
def _mdDevitationsFn(self, it):
mdFn = self._getPath('deviations.xmd')
return "iter_%03d@" % it + mdFn
def _setLastIter(self, iterN):
self._lastIter.set(iterN)
self._store(self._lastIter)
def getLastIter(self):
return self._lastIter.get()
def _fillParticlesFromIter(self, partSet, iteration):
print("_fillParticlesFromIter")
import pyworkflow.em.metadata as md
imgSet = self.inputParticles.get()
imgFn = "all_exp_images@" + self._getFileName(
'docfileInputAnglesIters', iter=iteration, ref=1)
partSet.copyInfo(imgSet)
partSet.setAlignmentProj()
partSet.copyItems(imgSet,
updateItemCallback=self._createItemMatrix,
itemDataIterator=md.iterRows(
imgFn, sortByLabel=md.MDL_ITEM_ID))
def _createItemMatrix(self, item, row):
from pyworkflow.em.packages.xmipp3.convert import createItemMatrix
import pyworkflow.em as em
createItemMatrix(item, row, align=em.ALIGN_PROJ)
def _getIterParticles(self, it, clean=False):
import pyworkflow.em as em
""" Return a classes .sqlite file for this iteration.
If the file doesn't exists, it will be created by
converting from this iteration data.star file.
"""
dataParticles = self._getFileName('particlesScipion', iter=it)
if clean:
cleanPath(dataParticles)
if not exists(dataParticles):
partSet = em.SetOfParticles(filename=dataParticles)
self._fillParticlesFromIter(partSet, it)
partSet.write()
partSet.close()
else:
partSet = em.SetOfParticles(filename=dataParticles)
imgSet = self.inputParticles.get()
partSet.copyInfo(imgSet)
partSet.setAlignmentProj()
return partSet
class ProtAnnotateMembranes(EMProtocol):
""" Manual annotation tool for segmented membranes
"""
_label = 'annotate segmented membranes'
_devStatus = BETA
def __init__(self, **kwargs):
EMProtocol.__init__(self, **kwargs)
self._objectsToGo = Integer()
self._provider = None
self._tomoList = None
def _defineParams(self, form):
form.addSection(label='Input')
form.addParam(
'inputTomoMasks',
PointerParam,
label="Input Tomo Masks",
important=True,
pointerClass='SetOfTomoMasks',
allowsNull=False,
help=
'Select the Tomogram Masks (segmented tomograms) for the membrane annotation.'
)
# --------------------------- INSERT steps functions ----------------------
def _insertAllSteps(self):
self._initialize()
self._insertFunctionStep(self.runMembraneAnnotator, interactive=True)
# --------------------------- STEPS functions -----------------------------
def runMembraneAnnotator(self):
# There are still some objects which haven't been annotated --> launch GUI
self._getAnnotationStatus()
if self._objectsToGo.get() > 0:
MembAnnotatorDialog(None,
self._getExtraPath(),
provider=self._provider,
prot=self)
# All the objetcs have been annotated --> create output objects
self._getAnnotationStatus()
if self._objectsToGo.get() == 0:
print("\n==> Generating the outputs")
labelledSet = self._genOutputSetOfTomoMasks()
self._defineOutputs(outputSetofTomoMasks=labelledSet)
self._store()
# --------------------------- INFO functions -----------------------------------
def _summary(self):
summary = []
objects2go = self._objectsToGo.get()
if objects2go is not None:
if objects2go > 0:
summary.append(
'*%i* remaining segmentations to be annotated.' %
objects2go)
else:
summary.append(
'All segmentations have been already annotated.')
return summary
# --------------------------- UTIL functions -----------------------------------
def _initialize(self):
self._tomoList = [
tomo.clone() for tomo in self.inputTomoMasks.get().iterItems()
]
self._provider = MembAnnotatorProvider(self._tomoList,
self._getExtraPath(),
'membAnnotator')
self._getAnnotationStatus()
def _getAnnotationStatus(self):
"""Check if all the tomo masks have been annotated and store current status in a text file"""
doneTomes = [
self._provider.getObjectInfo(tomo)['tags'] == 'done'
for tomo in self._tomoList
]
self._objectsToGo.set(len(self._tomoList) - sum(doneTomes))
def _getCurrentTomoMaskFile(self, inTomoFile):
baseName = removeBaseExt(inTomoFile)
return glob.glob(self._getExtraPath(baseName + '_materials.mrc'))[0]
def _genOutputSetOfTomoMasks(self):
tomoMaskSet = SetOfTomoMasks.create(self._getPath(),
template='tomomasks%s.sqlite',
suffix='annotated')
inTomoSet = self.inputTomoMasks.get()
tomoMaskSet.copyInfo(inTomoSet)
counter = 1
for inTomo in inTomoSet.iterItems():
tomoMask = TomoMask()
inTomoFile = inTomo.getVolName()
tomoMask.copyInfo(inTomo)
tomoMask.setLocation(
(counter, self._getCurrentTomoMaskFile(inTomoFile)))
tomoMask.setVolName(inTomoFile)
tomoMaskSet.append(tomoMask)
counter += 1
return tomoMaskSet
