def writeSetOfVolumes(volSet, volXml, volDir):
""" Convert a SetOfVolumes to a xml file used by PyTom.
The volumes will be converted to .mrc format if not are '.em' or '.mrc'
Params:
volSet: input SetOfVolumes.
volXml: filename where to write the xml file.
volDir: where to create links or copies (converted to mrc).
"""
# Add to the path the root to pytom
backupPath = list(sys.path)
addPyTomPaths()
from pytom.basic.structures import Particle, ParticleList, Wedge, SingleTiltWedge
from pytom.score.score import Score, PeakPrior, xcfScore
from pytom.frm.FRMAlignment import FRMScore
w = SingleTiltWedge()
#s = PeakPrior()
pl = ParticleList()
ih = em.convert.ImageHandler()
for vol in volSet:
index, fn = vol.getLocation()
convert = True # by default, convert, which is the save way
if index == em.NO_INDEX: # means single volumes
volName = os.path.basename(fn)
if fn.endswith('.em') or fn.endswith('.mrc'):
convert = False # we can just create a link in this case
else:
volName = 'volume_%03d.mrc' % vol.getObjId()
volFn = os.path.join(volDir, volName)
if convert:
ih.convert(vol, volFn)
else:
pwutils.createLink(fn, volFn)
# Make the volumes names relative to the xml file
# where the programs will be executed
volRel = os.path.relpath(volFn, os.path.dirname(volXml))
p = Particle()
s = xcfScore()
s.setValue(1.0)
pytomInfo = getattr(vol, 'pytomInfo', None)
if pytomInfo is None:
p.setWedge(w)
else:
p.fromXML(pytomInfo.get()) # Get stored XML format from PyTom
p.setFilename(volRel)
p.setScore(s)
pl.append(p)
pl.toXMLFile(volXml)
#pl.setWedgeAllParticles(w)
sys.path = backupPath
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to result attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'Is not a lxml.etree._Element! You must provide a valid XML object.'
)
from pytom.score.score import fromXML as fromXMLScore
from pytom.angles.angle import AngleObject
if xmlObj.tag == "Result":
result = xmlObj
else:
result = xmlObj.xpath('Result')
if len(result) == 0:
raise PyTomClassError(
"This XML is not an MaximisationResult. No Result provided."
)
result = result[0]
from pytom.basic.structures import Particle, Reference, Rotation, Shift
particle_element = result.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self._particle = p
r = result.xpath('Reference')
ref = Reference('')
ref.fromXML(r[0])
self._reference = ref
scoreXML = result.xpath('Score')[0]
self._score = fromXMLScore(scoreXML)
shiftXML = result.xpath('Shift')[0]
self._shift = Shift()
self._shift.fromXML(shiftXML)
rotationXML = result.xpath('Rotation')[0]
self._rotation = Rotation()
self._rotation.fromXML(rotationXML)
angleElement = result.xpath('Angles')
ang = AngleObject()
self._angleObject = ang.fromXML(angleElement[0])
def fromXML(self, xmlObj):
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError('You must provide a valid XML object.')
if xmlObj.tag == "ParticleList":
main = xmlObj
else:
main = xmlObj.xpath('ParticleList')
if len(main) == 0:
from pytom.basic.exceptions import PyTomClassError
raise PyTomClassError("This XML is not a ParticleList.")
main = main[0]
for p in main.xpath('FoundParticle'):
particle = FoundParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('ClassifiedParticle'):
particle = ClassifiedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('SimulatedParticle'):
from pytom.localization.simulation import SimulatedParticle
particle = SimulatedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('IdentifiedParticle'):
from pytom.localization.simulation import IdentifiedParticle
particle = IdentifiedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('Particle'): # now also support reading class Particle, but will convert to FoundParticle
from pytom.basic.structures import Particle
pp = Particle()
pp.fromXML(p)
# convert to FoundParticle
particle = FoundParticle()
particle.fromParticle(pp)
self.pl.append(particle)
class GrowingAverageInterimResult(PyTomClass):
"""
GrowingAverageInterimResult:
"""
def __init__(self,particle,reference,rotation,shift,score):
self.particle = particle
self.reference = reference
self.rotation = rotation
self.shift = shift
self.score = score
def getFilename(self):
return self.particle.getFilename()
def getWedgeInfo(self):
return self.particle.getWedgeInfo()
def getRotation(self):
return self.rotation
def getShift(self):
return self.shift
def toXML(self):
"""
toXML : Compiles a XML file from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
result_element = etree.Element('GrowingAverageInterimResult')
result_element.append(self.particle.toXML())
result_element.append(self.reference.toXML())
result_element.append(self.rotation.toXML())
result_element.append(self.shift.toXML())
result_element.append(self.score.toXML())
return result_element
def fromXML(self,xmlObj):
"""
fromXML:
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element :
raise Exception('Is not a lxml.etree._Element! You must provide a valid XMLobject.')
if xmlObj.tag == 'GrowingAverageInterimResult':
result_element = xmlObj
else:
Exception('XML object is not a GrowingAverageInterimResult! You must provide a valid GrowingAverageInterimResultXML object.')
from pytom.basic.structures import Particle,Reference
particleXML = result_element.xpath('/GrowingAverageInterimResult/Particle')[0]
self.particle = Particle('')
self.particle.fromXML(particleXML)
referenceXML = result_element.xpath('/GrowingAverageInterimResult/Result')[0]
self.reference = Reference('')
self.reference.fromXML(referenceXML)
class CorrelationVectorJob(PyTomClass):
"""
CorrelationVectorJob: All settings needed for a correlation vector job. Explore class for more info.
"""
def __init__(self,particle=None,particleList=None,mask=None,particleIndex = None,applyWedge = True,binningFactor=0,lowestFrequency=-1,highestFrequency=-1):
"""
__init__:
@param particle: Particle
@type particle: pytom.alignment.structures.Particle
@param particleList: ParticleList of all particles will be correlated with self._particle
@type particleList: pytom.alignment.structures.ParticleList
@param mask: Mask used for correlation
@type mask: str
@param applyWedge: Apply wedge during correlation. True by default, disable for rotation classification.
@type applyWedge: Bool
@param binningFactor: Binning factor accroding to libtomc definition. 0 by default.
@type binningFactor: unsigned int
@param lowestFrequency: Lowest frequency for bandpass in nyquist
@type lowestFrequency: float
@param highestFrequency: Highest frequency for bandpass in nyquist
@type highestFrequency: float
"""
from pytom.basic.structures import Particle,ParticleList,Mask
if particle and particle.__class__ != Particle:
raise ParameterError('You must provide a Particle object.')
if particleList and particleList.__class__ != ParticleList:
raise ParameterError('You must provide a ParticleList object.')
self._particle = particle
self._particleList = particleList
if not mask:
mask = Mask()
elif mask.__class__ == str:
mask = Mask(mask)
elif mask.__class__ != Mask:
from pytom.basic.exceptions import ParameterError
raise ParameterError('Mask must be a string or Mask object!')
self._mask = mask
self._particleIndex = particleIndex
self._applyWedge = applyWedge
self._binningFactor = binningFactor
self._lowestFrequency = lowestFrequency
self._highestFrequency = highestFrequency
def getMask(self):
return self._mask
def getParticle(self):
return self._particle
def getParticleList(self):
return self._particleList
def getParticleIndex(self):
return self._particleIndex
def getApplyWedge(self):
return self._applyWedge
def getBinning(self):
return self._binningFactor
def getLowestFrequency(self):
return self._lowestFrequency
def getHighestFrequency(self):
return self._highestFrequency
def toXML(self):
"""
toXML : Compiles a XML object from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
jobElement = etree.Element('CorrelationVectorJob',ParticleIndex = self._particleIndex.__str__(),ApplyWedge=self._applyWedge.__str__(),Binning = self._binningFactor.__str__(),LowestFrequency = str(self._lowestFrequency),HighestFrequency = str(self._highestFrequency))
jobElement.append(self._particle.toXML())
jobElement.append(self._particleList.toXML())
jobElement.append(self._mask.toXML())
return jobElement
def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVectorJob object.')
from pytom.basic.structures import Particle,ParticleList,Mask
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
maskObject = xmlObj.xpath('Mask')[0]
self._mask = Mask()
self._mask.fromXML(maskObject)
self._particleIndex = xmlObj.get('ParticleIndex')
self._applyWedge = xmlObj.get('ApplyWedge') == 'True'
self._binningFactor = float(xmlObj.get('Binning'))
self._lowestFrequency = float(xmlObj.get('LowestFrequency'))
self._highestFrequency = float(xmlObj.get('HighestFrequency'))
class CorrelationVector(PyTomClass):
"""
CorrelationVector: Stores a vector of correlation values of one particle with many others from particle list
@todo: Add unittest
"""
def __init__(self,particle=None,particleList=None,particleIndex=None):
self._correlations = []
self._particle = particle
self._particleList = particleList
self._particleIndex = particleIndex
def append(self,value):
self._correlations.append(value)
def getParticleIndex(self):
return self._particleIndex
def __len__(self):
return len(self._correlations)
def __getitem__(self,key):
"""
"""
if isinstance(key, int):
if key < len(self):
return self._correlations[key]
else:
raise IndexError('Index out of range.')
else:
assert False
def toXML(self):
"""
toXML : Compiles a XML file from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
vectorElement = etree.Element("CorrelationVector",ParticleIndex = self._particleIndex.__str__())
vectorElement.append(self._particle.toXML())
vectorElement.append(self._particleList.toXML())
for i in range(len(self._correlations)):
valueElement = etree.Element("Correlation",Index=i.__str__(),Value=self._correlations[i].__str__())
vectorElement.append(valueElement)
return vectorElement
def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element,tostring
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVector object.')
from pytom.basic.structures import Particle,ParticleList
self._particleIndex = int(xmlObj.get('ParticleIndex'))
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
values = xmlObj.xpath('Correlation')
self._correlations = [0 for _ in range(len(values))]
for v in values:
index = int(v.get('Index'))
self._correlations[index] = float(v.get('Value'))
def createClassificationResultDictionaries(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: A dictionary that maps new classnames to groundTruthClasses
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleList)
newClassesToGroundTruthMap = {
} #maps new class name to a ground truth class name
gtClassNames = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNames[particle.getClassName()] = False
if verbose:
print('gtClassNames : ', gtClassNames)
groundTruthParticleListXML = groundTruthParticleList.toXML()
for newClass in classifiedParticleList.splitByClass():
if verbose:
print('')
print('newClass : ', newClass)
particlesFromGroundTruth = ParticleList('/')
for particle in newClass:
#collect all particles that were assigned to newClass
particleName = particle.getFilename()
#gtParticle = groundTruthParticleList.getParticleByFilename(particleName)
particleXML = groundTruthParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' + str(particleName) +
'"]')
gtParticle = Particle('a')
gtParticle.fromXML(particleXML[0])
particlesFromGroundTruth.append(gtParticle)
if verbose:
print('len(particlesFromGroundTruth) : ',
len(particlesFromGroundTruth))
#sort classes according to size to descending order
sortedClasses = sorted(particlesFromGroundTruth.splitByClass(),
key=lambda x: len(x),
reverse=True)
classWasAssigned = False
classIndex = 0
if verbose:
print('len(sortedClasses) : ', len(sortedClasses))
while not classWasAssigned and classIndex < len(sortedClasses):
sortedClass = sortedClasses[classIndex]
className = sortedClass[0].getClassName()
if verbose:
print('className : ' + className)
print('len(sortedClass) : ', len(sortedClass))
classWasAssigned = not gtClassNames[className]
if verbose:
print('classWasAssigned : ', classWasAssigned)
if not classWasAssigned:
classIndex = classIndex + 1
else:
gtClassNames[className] = True
newClassesToGroundTruthMap[
newClass[0].getClassName()] = className
if verbose:
print('gtClassNames : ', gtClassNames)
print('newClassesToGroundTruthMap : ',
newClassesToGroundTruthMap)
return newClassesToGroundTruthMap
def assessClassification(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: [trueHits,falseHits,trueHitsPercent,falseHitsPercent,numberClusters,[clusterSizes]]
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleListFile = groundTruthParticleList
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleListFile)
gtClassNamesAssigned = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNamesAssigned[particle.getClassName()] = False
if verbose:
print('GT Classes ', gtClassNamesAssigned)
gtClassesPerClass = {}
newClasses = classifiedParticleList.splitByClass()
newClassNamesAssigned = {}
numberClasses = len(newClasses)
classSizes = []
for i in range(len(newClasses)):
classSizes.append(len(newClasses[i]))
for newClass in newClasses:
newClassParticleList = ParticleList(newClass.getDirectory())
newClassNamesAssigned[newClass[0].getClassName()] = False
for particle in newClass:
pp = groundTruthParticleList[particle.getFilename()]
newClassParticleList.append(pp)
gtClassSizeDictionary = {}
for gtClass in newClassParticleList.splitByClass():
particle = gtClass[0]
gtParticle = groundTruthParticleList[particle.getFilename()]
gtClassSizeDictionary[gtParticle.getClassName()] = len(gtClass)
gtClassesPerClass[newClass[0].getClassName()] = [
newClassParticleList, gtClassSizeDictionary
]
if verbose:
print('Class distribution dictionary')
for k in list(gtClassesPerClass.keys()):
print(k, gtClassesPerClass[k])
gtToClassDictionary = {}
for gtName in list(gtClassNamesAssigned.keys()):
newClassIndex = 0
classSizeList = []
largestClass = -1
maxClassName = 'unknown'
assigned = False
for newClassName in list(gtClassesPerClass.keys()):
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
if verbose:
print('GT Name', gtName, ' New Class Name', newClassName)
print('GT Name Size', gtClassSizeDictionary)
try:
if verbose:
print(gtClassSizeDictionary[gtName])
if largestClass < gtClassSizeDictionary[
gtName] and not newClassNamesAssigned[newClassName]:
largestClass = gtClassSizeDictionary[gtName]
maxClassName = newClassName
if verbose:
print('SWAP')
except KeyError:
pass
gtToClassDictionary[gtName] = maxClassName
newClassNamesAssigned[maxClassName] = True
gtClassNamesAssigned[gtName] = True
for newClassName in list(gtClassesPerClass.keys()):
try:
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
del gtClassSizeDictionary[gtName]
l[1] = gtClassSizeDictionary
gtClassesPerClass[newClassName] = l
except KeyError:
pass
if verbose:
print('GT to New Dictionary')
print(gtToClassDictionary)
trueHits = 0
falseHits = 0
classifiedParticleListXML = classifiedParticleList.toXML()
for gtParticle in groundTruthParticleList:
particleXML = classifiedParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' +
str(gtParticle.getFilename()) + '"]')
particle = Particle('a')
particle.fromXML(particleXML[0])
if particle.getClassName() == gtToClassDictionary[
gtParticle.getClassName()]:
trueHits += 1
else:
falseHits += 1
return [
trueHits, falseHits,
float(trueHits) / len(classifiedParticleList),
float(falseHits) / len(classifiedParticleList), numberClasses,
classSizes
]
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'You must provide a valid XML-MaximisationJob object.')
if xmlObj.tag == "JobDescription":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('JobDescription')
if len(jobDescription) == 0:
from pytom.basic.structures import PyTomClassError
raise PyTomClassError("This XML is not an JobDescription.")
jobDescription = jobDescription[0]
from pytom.angles.angle import AngleObject
from pytom.score.score import fromXML as fromXMLScore
from pytom.alignment.preprocessing import Preprocessing
from pytom.basic.structures import Mask, Particle, Reference, ReferenceList
self.binning = int(jobDescription.get('Binning'))
particle_element = jobDescription.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self.particle = p
r = jobDescription.xpath('Reference')
if len(r) > 0:
ref = Reference('')
ref.fromXML(r[0])
self.reference = ref
else:
r = jobDescription.xpath('ReferenceList')
ref = ReferenceList()
ref.fromXML(r[0])
self.reference = ref
mask = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(mask)
self.numberRefinementRounds = jobDescription.get(
'NumberRefinementRounds')
self.numberRefinementRounds = int(self.numberRefinementRounds)
score = jobDescription.xpath('Score')
self.score = fromXMLScore(score[0])
angles = jobDescription.xpath('Angles')
ang = AngleObject()
self.rotations = ang.fromXML(angles[0])
preObj = xmlObj.xpath('Preprocessing')
if len(preObj) == 0:
self.preprocessing = Preprocessing()
else:
p = Preprocessing()
p.fromXML(preObj[0])
self.preprocessing = p
class ClusterSwap(PyTomClass):
"""
ClusterSwap: Saves conditions under which a class swap of a particle happened.
"""
def __init__(self,
particle=None,
oldClusterName=None,
oldScore=None,
newClusterName=None,
newScore=None):
self._particle = particle
self._oldClusterName = oldClusterName
self._oldScore = oldScore
self._newClusterName = newClusterName
self._newScore = newScore
def toXML(self):
from lxml import etree
swapElement = etree.Element('ClassSwap',
OldClusterName=str(self._oldClusterName),
NewClusterName=str(self._newClusterName))
swapElement.append(self._particle.toXML())
os = self._oldScore.toXML()
os.set('OriginalType', str(os.tag))
os.tag = 'OldScore'
swapElement.append(os)
ns = self._newScore.toXML()
ns.set('OriginalType', str(ns.tag))
ns.tag = 'NewScore'
swapElement.append(ns)
return swapElement
def fromXML(self, xmlObj):
from lxml.etree import _Element
from pytom.basic.structures import Particle
from pytom.score.score import fromXML as scoreFromXML
if xmlObj.__class__ != _Element:
raise Exception(
'Is not a lxml.etree._Element! You must provide a valid XMLobject.'
)
self._oldClusterName = str(xmlObj.get('OldClusterName'))
self._newClusterName = str(xmlObj.get('NewClusterName'))
oldScore = xmlObj.xpath('OldScore')[0]
oldScore.tag = oldScore.get('OriginalType')
self._oldScore = scoreFromXML(oldScore)
newScore = xmlObj.xpath('NewScore')[0]
newScore.tag = newScore.get('OriginalType')
self._newScore = scoreFromXML(newScore)
particle = xmlObj.xpath('Particle')[0]
self._particle = Particle('')
self._particle.fromXML(particle)