def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVectorJob object.')
from pytom.basic.structures import Particle,ParticleList,Mask
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
maskObject = xmlObj.xpath('Mask')[0]
self._mask = Mask()
self._mask.fromXML(maskObject)
self._particleIndex = xmlObj.get('ParticleIndex')
self._applyWedge = xmlObj.get('ApplyWedge') == 'True'
self._binningFactor = float(xmlObj.get('Binning'))
self._lowestFrequency = float(xmlObj.get('LowestFrequency'))
self._highestFrequency = float(xmlObj.get('HighestFrequency'))
def fromXML(self, xmlObj):
from lxml.etree import _Element
from pytom.basic.structures import Particle
from pytom.score.score import fromXML as scoreFromXML
if xmlObj.__class__ != _Element:
raise Exception(
'Is not a lxml.etree._Element! You must provide a valid XMLobject.'
)
self._oldClusterName = str(xmlObj.get('OldClusterName'))
self._newClusterName = str(xmlObj.get('NewClusterName'))
oldScore = xmlObj.xpath('OldScore')[0]
oldScore.tag = oldScore.get('OriginalType')
self._oldScore = scoreFromXML(oldScore)
newScore = xmlObj.xpath('NewScore')[0]
newScore.tag = newScore.get('OriginalType')
self._newScore = scoreFromXML(newScore)
particle = xmlObj.xpath('Particle')[0]
self._particle = Particle('')
self._particle.fromXML(particle)
def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element,tostring
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVector object.')
from pytom.basic.structures import Particle,ParticleList
self._particleIndex = int(xmlObj.get('ParticleIndex'))
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
values = xmlObj.xpath('Correlation')
self._correlations = [0 for _ in range(len(values))]
for v in values:
index = int(v.get('Index'))
self._correlations[index] = float(v.get('Value'))
def fromXML(self,xmlObj):
"""
fromXML:
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element :
raise Exception('Is not a lxml.etree._Element! You must provide a valid XMLobject.')
if xmlObj.tag == 'GrowingAverageInterimResult':
result_element = xmlObj
else:
Exception('XML object is not a GrowingAverageInterimResult! You must provide a valid GrowingAverageInterimResultXML object.')
from pytom.basic.structures import Particle,Reference
particleXML = result_element.xpath('/GrowingAverageInterimResult/Particle')[0]
self.particle = Particle('')
self.particle.fromXML(particleXML)
referenceXML = result_element.xpath('/GrowingAverageInterimResult/Result')[0]
self.reference = Reference('')
self.reference.fromXML(referenceXML)
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to result attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'Is not a lxml.etree._Element! You must provide a valid XML object.'
)
from pytom.score.score import fromXML as fromXMLScore
from pytom.angles.angle import AngleObject
if xmlObj.tag == "Result":
result = xmlObj
else:
result = xmlObj.xpath('Result')
if len(result) == 0:
raise PyTomClassError(
"This XML is not an MaximisationResult. No Result provided."
)
result = result[0]
from pytom.basic.structures import Particle, Reference, Rotation, Shift
particle_element = result.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self._particle = p
r = result.xpath('Reference')
ref = Reference('')
ref.fromXML(r[0])
self._reference = ref
scoreXML = result.xpath('Score')[0]
self._score = fromXMLScore(scoreXML)
shiftXML = result.xpath('Shift')[0]
self._shift = Shift()
self._shift.fromXML(shiftXML)
rotationXML = result.xpath('Rotation')[0]
self._rotation = Rotation()
self._rotation.fromXML(rotationXML)
angleElement = result.xpath('Angles')
ang = AngleObject()
self._angleObject = ang.fromXML(angleElement[0])
def __init__(self,
particle='',
reference=-1.0,
score=-1.0,
shift=-1.0,
rotation=-1.0,
angleObject=-1):
from pytom.basic.structures import Particle, Reference, Shift, Rotation
from numpy import long
if particle.__class__ == str:
self._particle = Particle(particle)
elif particle.__class__ == Particle:
self._particle = particle
else:
self._particle = Particle()
if reference.__class__ == str:
self._reference = Reference(reference)
elif reference.__class__ == Reference:
self._reference = reference
else:
self._reference = Reference()
if shift.__class__ == list:
self._shift = Shift(shift)
elif shift.__class__ == float:
self._shift = Shift()
else:
self._shift = shift
if rotation.__class__ == list:
self._rotation = Rotation(rotation)
elif rotation.__class__ == Rotation:
self._rotation = rotation
else:
self._rotation = Rotation()
if score.__class__ == float:
from pytom.score.score import xcfScore
self._score = xcfScore()
else:
self._score = score
if angleObject.__class__ == float or isinstance(
angleObject, (int, long)):
from pytom.angles.angleList import AngleList
self._angleObject = AngleList()
else:
self._angleObject = angleObject
def writeSetOfVolumes(volSet, volXml, volDir):
""" Convert a SetOfVolumes to a xml file used by PyTom.
The volumes will be converted to .mrc format if not are '.em' or '.mrc'
Params:
volSet: input SetOfVolumes.
volXml: filename where to write the xml file.
volDir: where to create links or copies (converted to mrc).
"""
# Add to the path the root to pytom
backupPath = list(sys.path)
addPyTomPaths()
from pytom.basic.structures import Particle, ParticleList, Wedge, SingleTiltWedge
from pytom.score.score import Score, PeakPrior, xcfScore
from pytom.frm.FRMAlignment import FRMScore
w = SingleTiltWedge()
#s = PeakPrior()
pl = ParticleList()
ih = em.convert.ImageHandler()
for vol in volSet:
index, fn = vol.getLocation()
convert = True # by default, convert, which is the save way
if index == em.NO_INDEX: # means single volumes
volName = os.path.basename(fn)
if fn.endswith('.em') or fn.endswith('.mrc'):
convert = False # we can just create a link in this case
else:
volName = 'volume_%03d.mrc' % vol.getObjId()
volFn = os.path.join(volDir, volName)
if convert:
ih.convert(vol, volFn)
else:
pwutils.createLink(fn, volFn)
# Make the volumes names relative to the xml file
# where the programs will be executed
volRel = os.path.relpath(volFn, os.path.dirname(volXml))
p = Particle()
s = xcfScore()
s.setValue(1.0)
pytomInfo = getattr(vol, 'pytomInfo', None)
if pytomInfo is None:
p.setWedge(w)
else:
p.fromXML(pytomInfo.get()) # Get stored XML format from PyTom
p.setFilename(volRel)
p.setScore(s)
pl.append(p)
pl.toXMLFile(volXml)
#pl.setWedgeAllParticles(w)
sys.path = backupPath
def initialize(pl, settings):
from pytom.basic.structures import Particle
# from pytom.alignment.alignmentFunctions import average2
from pytom.basic.filter import lowpassFilter
print("Initializing the class centroids ...")
pl = pl.copy()
pl.sortByScore()
if settings["noise"]:
pl = pl[:int((1-settings["noise"])*len(pl))]
K = settings["ncluster"]
freq = settings["frequency"]
kn = len(pl)//K
references = {}
frequencies = {}
# get the first class centroid
pp = pl[:kn]
# avg, fsc = average2(pp, norm=True, verbose=False)
pp.setClassAllParticles('0')
res, tmp, tmp2 = calculate_averages(pp, settings["binning"], None, outdir=settings["output_directory"])
avg = res['0']
avg = lowpassFilter(avg, freq, freq/10.)[0]
avg.write(os.path.join(settings['output_directory'], 'initial_0.em') )
p = Particle(os.path.join(settings['output_directory'], 'initial_0.em'))
p.setClass('0')
references['0'] = p
frequencies['0'] = freq
for k in range(1, K):
distances = [4]*len(pl)
for c, ref in references.items():
args = list(zip(pl, [ref]*len(pl), [freq]*len(pl), [settings["fmask"]]*len(pl), [settings["binning"]]*len(pl)))
dist = mpi.parfor(distance, args)
for i in range(len(pl)):
if distances[i] > dist[i]:
distances[i] = dist[i]
distances = np.asarray(distances)
print('sum distances: ', distances.sum())
distances = distances/np.sum(distances)
idx = np.random.choice(len(pl), kn, replace=False, p=distances)
pp = ParticleList()
for i in idx:
pp.append(pl[int(i)])
# avg, fsc = average2(pp, norm=True, verbose=False)
pp.setClassAllParticles('0')
res, tmp, tmp2 = calculate_averages(pp, settings["binning"], None, outdir=settings["output_directory"])
avg = res['0']
avg = lowpassFilter(avg, freq, freq/10.)[0]
kname = os.path.join(settings['output_directory'], 'initial_{}.em'.format(k))
avg.write(kname)
p = Particle(kname)
p.setClass(str(k))
references[str(k)] = p
frequencies[str(k)] = freq
return references, frequencies
def toParticle(self):
"""
toParticle: Converts this object to a Particle object.
@rtype: L{pytom.basic.structures.Particle}
"""
from pytom.basic.structures import Particle
return Particle(filename=self.filename, rotation=self.orient, pickPosition=self.pos, score=self.score)
def fromXML(self, xmlObj):
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError('You must provide a valid XML object.')
if xmlObj.tag == "ParticleList":
main = xmlObj
else:
main = xmlObj.xpath('ParticleList')
if len(main) == 0:
from pytom.basic.exceptions import PyTomClassError
raise PyTomClassError("This XML is not a ParticleList.")
main = main[0]
for p in main.xpath('FoundParticle'):
particle = FoundParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('ClassifiedParticle'):
particle = ClassifiedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('SimulatedParticle'):
from pytom.localization.simulation import SimulatedParticle
particle = SimulatedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('IdentifiedParticle'):
from pytom.localization.simulation import IdentifiedParticle
particle = IdentifiedParticle()
particle.fromXML(p)
self.pl.append(particle)
for p in main.xpath('Particle'): # now also support reading class Particle, but will convert to FoundParticle
from pytom.basic.structures import Particle
pp = Particle()
pp.fromXML(p)
# convert to FoundParticle
particle = FoundParticle()
particle.fromParticle(pp)
self.pl.append(particle)
def __init__(self,
particle='',
reference='',
score='',
rotations='',
mask='',
numberRefinementRounds=1,
preprocessing='',
binning=1):
"""
@param particle: particle to be aligned
@type particle: L{pytom.basic.structures.Particle} or str
@param reference: reference for particle alignment
@param score: type of score used for alignment
@param rotations: rotations scanned in search
@param mask: mask on reference
@param numberRefinementRounds: iteration number
@param preprocessing: preprocessing parameters
@param binning: Binning Factor
@type binning: int
"""
from lxml.etree import _Element
if particle.__class__ == _Element:
self.fromXML(particle)
else:
if particle.__class__ == str:
from pytom.basic.structures import Particle
self.particle = Particle(particle)
else:
self.particle = particle
self.reference = reference
self.score = score
self.rotations = rotations
self.mask = mask
self.numberRefinementRounds = numberRefinementRounds
self.preprocessing = preprocessing
self.binning = binning
def calculate_difference_map_proxy(r1, band1, r2, band2, mask, focus_mask, binning, iteration, sigma, threshold, outdir='./'):
from pytom_volume import read, vol, pasteCenter
from pytom.basic.structures import Particle, Mask
import os
from pytom.basic.transformations import resize
v1 = r1.getVolume()
v2 = r2.getVolume()
if mask:
maskBin = read(mask, 0,0,0,0,0,0,0,0,0, binning, binning, binning)
if v1.sizeX() != maskBin.sizeX() or v1.sizeY() != maskBin.sizeY() or v1.sizeZ() != maskBin.sizeZ():
mask = vol(v1.sizeX(), v1.sizeY(), v1.sizeZ())
mask.setAll(0)
pasteCenter(maskBin, mask)
else:
mask = maskBin
else:
mask = None
if focus_mask:
focusBin = read(focus_mask, 0,0,0,0,0,0,0,0,0, binning, binning, binning)
if v1.sizeX() != focusBin.sizeX() or v1.sizeY() != focusBin.sizeY() or v1.sizeZ() != focusBin.sizeZ():
focus_mask = vol(v1.sizeX(), v1.sizeY(), v1.sizeZ())
focus_mask.setAll(0)
pasteCenter(focusBin, focus_mask)
else:
focus_mask = focusBin
else:
focus_mask = None
if not focus_mask is None and not mask is None:
if mask.sizeX() != focus_mask.sizeX():
raise Exception('Focussed mask and alignment mask do not have the same dimensions. This cannot be correct.')
(dmap1, dmap2) = calculate_difference_map(v1, band1, v2, band2, mask, focus_mask, True, sigma, threshold)
fname1 = os.path.join(outdir, 'iter'+str(iteration)+'_dmap_'+str(r1.getClass())+'_'+str(r2.getClass())+'.em')
dmap1.write(fname1)
fname2 = os.path.join(outdir, 'iter'+str(iteration)+'_dmap_'+str(r2.getClass())+'_'+str(r1.getClass())+'.em')
dmap2.write(fname2)
dp1 = Particle(fname1)
dp1.setClass(r1.getClass())
dp2 = Particle(fname2)
dp2.setClass(r2.getClass())
return (dp1, dp2)
def simulationDescriptionToParticleList(directory,prefix = ''):
"""
simulationDescriptionToParticleList:
"""
lenDir = len(directory)
if not directory[lenDir-1] == '/':
directory = directory + '/'
xmlFile = directory + 'desc.xml'
from lxml import etree
simulationXML = etree.parse(xmlFile)
#print etree.tostring(simulationXML,pretty_print=True)
particles = simulationXML.xpath('particle')
parameters = simulationXML.xpath('Simulation_Parameters')
wedge = int(parameters[0].get('wangleEnd'))/2
from pytom.basic.structures import Particle,ParticleList,WedgeInfo
wi = WedgeInfo(wedge,[0.0,0.0,0.0])
pl = ParticleList(directory)
for particle in particles:
filename = prefix + particle.get('filename')
rotation = particle.get('rotation')
rotation = rotation.replace('[','')
rotation = rotation.replace(']','')
rotation = rotation.split(',')
rotation = [float(rotation[0]),float(rotation[1]),float(rotation[2])]
shift = particle.get('shift')
shift = shift.replace('[','')
shift = shift.replace(']','')
shift = shift.split(',')
shift = [int(round(float(shift[0]))),int(round(float(shift[1]))),int(round(float(shift[2])))]
p = Particle(filename,rotation = rotation,shift = shift,wedge=wi)
pl.append(p)
#al.toXMLFile(directory+'AlignmentList.xml')
return pl
class CorrelationVector(PyTomClass):
"""
CorrelationVector: Stores a vector of correlation values of one particle with many others from particle list
@todo: Add unittest
"""
def __init__(self,particle=None,particleList=None,particleIndex=None):
self._correlations = []
self._particle = particle
self._particleList = particleList
self._particleIndex = particleIndex
def append(self,value):
self._correlations.append(value)
def getParticleIndex(self):
return self._particleIndex
def __len__(self):
return len(self._correlations)
def __getitem__(self,key):
"""
"""
if isinstance(key, int):
if key < len(self):
return self._correlations[key]
else:
raise IndexError('Index out of range.')
else:
assert False
def toXML(self):
"""
toXML : Compiles a XML file from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
vectorElement = etree.Element("CorrelationVector",ParticleIndex = self._particleIndex.__str__())
vectorElement.append(self._particle.toXML())
vectorElement.append(self._particleList.toXML())
for i in range(len(self._correlations)):
valueElement = etree.Element("Correlation",Index=i.__str__(),Value=self._correlations[i].__str__())
vectorElement.append(valueElement)
return vectorElement
def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element,tostring
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVector object.')
from pytom.basic.structures import Particle,ParticleList
self._particleIndex = int(xmlObj.get('ParticleIndex'))
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
values = xmlObj.xpath('Correlation')
self._correlations = [0 for _ in range(len(values))]
for v in values:
index = int(v.get('Index'))
self._correlations[index] = float(v.get('Value'))
class GrowingAverageInterimResult(PyTomClass):
"""
GrowingAverageInterimResult:
"""
def __init__(self,particle,reference,rotation,shift,score):
self.particle = particle
self.reference = reference
self.rotation = rotation
self.shift = shift
self.score = score
def getFilename(self):
return self.particle.getFilename()
def getWedgeInfo(self):
return self.particle.getWedgeInfo()
def getRotation(self):
return self.rotation
def getShift(self):
return self.shift
def toXML(self):
"""
toXML : Compiles a XML file from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
result_element = etree.Element('GrowingAverageInterimResult')
result_element.append(self.particle.toXML())
result_element.append(self.reference.toXML())
result_element.append(self.rotation.toXML())
result_element.append(self.shift.toXML())
result_element.append(self.score.toXML())
return result_element
def fromXML(self,xmlObj):
"""
fromXML:
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element :
raise Exception('Is not a lxml.etree._Element! You must provide a valid XMLobject.')
if xmlObj.tag == 'GrowingAverageInterimResult':
result_element = xmlObj
else:
Exception('XML object is not a GrowingAverageInterimResult! You must provide a valid GrowingAverageInterimResultXML object.')
from pytom.basic.structures import Particle,Reference
particleXML = result_element.xpath('/GrowingAverageInterimResult/Particle')[0]
self.particle = Particle('')
self.particle.fromXML(particleXML)
referenceXML = result_element.xpath('/GrowingAverageInterimResult/Result')[0]
self.reference = Reference('')
self.reference.fromXML(referenceXML)
def classify(pl, settings):
"""
auto-focused classification
@param pl: particle list
@type pl: L{pytom.basic.structures.ParticleList}
@param settings: settings for autofocus classification
@type settings: C{dict}
"""
from pytom.basic.structures import Particle, Shift, Rotation
from pytom.basic.filter import lowpassFilter
# make the particle list picklable
pl.pickle()
# define the starting status
offset = settings["offset"]
binning = settings["binning"]
mask = settings["mask"]
sfrequency = settings["frequency"] # starting frequency
outdir = settings["output_directory"]
references = {}
frequencies = {}
ncluster = 0
if settings["external"]: # use external references
for class_label, fname in enumerate(settings["external"]):
p = Particle(fname)
p.setClass(str(class_label))
references[str(class_label)] = p
frequencies[str(class_label)] = sfrequency
ncluster += 1
else:
if not settings["resume"]:
if not settings["ncluster"]:
print("Must specify the number of clusters!")
return
# k-means++ way to initialize
ncluster = settings["ncluster"]
references, frequencies = initialize(pl, settings)
else:
avgs, tmp, tmp2 = calculate_averages(pl,
binning,
mask,
outdir=outdir)
for class_label, r in avgs.items():
fname = os.path.join(
outdir, 'initial_class' + str(class_label) + '.em')
rr = lowpassFilter(r, sfrequency, sfrequency / 10.)[0]
rr.write(fname)
p = Particle(fname)
p.setClass(str(class_label))
references[str(class_label)] = p
frequencies[str(class_label)] = sfrequency
ncluster += 1
# start the classification
for i in range(settings["niteration"]):
if ncluster < 2:
print('Not enough number of clusters. Exit!')
break
print("Starting iteration %d ..." % i)
old_pl = pl.copy()
# compute the difference maps
print("Calculate difference maps ...")
args = []
for pair in combinations(list(references.keys()), 2):
args.append((references[pair[0]], frequencies[pair[0]],
references[pair[1]], frequencies[pair[1]], mask,
settings["fmask"], binning, i, settings["sigma"],
settings["threshold"], outdir))
dmaps = {}
res = mpi.parfor(calculate_difference_map_proxy, args)
for r in res:
dmaps[(r[0].getClass(), r[1].getClass())] = r
# start the alignments
print("Start alignments ...")
scores = calculate_scores(pl, references, frequencies, offset, binning,
mask, settings["noalign"])
# determine the class labels & track the class changes
pl = determine_class_labels(pl, references, frequencies, scores, dmaps,
binning, settings["noise"])
# kick out the small classes
pls = pl.copy().splitByClass()
nlabels = {}
for pp in pls:
nlabels[pp[0].getClass()] = len(pp)
print("Number of class " + str(pp[0].getClass()) + ": " +
str(len(pp)))
max_labels = np.max(list(nlabels.values()))
to_delete = []
if settings["dispersion"]:
min_labels = float(max_labels) / settings["dispersion"]
for key, value in nlabels.items():
if value <= min_labels:
to_delete.append(key)
for pp in pls:
if pp[0].getClass() in to_delete:
pp.setClassAllParticles('-1')
print("Set class " + str(pp[0].getClass()) + " to noise")
# split the top n classes
pl = split_topn_classes(pls, len(to_delete))
# update the references
print("Calculate averages ...")
avgs, freqs, wedgeSum = calculate_averages(pl,
binning,
mask,
outdir=outdir)
ncluster = 0
references = {}
for class_label, r in avgs.items():
if not settings["fixed_frequency"]:
freq = freqs[str(class_label)]
else:
freq = sfrequency
frequencies[str(class_label)] = int(freq)
print('Resolution of class %s: %d' % (str(class_label), freq))
fname = os.path.join(
outdir, 'iter' + str(i) + '_class' + str(class_label) + '.em')
rr = lowpassFilter(r, freq, freq / 10.)[0]
rr.write(fname)
p = Particle(fname)
p.setClass(str(class_label))
references[str(class_label)] = p
ncluster += 1
w = wedgeSum[str(class_label)]
fname = os.path.join(
outdir,
'iter' + str(i) + '_class' + str(class_label) + '_wedge.em')
w.write(fname)
# write the result to the disk
pl.toXMLFile(
os.path.join(outdir, 'classified_pl_iter' + str(i) + '.xml'))
# check the stopping criterion
if compare_pl(old_pl, pl):
break
dir_name, name_prefix, wedge_angle, output, bHelp = parse_script_options(
sys.argv[1:], helper)
except:
sys.exit()
if bHelp is True:
print(helper)
sys.exit()
if name_prefix is None:
name_prefix = ''
wedge_angle = float(wedge_angle)
w = SingleTiltWedge(wedge_angle)
pl = ParticleList()
all_files = os.listdir(dir_name)
for fname in all_files:
p = None
name_suffix = fname.split('.')[-1]
if len(name_prefix):
if name_prefix in fname and name_suffix in ['em', 'mrc']:
p = Particle(dir_name + '/' + fname)
else:
if name_suffix in ['em', 'mrc']:
p = Particle(dir_name + '/' + fname)
if p is not None:
pl.append(p)
pl.setWedgeAllParticles(w)
pl.toXMLFile(output)
class CorrelationVectorJob(PyTomClass):
"""
CorrelationVectorJob: All settings needed for a correlation vector job. Explore class for more info.
"""
def __init__(self,particle=None,particleList=None,mask=None,particleIndex = None,applyWedge = True,binningFactor=0,lowestFrequency=-1,highestFrequency=-1):
"""
__init__:
@param particle: Particle
@type particle: pytom.alignment.structures.Particle
@param particleList: ParticleList of all particles will be correlated with self._particle
@type particleList: pytom.alignment.structures.ParticleList
@param mask: Mask used for correlation
@type mask: str
@param applyWedge: Apply wedge during correlation. True by default, disable for rotation classification.
@type applyWedge: Bool
@param binningFactor: Binning factor accroding to libtomc definition. 0 by default.
@type binningFactor: unsigned int
@param lowestFrequency: Lowest frequency for bandpass in nyquist
@type lowestFrequency: float
@param highestFrequency: Highest frequency for bandpass in nyquist
@type highestFrequency: float
"""
from pytom.basic.structures import Particle,ParticleList,Mask
if particle and particle.__class__ != Particle:
raise ParameterError('You must provide a Particle object.')
if particleList and particleList.__class__ != ParticleList:
raise ParameterError('You must provide a ParticleList object.')
self._particle = particle
self._particleList = particleList
if not mask:
mask = Mask()
elif mask.__class__ == str:
mask = Mask(mask)
elif mask.__class__ != Mask:
from pytom.basic.exceptions import ParameterError
raise ParameterError('Mask must be a string or Mask object!')
self._mask = mask
self._particleIndex = particleIndex
self._applyWedge = applyWedge
self._binningFactor = binningFactor
self._lowestFrequency = lowestFrequency
self._highestFrequency = highestFrequency
def getMask(self):
return self._mask
def getParticle(self):
return self._particle
def getParticleList(self):
return self._particleList
def getParticleIndex(self):
return self._particleIndex
def getApplyWedge(self):
return self._applyWedge
def getBinning(self):
return self._binningFactor
def getLowestFrequency(self):
return self._lowestFrequency
def getHighestFrequency(self):
return self._highestFrequency
def toXML(self):
"""
toXML : Compiles a XML object from result object
rtype : L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml import etree
jobElement = etree.Element('CorrelationVectorJob',ParticleIndex = self._particleIndex.__str__(),ApplyWedge=self._applyWedge.__str__(),Binning = self._binningFactor.__str__(),LowestFrequency = str(self._lowestFrequency),HighestFrequency = str(self._highestFrequency))
jobElement.append(self._particle.toXML())
jobElement.append(self._particleList.toXML())
jobElement.append(self._mask.toXML())
return jobElement
def fromXML(self,xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@type xmlObj: L{lxml.etree._Element}
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise ParameterError('You must provide a valid XML-CorrelationVectorJob object.')
from pytom.basic.structures import Particle,ParticleList,Mask
particleObject = xmlObj.xpath('Particle')
self._particle = Particle('.')
self._particle.fromXML(particleObject[0])
particleListObject = xmlObj.xpath('ParticleList')
self._particleList = ParticleList('/')
self._particleList.fromXML(particleListObject[0])
maskObject = xmlObj.xpath('Mask')[0]
self._mask = Mask()
self._mask.fromXML(maskObject)
self._particleIndex = xmlObj.get('ParticleIndex')
self._applyWedge = xmlObj.get('ApplyWedge') == 'True'
self._binningFactor = float(xmlObj.get('Binning'))
self._lowestFrequency = float(xmlObj.get('LowestFrequency'))
self._highestFrequency = float(xmlObj.get('HighestFrequency'))
class MaximisationJob(PyTomClass):
"""
MaximisationJob : Stores all infos needed for a maximisation job
"""
def __init__(self,
particle='',
reference='',
score='',
rotations='',
mask='',
numberRefinementRounds=1,
preprocessing='',
binning=1):
"""
@param particle: particle to be aligned
@type particle: L{pytom.basic.structures.Particle} or str
@param reference: reference for particle alignment
@param score: type of score used for alignment
@param rotations: rotations scanned in search
@param mask: mask on reference
@param numberRefinementRounds: iteration number
@param preprocessing: preprocessing parameters
@param binning: Binning Factor
@type binning: int
"""
from lxml.etree import _Element
if particle.__class__ == _Element:
self.fromXML(particle)
else:
if particle.__class__ == str:
from pytom.basic.structures import Particle
self.particle = Particle(particle)
else:
self.particle = particle
self.reference = reference
self.score = score
self.rotations = rotations
self.mask = mask
self.numberRefinementRounds = numberRefinementRounds
self.preprocessing = preprocessing
self.binning = binning
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'You must provide a valid XML-MaximisationJob object.')
if xmlObj.tag == "JobDescription":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('JobDescription')
if len(jobDescription) == 0:
from pytom.basic.structures import PyTomClassError
raise PyTomClassError("This XML is not an JobDescription.")
jobDescription = jobDescription[0]
from pytom.angles.angle import AngleObject
from pytom.score.score import fromXML as fromXMLScore
from pytom.alignment.preprocessing import Preprocessing
from pytom.basic.structures import Mask, Particle, Reference, ReferenceList
self.binning = int(jobDescription.get('Binning'))
particle_element = jobDescription.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self.particle = p
r = jobDescription.xpath('Reference')
if len(r) > 0:
ref = Reference('')
ref.fromXML(r[0])
self.reference = ref
else:
r = jobDescription.xpath('ReferenceList')
ref = ReferenceList()
ref.fromXML(r[0])
self.reference = ref
mask = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(mask)
self.numberRefinementRounds = jobDescription.get(
'NumberRefinementRounds')
self.numberRefinementRounds = int(self.numberRefinementRounds)
score = jobDescription.xpath('Score')
self.score = fromXMLScore(score[0])
angles = jobDescription.xpath('Angles')
ang = AngleObject()
self.rotations = ang.fromXML(angles[0])
preObj = xmlObj.xpath('Preprocessing')
if len(preObj) == 0:
self.preprocessing = Preprocessing()
else:
p = Preprocessing()
p.fromXML(preObj[0])
self.preprocessing = p
def toXML(self):
"""
toXML : Compiles a XML file from job object
@author: Thomas Hrabe
"""
from lxml import etree
jobElement = etree.Element("JobDescription")
jobElement.set("NumberRefinementRounds",
str(self.numberRefinementRounds))
jobElement.set("Binning", str(self.binning))
jobElement.append(self.particle.toXML())
jobElement.append(self.reference.toXML())
jobElement.append(self.mask.toXML())
jobElement.append(self.rotations.toXML())
if self.score.__class__ == str:
jobElement.append(self.score)
else:
jobElement.append(self.score.toXML())
preObj = self.preprocessing.toXML()
jobElement.append(preObj)
return jobElement
def check(self):
"""
check: Performs check on self whether all settings were sane. Paths and Files exists
"""
from pytom.tools.files import checkFileExists, checkDirExists
returnValue = checkFileExists(self.particle.getFilename())
if not returnValue:
raise IOError(str(self.particle.getFilename()) + ' not found!')
returnValue = returnValue and checkFileExists(
self.reference.getFilename())
if not returnValue:
raise IOError(str(self.reference) + ' not found!')
return returnValue
class MaximisationResult(PyTomClass):
"""
MaximisationResult : Stores results of one maximisation process
"""
def __init__(self,
particle='',
reference=-1.0,
score=-1.0,
shift=-1.0,
rotation=-1.0,
angleObject=-1):
from pytom.basic.structures import Particle, Reference, Shift, Rotation
from numpy import long
if particle.__class__ == str:
self._particle = Particle(particle)
elif particle.__class__ == Particle:
self._particle = particle
else:
self._particle = Particle()
if reference.__class__ == str:
self._reference = Reference(reference)
elif reference.__class__ == Reference:
self._reference = reference
else:
self._reference = Reference()
if shift.__class__ == list:
self._shift = Shift(shift)
elif shift.__class__ == float:
self._shift = Shift()
else:
self._shift = shift
if rotation.__class__ == list:
self._rotation = Rotation(rotation)
elif rotation.__class__ == Rotation:
self._rotation = rotation
else:
self._rotation = Rotation()
if score.__class__ == float:
from pytom.score.score import xcfScore
self._score = xcfScore()
else:
self._score = score
if angleObject.__class__ == float or isinstance(
angleObject, (int, long)):
from pytom.angles.angleList import AngleList
self._angleObject = AngleList()
else:
self._angleObject = angleObject
def toParticle(self):
"""
toParticle: Converts this object to a Particle object.
@return:
@rtype: L{pytom.basic.structures.Particle}
"""
from pytom.basic.structures import Particle
particle = self._particle
particle.setRotation(self._rotation)
particle.setShift(self._shift)
particle.setScore(self._score)
return particle
def getParticle(self):
return self._particle
def getShift(self):
from pytom.basic.structures import Shift
if self._shift.__class__ == list:
return Shift(self._shift)
else:
return self._shift
def setShift(self, shift):
from pytom.basic.structures import Shift
assert shift.__class__ == Shift
self._shift = shift
def getAngleObject(self):
return self._angleObject
def getRotation(self):
from pytom.basic.structures import Rotation
if self._rotation.__class__ == list:
return Rotation(self._rotation[0], self._rotation[1],
self._rotation[2])
else:
return self._rotation.copy()
def getScore(self):
"""
getScore: Returns score object
"""
return self._score
def setRotation(self, rotation):
"""
setRotation:
@param rotation:
"""
from pytom.basic.structures import Rotation
if rotation.__class__ == list:
rotation = Rotation(rotation)
self._rotation = rotation
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to result attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'Is not a lxml.etree._Element! You must provide a valid XML object.'
)
from pytom.score.score import fromXML as fromXMLScore
from pytom.angles.angle import AngleObject
if xmlObj.tag == "Result":
result = xmlObj
else:
result = xmlObj.xpath('Result')
if len(result) == 0:
raise PyTomClassError(
"This XML is not an MaximisationResult. No Result provided."
)
result = result[0]
from pytom.basic.structures import Particle, Reference, Rotation, Shift
particle_element = result.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self._particle = p
r = result.xpath('Reference')
ref = Reference('')
ref.fromXML(r[0])
self._reference = ref
scoreXML = result.xpath('Score')[0]
self._score = fromXMLScore(scoreXML)
shiftXML = result.xpath('Shift')[0]
self._shift = Shift()
self._shift.fromXML(shiftXML)
rotationXML = result.xpath('Rotation')[0]
self._rotation = Rotation()
self._rotation.fromXML(rotationXML)
angleElement = result.xpath('Angles')
ang = AngleObject()
self._angleObject = ang.fromXML(angleElement[0])
def toXML(self):
"""
toXML : Compiles a XML from result object
@author: Thomas Hrabe
"""
from lxml import etree
resultElement = etree.Element("Result")
resultElement.append(self._particle.toXML())
if self._reference.hasGeneratedByInfo():
from pytom.basic.structures import Reference
newRef = Reference(self._reference.getReferenceFilename())
resultElement.append(newRef.toXML())
else:
resultElement.append(self._reference.toXML())
resultElement.append(self._shift.toXML())
resultElement.append(self._rotation.toXML())
resultElement.append(self._score.toXML())
resultElement.append(self._angleObject.toXML())
return resultElement
def copy(self):
return MaximisationResult(self._particle, self._reference, self._score,
self._shift, self._rotation,
self._angleObject)
if not dest_dir:
dest_dir = '.'
from pytom_volume import read, subvolume
v = read(vol_filename)
from pytom.basic.structures import ParticleList, Particle, WedgeInfo
pl = ParticleList("./")
pl.fromXMLFile(pl_filename)
def regulaize(xx, dim):
if xx*binning-radius < 0:
if 2*radius > dim:
raise Exception("Volume size to be cut is too big!")
return 0
if xx*binning+radius > dim:
if dim-2*radius < 0:
raise Exception("Volume size to be cut is too big!")
return dim-2*radius
return xx*binning-radius
res = ParticleList(dest_dir)
for p in pl:
x,y,z = p.getPickPosition().toVector()
x = regulaize(int(x), v.sizeX()); y = regulaize(int(y), v.sizeY()); z = regulaize(int(z), v.sizeZ())
new_vol = subvolume(v, x, y, z, 2*radius, 2*radius, 2*radius)
name = dest_dir+'/'+p.getFilename()
new_vol.write(name) # write the subtomograms to the disk
res.append(Particle(name, p.getRotation(), None, WedgeInfo(w), 1, p.getPickPosition(), p.getScore())) # append it to the particle list for alignment
res.toXMLFile(dest_dir+'/'+res_name)
def create_RandomParticleList(reffile,
pl_filename='pl.xml',
pdir='./testparticles',
nparticles=10):
"""
@param reffile: reference file
@type reffile: C{str}
@param nparticles: number of particles (default: 10)
@type nparticles: C{int}
@param pl_filename: particle list filename
@type pl_filename: C{str}
@param pdir: particle directory
@type pdir: C{str}
@return: particleList
@rtype: L{pytom.basic.ParticleList}
"""
from pytom.basic.structures import Particle, ParticleList, Rotation, Shift
from pytom_volume import vol, rotate, shift, read
from pytom.basic.transformations import general_transform_crop
from pytom.simulation.whiteNoise import add as addNoise
import random
from os import mkdir
from pytom.score.score import FLCFScore as score
try:
mkdir(pdir)
except FileExistsError:
print('directory ' + pdir + ' existed already - using this one')
random.seed(0)
pl = ParticleList(directory='./')
ref = read(reffile)
for ii in range(0, nparticles):
rot = Rotation(random.uniform(0, 360), random.uniform(0, 360),
random.uniform(0, 180))
shift = Shift(x=random.uniform(-5, 5),
y=random.uniform(-5, 5),
z=random.uniform(-5, 5))
rotvol = general_transform_crop(v=ref,
rot=rot,
shift=shift,
scale=None,
order=[0, 1, 2])
# add some noise
noisy = addNoise(volume=rotvol, SNR=1)
fname = pdir + '/particle_' + str(ii) + '.em'
noisy.write(fname)
p = Particle(filename=fname,
rotation=rot,
shift=shift,
wedge=None,
className=0,
pickPosition=None,
score=score,
sourceInfo=None)
p.setScoreValue(0.0)
pl.append(particle=p)
pl.setFileName(filename=pl_filename)
pl.toXMLFile(filename=pl_filename)
return pl
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@author: Thomas Hrabe
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
from pytom.basic.exceptions import ParameterError
raise ParameterError(
'You must provide a valid XML-MaximisationJob object.')
if xmlObj.tag == "JobDescription":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('JobDescription')
if len(jobDescription) == 0:
from pytom.basic.structures import PyTomClassError
raise PyTomClassError("This XML is not an JobDescription.")
jobDescription = jobDescription[0]
from pytom.angles.angle import AngleObject
from pytom.score.score import fromXML as fromXMLScore
from pytom.alignment.preprocessing import Preprocessing
from pytom.basic.structures import Mask, Particle, Reference, ReferenceList
self.binning = int(jobDescription.get('Binning'))
particle_element = jobDescription.xpath('Particle')[0]
p = Particle('')
p.fromXML(particle_element)
self.particle = p
r = jobDescription.xpath('Reference')
if len(r) > 0:
ref = Reference('')
ref.fromXML(r[0])
self.reference = ref
else:
r = jobDescription.xpath('ReferenceList')
ref = ReferenceList()
ref.fromXML(r[0])
self.reference = ref
mask = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(mask)
self.numberRefinementRounds = jobDescription.get(
'NumberRefinementRounds')
self.numberRefinementRounds = int(self.numberRefinementRounds)
score = jobDescription.xpath('Score')
self.score = fromXMLScore(score[0])
angles = jobDescription.xpath('Angles')
ang = AngleObject()
self.rotations = ang.fromXML(angles[0])
preObj = xmlObj.xpath('Preprocessing')
if len(preObj) == 0:
self.preprocessing = Preprocessing()
else:
p = Preprocessing()
p.fromXML(preObj[0])
self.preprocessing = p
def paverage(particleList, norm, binning, verbose, outdir='./'):
from pytom_volume import read, vol
from pytom_volume import transformSpline as transform
from pytom.basic.structures import Particle
from pytom.basic.normalise import mean0std1
from pytom.tools.ProgressBar import FixedProgBar
from pytom.basic.transformations import resize
if len(particleList) == 0:
raise RuntimeError('The particlelist provided is empty. Aborting!')
if verbose:
progressBar = FixedProgBar(0, len(particleList), 'Particles averaged ')
progressBar.update(0)
numberAlignedParticles = 0
result = None
wedgeSum = None
newParticle = None
for particleObject in particleList:
particle = read(particleObject.getFilename(), 0, 0, 0, 0, 0, 0, 0, 0,
0, 1, 1, 1)
if binning != 1:
particle, particlef = resize(volume=particle,
factor=1. / binning,
interpolation='Fourier')
if norm:
mean0std1(particle)
wedgeInfo = particleObject.getWedge()
if result is None: # initialization
sizeX = particle.sizeX()
sizeY = particle.sizeY()
sizeZ = particle.sizeZ()
newParticle = vol(sizeX, sizeY, sizeZ)
centerX = sizeX // 2
centerY = sizeY // 2
centerZ = sizeZ // 2
result = vol(sizeX, sizeY, sizeZ)
result.setAll(0.0)
wedgeSum = wedgeInfo.returnWedgeVolume(sizeX, sizeY, sizeZ)
wedgeSum.setAll(0)
# create spectral wedge weighting
rotation = particleObject.getRotation()
wedge = wedgeInfo.returnWedgeVolume(sizeX, sizeY, sizeZ, False,
rotation.invert())
wedgeSum += wedge
# shift and rotate particle
shiftV = particleObject.getShift()
newParticle.setAll(0)
transform(particle, newParticle, -rotation[1], -rotation[0],
-rotation[2], centerX, centerY, centerZ,
-shiftV[0] // binning, -shiftV[1] // binning,
-shiftV[2] // binning, 0, 0, 0)
result += newParticle
if verbose:
numberAlignedParticles = numberAlignedParticles + 1
progressBar.update(numberAlignedParticles)
# write to the disk
fname_result = os.path.join(outdir, 'avg_{}.em'.format(mpi.rank))
fname_wedge = os.path.join(outdir, 'wedge_{}.em'.format(mpi.rank))
result.write(fname_result)
result = Particle(fname_result)
wedgeSum.write(fname_wedge)
wedgeSum = Particle(fname_wedge)
return (result, wedgeSum)
def assessClassification(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: [trueHits,falseHits,trueHitsPercent,falseHitsPercent,numberClusters,[clusterSizes]]
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleListFile = groundTruthParticleList
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleListFile)
gtClassNamesAssigned = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNamesAssigned[particle.getClassName()] = False
if verbose:
print('GT Classes ', gtClassNamesAssigned)
gtClassesPerClass = {}
newClasses = classifiedParticleList.splitByClass()
newClassNamesAssigned = {}
numberClasses = len(newClasses)
classSizes = []
for i in range(len(newClasses)):
classSizes.append(len(newClasses[i]))
for newClass in newClasses:
newClassParticleList = ParticleList(newClass.getDirectory())
newClassNamesAssigned[newClass[0].getClassName()] = False
for particle in newClass:
pp = groundTruthParticleList[particle.getFilename()]
newClassParticleList.append(pp)
gtClassSizeDictionary = {}
for gtClass in newClassParticleList.splitByClass():
particle = gtClass[0]
gtParticle = groundTruthParticleList[particle.getFilename()]
gtClassSizeDictionary[gtParticle.getClassName()] = len(gtClass)
gtClassesPerClass[newClass[0].getClassName()] = [
newClassParticleList, gtClassSizeDictionary
]
if verbose:
print('Class distribution dictionary')
for k in list(gtClassesPerClass.keys()):
print(k, gtClassesPerClass[k])
gtToClassDictionary = {}
for gtName in list(gtClassNamesAssigned.keys()):
newClassIndex = 0
classSizeList = []
largestClass = -1
maxClassName = 'unknown'
assigned = False
for newClassName in list(gtClassesPerClass.keys()):
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
if verbose:
print('GT Name', gtName, ' New Class Name', newClassName)
print('GT Name Size', gtClassSizeDictionary)
try:
if verbose:
print(gtClassSizeDictionary[gtName])
if largestClass < gtClassSizeDictionary[
gtName] and not newClassNamesAssigned[newClassName]:
largestClass = gtClassSizeDictionary[gtName]
maxClassName = newClassName
if verbose:
print('SWAP')
except KeyError:
pass
gtToClassDictionary[gtName] = maxClassName
newClassNamesAssigned[maxClassName] = True
gtClassNamesAssigned[gtName] = True
for newClassName in list(gtClassesPerClass.keys()):
try:
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
del gtClassSizeDictionary[gtName]
l[1] = gtClassSizeDictionary
gtClassesPerClass[newClassName] = l
except KeyError:
pass
if verbose:
print('GT to New Dictionary')
print(gtToClassDictionary)
trueHits = 0
falseHits = 0
classifiedParticleListXML = classifiedParticleList.toXML()
for gtParticle in groundTruthParticleList:
particleXML = classifiedParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' +
str(gtParticle.getFilename()) + '"]')
particle = Particle('a')
particle.fromXML(particleXML[0])
if particle.getClassName() == gtToClassDictionary[
gtParticle.getClassName()]:
trueHits += 1
else:
falseHits += 1
return [
trueHits, falseHits,
float(trueHits) / len(classifiedParticleList),
float(falseHits) / len(classifiedParticleList), numberClasses,
classSizes
]
def growingAverageNew(particleList=None,angleObject=None,maskFile=None,scoreObject=None,startClassNumber=0,destinationDirectory='.',preprocessing = None,binning=1,verbose=False):
"""
"""
from pytom.alignment.alignmentFunctions import bestAlignment
from pytom.basic.structures import Reference,Particle,Rotation,ParticleList
from pytom.alignment.preprocessing import Preprocessing
if not preprocessing:
preprocessing = Preprocessing()
numberOfClasses = len(particleList.splitByClass())
if verbose:
print('Processing ' + str(numberOfClasses) + ' classes.')
print('Generating start average')
startAverageList = particleList.particlesFromClass(float(startClassNumber))
startAverageList.average(destinationDirectory + '/GA_it0.em',progressBar=verbose)
currentReference = Reference(destinationDirectory + '/GA_it0.em')
growingAverageParticleList = ParticleList(particleList.getDirectory())
for p in startAverageList:
p.setRotation(Rotation(0,0,0))
growingAverageParticleList.append(p)
for i in range(2,numberOfClasses):
currentParticleList = particleList.particlesFromClass(float(i))
if verbose:
print('Generating ' + str(i) + '. class average')
currentParticleList.average(destinationDirectory + '/CA_it'+str(i)+'.em',progressBar=verbose)
currentParticle = Particle(destinationDirectory + '/CA_it'+str(i)+'.em',wedgeInfo=currentParticleList[0].getWedgeInfo())
if verbose:
print('Running alignment iteration ' + str(i))
print(currentParticle)
print(currentReference)
currentPeak = bestAlignment(currentParticle.getVolume(),currentReference.getVolume(),currentReference.getWeighting(),currentParticle.getWedgeInfo(),angleObject,scoreObject,maskFile,preprocessing=preprocessing,binning=binning)
if verbose:
print('Parameters determined:')
print(currentPeak)
for p in currentParticleList:
p.setRotation(currentPeak.getRotation())
p.setShift(currentPeak.getShift())
growingAverageParticleList.append(p)
if verbose:
print('Generating growing average ' + str(i))
growingAverageParticleList.average(destinationDirectory + '/GA_it'+ str(i) +'.em',progressBar=verbose)
currentReference = Reference(destinationDirectory + '/GA_it'+ str(i) +'.em')
angleObject.reset()
def createClassificationResultDictionaries(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: A dictionary that maps new classnames to groundTruthClasses
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleList)
newClassesToGroundTruthMap = {
} #maps new class name to a ground truth class name
gtClassNames = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNames[particle.getClassName()] = False
if verbose:
print('gtClassNames : ', gtClassNames)
groundTruthParticleListXML = groundTruthParticleList.toXML()
for newClass in classifiedParticleList.splitByClass():
if verbose:
print('')
print('newClass : ', newClass)
particlesFromGroundTruth = ParticleList('/')
for particle in newClass:
#collect all particles that were assigned to newClass
particleName = particle.getFilename()
#gtParticle = groundTruthParticleList.getParticleByFilename(particleName)
particleXML = groundTruthParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' + str(particleName) +
'"]')
gtParticle = Particle('a')
gtParticle.fromXML(particleXML[0])
particlesFromGroundTruth.append(gtParticle)
if verbose:
print('len(particlesFromGroundTruth) : ',
len(particlesFromGroundTruth))
#sort classes according to size to descending order
sortedClasses = sorted(particlesFromGroundTruth.splitByClass(),
key=lambda x: len(x),
reverse=True)
classWasAssigned = False
classIndex = 0
if verbose:
print('len(sortedClasses) : ', len(sortedClasses))
while not classWasAssigned and classIndex < len(sortedClasses):
sortedClass = sortedClasses[classIndex]
className = sortedClass[0].getClassName()
if verbose:
print('className : ' + className)
print('len(sortedClass) : ', len(sortedClass))
classWasAssigned = not gtClassNames[className]
if verbose:
print('classWasAssigned : ', classWasAssigned)
if not classWasAssigned:
classIndex = classIndex + 1
else:
gtClassNames[className] = True
newClassesToGroundTruthMap[
newClass[0].getClassName()] = className
if verbose:
print('gtClassNames : ', gtClassNames)
print('newClassesToGroundTruthMap : ',
newClassesToGroundTruthMap)
return newClassesToGroundTruthMap
def create_RandomParticleList(reffile,
pl_filename='pl.xml',
pdir='./testparticles',
nparticles=10):
"""
@param reffile: reference file
@type reffile: C{str}
@param nparticles: number of particles (default: 10)
@type nparticles: C{int}
@param pl_filename: particle list filename
@type pl_filename: C{str}
@param pdir: particle directory
@type pdir: C{str}
@return: particleList
@rtype: L{pytom.basic.ParticleList}
"""
from pytom.basic.structures import Particle, ParticleList, Rotation, Shift, Wedge
from pytom_volume import vol, rotate, shift, read
from pytom.basic.transformations import general_transform_crop
from pytom.basic.functions import initSphere
from pytom.simulation.whiteNoise import add as addNoise
import random
from os import mkdir
from pytom.score.score import FLCFScore as score
try:
mkdir(pdir)
except FileExistsError:
print('directory ' + pdir + ' existed already - using this one')
random.seed(0)
a = 0
wedge = Wedge(wedgeAngles=[30.0, 30.0], cutoffRadius=50.0)
pl = ParticleList(directory='./')
ref1 = read(reffile)
ref2 = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=45)
#ref2.write('testData/mask_45.em')
parts = {0: ref1, 1: ref2}
for ii in range(0, nparticles):
if not ii % 2:
ref = read(reffile)
else:
ref = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=30,
smooth=30 / 10)
rot = Rotation(random.uniform(0, 360), random.uniform(0, 360),
random.uniform(0, 180))
shift = Shift(x=a * random.uniform(-5, 5),
y=a * random.uniform(-5, 5),
z=a * random.uniform(-5, 5))
rotvol = general_transform_crop(v=ref,
rot=rot,
shift=shift,
scale=None,
order=[0, 1, 2])
# add some noise
noisy = addNoise(volume=rotvol, SNR=1)
fname = pdir + '/particle_' + str(ii) + '.em'
#noisy.write( fname)
p = Particle(filename=fname,
rotation=rot,
shift=shift,
wedge=wedge,
className=0,
pickPosition=None,
score=score,
sourceInfo=None)
p.setScoreValue(0.0)
wg = p.getWedge().getWedgeObject()
wg.apply(noisy, Rotation(0, 0, 0)).write(fname)
pl.append(particle=p)
pl.setFileName(filename=pl_filename)
pl.toXMLFile(filename=pl_filename)
return pl
class ClusterSwap(PyTomClass):
"""
ClusterSwap: Saves conditions under which a class swap of a particle happened.
"""
def __init__(self,
particle=None,
oldClusterName=None,
oldScore=None,
newClusterName=None,
newScore=None):
self._particle = particle
self._oldClusterName = oldClusterName
self._oldScore = oldScore
self._newClusterName = newClusterName
self._newScore = newScore
def toXML(self):
from lxml import etree
swapElement = etree.Element('ClassSwap',
OldClusterName=str(self._oldClusterName),
NewClusterName=str(self._newClusterName))
swapElement.append(self._particle.toXML())
os = self._oldScore.toXML()
os.set('OriginalType', str(os.tag))
os.tag = 'OldScore'
swapElement.append(os)
ns = self._newScore.toXML()
ns.set('OriginalType', str(ns.tag))
ns.tag = 'NewScore'
swapElement.append(ns)
return swapElement
def fromXML(self, xmlObj):
from lxml.etree import _Element
from pytom.basic.structures import Particle
from pytom.score.score import fromXML as scoreFromXML
if xmlObj.__class__ != _Element:
raise Exception(
'Is not a lxml.etree._Element! You must provide a valid XMLobject.'
)
self._oldClusterName = str(xmlObj.get('OldClusterName'))
self._newClusterName = str(xmlObj.get('NewClusterName'))
oldScore = xmlObj.xpath('OldScore')[0]
oldScore.tag = oldScore.get('OriginalType')
self._oldScore = scoreFromXML(oldScore)
newScore = xmlObj.xpath('NewScore')[0]
newScore.tag = newScore.get('OriginalType')
self._newScore = scoreFromXML(newScore)
particle = xmlObj.xpath('Particle')[0]
self._particle = Particle('')
self._particle.fromXML(particle)
