def determineClassSwaps(particleList1,particleList2):
"""
determineClassSwaps
@param particleList1: The previous particleList
@param particleList2: The current particleList
@return: List of particles that swapped class and a corresponding list of [previous class, current class].
@rtype: [L{pytom.basic.structures.ParticleList},[previousClass,currentClass]]
"""
from pytom.basic.structures import ParticleList
returnList = ParticleList(particleList1.getDirectory())
classSwaps = []
for particle in particleList1:
particleName = particle.getFilename()
particleClass = particle.getClassName()
otherParticle = particleList2.getParticleByFilename(particleName)
otherParticleClass = otherParticle.getClassName()
if not particleClass == otherParticleClass:
returnList.append(otherParticle)
classSwaps.append([particleClass,otherParticleClass])
return [returnList,classSwaps]
def saveForFSC(newReferenceName, particleList, verbose=False):
"""
saveForFSC: Split particles into even / odd and calculate the Fourier Shell Correlation out of these two sets.
@param newReferenceName: Name of current alignment result
@type newReferenceName: str
@param particleList:
@type particleList: L{pytom.basic.structures.ParticleList}
"""
if len(particleList) < 2:
raise RuntimeError(
'ParticleList must have at least 2 elements to run saveForFSC!')
from pytom.basic.structures import ParticleList
from pytom.alignment.alignmentFunctions import average
even = ParticleList('/')
odd = ParticleList('/')
for particleCounter in range(len(particleList)):
if particleCounter % 2 == 0:
even.append(particleList[particleCounter])
else:
odd.append(particleList[particleCounter])
if verbose:
print('averaging even:')
average(even, newReferenceName + 'even.em', verbose)
if verbose:
print('averaging odd:')
average(odd, newReferenceName + 'odd.em', verbose)
def average_sub_pl(self, pl, name_prefix, weight_average):
"""For worker node, do two things, averaging & obtaining the even/odd partitions to save some time.
@param pl: particle list
@type ps: L{pytom.basic.structures.ParticleList}
@param name_prefix: name prefix output densities
@type name_prefix: C{str}
@param weight_average: weighted average
@type weight_average: C{str}
"""
from pytom.basic.structures import ParticleList
even = ParticleList('.')
odd = ParticleList('.')
for i in range(len(pl)):
if i % 2 == 0:
even.append(pl[i])
else:
odd.append(pl[i])
even.average(name_prefix + 'even.em',
progressBar=False,
createInfoVolumes=False,
_mpiParallel=False,
weighting=weight_average)
odd.average(name_prefix + 'odd.em',
progressBar=False,
createInfoVolumes=False,
_mpiParallel=False,
weighting=weight_average)
def initialize(pl, settings):
from pytom.basic.structures import Particle
# from pytom.alignment.alignmentFunctions import average2
from pytom.basic.filter import lowpassFilter
print("Initializing the class centroids ...")
pl = pl.copy()
pl.sortByScore()
if settings["noise"]:
pl = pl[:int((1-settings["noise"])*len(pl))]
K = settings["ncluster"]
freq = settings["frequency"]
kn = len(pl)//K
references = {}
frequencies = {}
# get the first class centroid
pp = pl[:kn]
# avg, fsc = average2(pp, norm=True, verbose=False)
pp.setClassAllParticles('0')
res, tmp, tmp2 = calculate_averages(pp, settings["binning"], None, outdir=settings["output_directory"])
avg = res['0']
avg = lowpassFilter(avg, freq, freq/10.)[0]
avg.write(os.path.join(settings['output_directory'], 'initial_0.em') )
p = Particle(os.path.join(settings['output_directory'], 'initial_0.em'))
p.setClass('0')
references['0'] = p
frequencies['0'] = freq
for k in range(1, K):
distances = [4]*len(pl)
for c, ref in references.items():
args = list(zip(pl, [ref]*len(pl), [freq]*len(pl), [settings["fmask"]]*len(pl), [settings["binning"]]*len(pl)))
dist = mpi.parfor(distance, args)
for i in range(len(pl)):
if distances[i] > dist[i]:
distances[i] = dist[i]
distances = np.asarray(distances)
print('sum distances: ', distances.sum())
distances = distances/np.sum(distances)
idx = np.random.choice(len(pl), kn, replace=False, p=distances)
pp = ParticleList()
for i in idx:
pp.append(pl[int(i)])
# avg, fsc = average2(pp, norm=True, verbose=False)
pp.setClassAllParticles('0')
res, tmp, tmp2 = calculate_averages(pp, settings["binning"], None, outdir=settings["output_directory"])
avg = res['0']
avg = lowpassFilter(avg, freq, freq/10.)[0]
kname = os.path.join(settings['output_directory'], 'initial_{}.em'.format(k))
avg.write(kname)
p = Particle(kname)
p.setClass(str(k))
references[str(k)] = p
frequencies[str(k)] = freq
return references, frequencies
def writeSetOfVolumes(volSet, volXml, volDir):
""" Convert a SetOfVolumes to a xml file used by PyTom.
The volumes will be converted to .mrc format if not are '.em' or '.mrc'
Params:
volSet: input SetOfVolumes.
volXml: filename where to write the xml file.
volDir: where to create links or copies (converted to mrc).
"""
# Add to the path the root to pytom
backupPath = list(sys.path)
addPyTomPaths()
from pytom.basic.structures import Particle, ParticleList, Wedge, SingleTiltWedge
from pytom.score.score import Score, PeakPrior, xcfScore
from pytom.frm.FRMAlignment import FRMScore
w = SingleTiltWedge()
#s = PeakPrior()
pl = ParticleList()
ih = em.convert.ImageHandler()
for vol in volSet:
index, fn = vol.getLocation()
convert = True # by default, convert, which is the save way
if index == em.NO_INDEX: # means single volumes
volName = os.path.basename(fn)
if fn.endswith('.em') or fn.endswith('.mrc'):
convert = False # we can just create a link in this case
else:
volName = 'volume_%03d.mrc' % vol.getObjId()
volFn = os.path.join(volDir, volName)
if convert:
ih.convert(vol, volFn)
else:
pwutils.createLink(fn, volFn)
# Make the volumes names relative to the xml file
# where the programs will be executed
volRel = os.path.relpath(volFn, os.path.dirname(volXml))
p = Particle()
s = xcfScore()
s.setValue(1.0)
pytomInfo = getattr(vol, 'pytomInfo', None)
if pytomInfo is None:
p.setWedge(w)
else:
p.fromXML(pytomInfo.get()) # Get stored XML format from PyTom
p.setFilename(volRel)
p.setScore(s)
pl.append(p)
pl.toXMLFile(volXml)
#pl.setWedgeAllParticles(w)
sys.path = backupPath
def simulationDescriptionToParticleList(directory,prefix = ''):
"""
simulationDescriptionToParticleList:
"""
lenDir = len(directory)
if not directory[lenDir-1] == '/':
directory = directory + '/'
xmlFile = directory + 'desc.xml'
from lxml import etree
simulationXML = etree.parse(xmlFile)
#print etree.tostring(simulationXML,pretty_print=True)
particles = simulationXML.xpath('particle')
parameters = simulationXML.xpath('Simulation_Parameters')
wedge = int(parameters[0].get('wangleEnd'))/2
from pytom.basic.structures import Particle,ParticleList,WedgeInfo
wi = WedgeInfo(wedge,[0.0,0.0,0.0])
pl = ParticleList(directory)
for particle in particles:
filename = prefix + particle.get('filename')
rotation = particle.get('rotation')
rotation = rotation.replace('[','')
rotation = rotation.replace(']','')
rotation = rotation.split(',')
rotation = [float(rotation[0]),float(rotation[1]),float(rotation[2])]
shift = particle.get('shift')
shift = shift.replace('[','')
shift = shift.replace(']','')
shift = shift.split(',')
shift = [int(round(float(shift[0]))),int(round(float(shift[1]))),int(round(float(shift[2])))]
p = Particle(filename,rotation = rotation,shift = shift,wedge=wi)
pl.append(p)
#al.toXMLFile(directory+'AlignmentList.xml')
return pl
def average_sub_pl(self, pl, name_prefix):
"""For worker node, this function has been rewritten.
"""
from pytom.basic.structures import ParticleList
even = ParticleList('.')
odd = ParticleList('.')
for i in range(len(pl)):
if i%2 == 0:
even.append(pl[i])
else:
odd.append(pl[i])
self.sum_sub_pl(even, name_prefix+'even')
self.sum_sub_pl(odd, name_prefix+'odd')
def toParticleList(self):
"""
toParticleList: Converts this object to a Particle List
@return: A particle list that can be used for further processing
@rtype: L{pytom.basic.structures.ParticleList}
"""
from pytom.basic.structures import ParticleList
pl = ParticleList('/')
for result in self._alignmentList:
p = result.toParticle()
pl.append(p)
return pl
def average_sub_pl(self, pl, name_prefix, weight_average):
"""For worker node, do two things, averaging & obtaining the even/odd partitions to save some time.
"""
from pytom.basic.structures import ParticleList
even = ParticleList('.')
odd = ParticleList('.')
for i in xrange(len(pl)):
if i % 2 == 0:
even.append(pl[i])
else:
odd.append(pl[i])
even.average(name_prefix + 'even.em',
progressBar=False,
createInfoVolumes=False,
_mpiParallel=False,
weighting=weight_average)
odd.average(name_prefix + 'odd.em',
progressBar=False,
createInfoVolumes=False,
_mpiParallel=False,
weighting=weight_average)
def average_sub_pl(self, pl, name_prefix):
"""For worker node, this function has been rewritten.
"""
from pytom.basic.structures import ParticleList
even = ParticleList('.')
odd = ParticleList('.')
for p in pl:
i = int(p.getClass())
if i % 2 == 0:
even.append(p)
else:
odd.append(p)
assert len(
even) > 0, "average_sub_pl: length even particle list == 0 :("
assert len(odd) > 0, "average_sub_pl: length odd particle list == 0 :("
# in some rare cases this would fail
# that is: this worker only get one class, in this case, one of the pl will be null
# just restart the job
self.sum_sub_pl(even, name_prefix + 'even')
self.sum_sub_pl(odd, name_prefix + 'odd')
def applySymmetryOnParticleList(particleList,symmetry):
"""
applySymmetryOnParticleList
@deprecated: use L{pytom.basic.structures.Symmetry.apply} instead!
"""
if symmetry.isOneFold():
return particleList
from pytom.basic.structures import ParticleList,Rotation
newList = ParticleList(particleList.getDirectory())
for i in range(len(particleList)):
particle = particleList[i]
originalRotation = particle.getRotation()
symmetry.setPsi(originalRotation.getPsi())
symmetry.setTheta(originalRotation.getTheta())
phi = originalRotation.getPhi()
angleList = symmetry.getAngleList(phi)
rotation = angleList.nextRotation()
rotation = angleList.nextRotation()
while not rotation == [None,None,None]:
p2 = particle.copy()
p2.setRotation(Rotation(rotation))
newList.append(p2)
rotation = angleList.nextRotation()
return newList + particleList
def growingAverageNew(particleList=None,angleObject=None,maskFile=None,scoreObject=None,startClassNumber=0,destinationDirectory='.',preprocessing = None,binning=1,verbose=False):
"""
"""
from pytom.alignment.alignmentFunctions import bestAlignment
from pytom.basic.structures import Reference,Particle,Rotation,ParticleList
from pytom.alignment.preprocessing import Preprocessing
if not preprocessing:
preprocessing = Preprocessing()
numberOfClasses = len(particleList.splitByClass())
if verbose:
print('Processing ' + str(numberOfClasses) + ' classes.')
print('Generating start average')
startAverageList = particleList.particlesFromClass(float(startClassNumber))
startAverageList.average(destinationDirectory + '/GA_it0.em',progressBar=verbose)
currentReference = Reference(destinationDirectory + '/GA_it0.em')
growingAverageParticleList = ParticleList(particleList.getDirectory())
for p in startAverageList:
p.setRotation(Rotation(0,0,0))
growingAverageParticleList.append(p)
for i in range(2,numberOfClasses):
currentParticleList = particleList.particlesFromClass(float(i))
if verbose:
print('Generating ' + str(i) + '. class average')
currentParticleList.average(destinationDirectory + '/CA_it'+str(i)+'.em',progressBar=verbose)
currentParticle = Particle(destinationDirectory + '/CA_it'+str(i)+'.em',wedgeInfo=currentParticleList[0].getWedgeInfo())
if verbose:
print('Running alignment iteration ' + str(i))
print(currentParticle)
print(currentReference)
currentPeak = bestAlignment(currentParticle.getVolume(),currentReference.getVolume(),currentReference.getWeighting(),currentParticle.getWedgeInfo(),angleObject,scoreObject,maskFile,preprocessing=preprocessing,binning=binning)
if verbose:
print('Parameters determined:')
print(currentPeak)
for p in currentParticleList:
p.setRotation(currentPeak.getRotation())
p.setShift(currentPeak.getShift())
growingAverageParticleList.append(p)
if verbose:
print('Generating growing average ' + str(i))
growingAverageParticleList.average(destinationDirectory + '/GA_it'+ str(i) +'.em',progressBar=verbose)
currentReference = Reference(destinationDirectory + '/GA_it'+ str(i) +'.em')
angleObject.reset()
if particlePath[-1] != os.sep:
particlePath += os.sep
for particle in res:
newParticle = particle.toParticle()
newParticle.setWedge(wedge)
newParticle.setFilename(particlePath + newParticle.getFilename())
if scale != 1.0:
pi = newParticle.getPickPosition()
pi.setX(pi.getX() * scale)
pi.setY(pi.getY() * scale)
pi.setZ(pi.getZ() * scale)
newParticle.setPickPosition(pi)
pl.append(newParticle)
pl.toXMLFile(plFilename)
if motlFilename:
from pytom.basic.structures import ParticleList
pl = ParticleList()
for newParticle in res:
if scale != 1.0:
pi = newParticle.getPickPosition()
pi.setX(pi.getX() * scale)
pi.setY(pi.getY() * scale)
pi.setZ(pi.getZ() * scale)
newParticle.setPickPosition(pi)
dir_name, name_prefix, wedge_angle, output, bHelp = parse_script_options(
sys.argv[1:], helper)
except:
sys.exit()
if bHelp is True:
print(helper)
sys.exit()
if name_prefix is None:
name_prefix = ''
wedge_angle = float(wedge_angle)
w = SingleTiltWedge(wedge_angle)
pl = ParticleList()
all_files = os.listdir(dir_name)
for fname in all_files:
p = None
name_suffix = fname.split('.')[-1]
if len(name_prefix):
if name_prefix in fname and name_suffix in ['em', 'mrc']:
p = Particle(dir_name + '/' + fname)
else:
if name_suffix in ['em', 'mrc']:
p = Particle(dir_name + '/' + fname)
if p is not None:
pl.append(p)
pl.setWedgeAllParticles(w)
pl.toXMLFile(output)
if not dest_dir:
dest_dir = '.'
from pytom_volume import read, subvolume
v = read(vol_filename)
from pytom.basic.structures import ParticleList, Particle, WedgeInfo
pl = ParticleList("./")
pl.fromXMLFile(pl_filename)
def regulaize(xx, dim):
if xx*binning-radius < 0:
if 2*radius > dim:
raise Exception("Volume size to be cut is too big!")
return 0
if xx*binning+radius > dim:
if dim-2*radius < 0:
raise Exception("Volume size to be cut is too big!")
return dim-2*radius
return xx*binning-radius
res = ParticleList(dest_dir)
for p in pl:
x,y,z = p.getPickPosition().toVector()
x = regulaize(int(x), v.sizeX()); y = regulaize(int(y), v.sizeY()); z = regulaize(int(z), v.sizeZ())
new_vol = subvolume(v, x, y, z, 2*radius, 2*radius, 2*radius)
name = dest_dir+'/'+p.getFilename()
new_vol.write(name) # write the subtomograms to the disk
res.append(Particle(name, p.getRotation(), None, WedgeInfo(w), 1, p.getPickPosition(), p.getScore())) # append it to the particle list for alignment
res.toXMLFile(dest_dir+'/'+res_name)
def create_RandomParticleList(reffile,
pl_filename='pl.xml',
pdir='./testparticles',
nparticles=10):
"""
@param reffile: reference file
@type reffile: C{str}
@param nparticles: number of particles (default: 10)
@type nparticles: C{int}
@param pl_filename: particle list filename
@type pl_filename: C{str}
@param pdir: particle directory
@type pdir: C{str}
@return: particleList
@rtype: L{pytom.basic.ParticleList}
"""
from pytom.basic.structures import Particle, ParticleList, Rotation, Shift
from pytom_volume import vol, rotate, shift, read
from pytom.basic.transformations import general_transform_crop
from pytom.simulation.whiteNoise import add as addNoise
import random
from os import mkdir
from pytom.score.score import FLCFScore as score
try:
mkdir(pdir)
except FileExistsError:
print('directory ' + pdir + ' existed already - using this one')
random.seed(0)
pl = ParticleList(directory='./')
ref = read(reffile)
for ii in range(0, nparticles):
rot = Rotation(random.uniform(0, 360), random.uniform(0, 360),
random.uniform(0, 180))
shift = Shift(x=random.uniform(-5, 5),
y=random.uniform(-5, 5),
z=random.uniform(-5, 5))
rotvol = general_transform_crop(v=ref,
rot=rot,
shift=shift,
scale=None,
order=[0, 1, 2])
# add some noise
noisy = addNoise(volume=rotvol, SNR=1)
fname = pdir + '/particle_' + str(ii) + '.em'
noisy.write(fname)
p = Particle(filename=fname,
rotation=rot,
shift=shift,
wedge=None,
className=0,
pickPosition=None,
score=score,
sourceInfo=None)
p.setScoreValue(0.0)
pl.append(particle=p)
pl.setFileName(filename=pl_filename)
pl.toXMLFile(filename=pl_filename)
return pl
def classifyParticleList(particleList, alignmentLists, verbose=False):
"""
classifyParticleList: Classifies input particle list according to scores determined in alignment lists.
@param particleList:
@param alignmentLists:
@param verbose:
@return: Will return particle list in same order as input but with assigned class memberships.
"""
from pytom.basic.structures import ParticleList
classifiedParticleList = ParticleList(particleList.getDirectory())
for alignmentList in alignmentLists:
alignmentList.sortByParticleList(particleList)
for particleIndex in range(len(particleList)):
particle = particleList[particleIndex]
score = particle.getScore()
if verbose:
print(particle.getFilename())
if not score:
bestScore = -999999999
else:
bestScore = particle.getScore().getWorstValue()
bestResult = []
bestCluster = []
for alignmentIterator in range(len(alignmentLists)):
alignmentList = alignmentLists[alignmentIterator]
result = alignmentList[particleIndex]
score = result.getScore()
value = float(score.getValue())
if verbose:
print(value)
if value >= bestScore:
bestResult = result
bestCluster = alignmentIterator
bestScore = value
if bestResult == []:
raise Exception('Could not determine best cluster for particle ' +
particle.getFilename())
classifiedParticle = bestResult.toParticle()
classifiedParticle.setClass(bestCluster)
if verbose:
print(classifiedParticle)
classifiedParticleList.append(classifiedParticle)
return classifiedParticleList
if help is True:
print(helper)
sys.exit()
cubeSize = int(cubeSize)
particleList = ParticleList()
try:
particleList.fromXMLFile(plFilename)
except:
from pytom.localization.structures import readParticleFile
particles = readParticleFile(plFilename)
particleList = ParticleList()
for particle in particles:
particleList.append(particle.toParticle())
particlePath = particleList[0].getFilename()
particleFolder = particlePath[0:particlePath.rfind('/')]
if not checkDirExists(particleFolder):
os.makedirs(particleFolder)
prog = FixedProgBar(0, len(particleList) - 1, '')
newParticleList = ParticleList()
vol = read(volFilename)
volX = vol.sizeX()
volY = vol.sizeY()
volZ = vol.sizeZ()
def create_RandomParticleList(reffile,
pl_filename='pl.xml',
pdir='./testparticles',
nparticles=10):
"""
@param reffile: reference file
@type reffile: C{str}
@param nparticles: number of particles (default: 10)
@type nparticles: C{int}
@param pl_filename: particle list filename
@type pl_filename: C{str}
@param pdir: particle directory
@type pdir: C{str}
@return: particleList
@rtype: L{pytom.basic.ParticleList}
"""
from pytom.basic.structures import Particle, ParticleList, Rotation, Shift, Wedge
from pytom_volume import vol, rotate, shift, read
from pytom.basic.transformations import general_transform_crop
from pytom.basic.functions import initSphere
from pytom.simulation.whiteNoise import add as addNoise
import random
from os import mkdir
from pytom.score.score import FLCFScore as score
try:
mkdir(pdir)
except FileExistsError:
print('directory ' + pdir + ' existed already - using this one')
random.seed(0)
a = 0
wedge = Wedge(wedgeAngles=[30.0, 30.0], cutoffRadius=50.0)
pl = ParticleList(directory='./')
ref1 = read(reffile)
ref2 = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=45)
#ref2.write('testData/mask_45.em')
parts = {0: ref1, 1: ref2}
for ii in range(0, nparticles):
if not ii % 2:
ref = read(reffile)
else:
ref = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=30,
smooth=30 / 10)
rot = Rotation(random.uniform(0, 360), random.uniform(0, 360),
random.uniform(0, 180))
shift = Shift(x=a * random.uniform(-5, 5),
y=a * random.uniform(-5, 5),
z=a * random.uniform(-5, 5))
rotvol = general_transform_crop(v=ref,
rot=rot,
shift=shift,
scale=None,
order=[0, 1, 2])
# add some noise
noisy = addNoise(volume=rotvol, SNR=1)
fname = pdir + '/particle_' + str(ii) + '.em'
#noisy.write( fname)
p = Particle(filename=fname,
rotation=rot,
shift=shift,
wedge=wedge,
className=0,
pickPosition=None,
score=score,
sourceInfo=None)
p.setScoreValue(0.0)
wg = p.getWedge().getWedgeObject()
wg.apply(noisy, Rotation(0, 0, 0)).write(fname)
pl.append(particle=p)
pl.setFileName(filename=pl_filename)
pl.toXMLFile(filename=pl_filename)
return pl
def classifyParticleListThreads(particleList,
alignmentLists,
criterion,
temperature,
verbose=False):
"""
classifyParticleListThreads: Same as above, but distributes classifyParticleList to threads
@param particleList:
@param alignmentLists:
@param criterion:
@param temperature:
@param verbose: Print classification process. (Default is False)
"""
from pytom.basic.structures import ParticleList
from pytom.cluster.mcoEXMXStructures import SwapList
for alignmentList in alignmentLists:
alignmentList.sortByParticleList(particleList)
temperature.initializeTemperature(alignmentLists)
priorClassNumber = len(particleList.splitByClass())
iteration = 0
clusterSizeSame = False
#prevent that annealing deletes clusters.
#leave at least one particle in class! repeat for 10 iterations if criterion.getAllowClassCollapse() == False!
while iteration < 10 and not clusterSizeSame:
swapList = SwapList()
classifiedParticleList = ParticleList(particleList.getDirectory())
for particleIndex in range(len(particleList)):
particle = particleList[particleIndex]
results = []
for alignmentIterator in range(len(alignmentLists)):
alignmentList = alignmentLists[alignmentIterator]
results.append(alignmentList[particleIndex])
#results = sorted(results, key=lambda MaximisationResult: MaximisationResult.getScore().getValue(),reverse=True)
if verbose:
print('')
print('')
print('')
print('----------------------')
print(results)
[bestResult, bestCluster,
swapList] = criterion.apply(results, temperature, swapList, False)
if bestResult == []:
raise RuntimeError(
'Could not determine best cluster for particle ' +
particle.getFilename())
classifiedParticle = bestResult.toParticle()
classifiedParticle.setClass(bestCluster)
classifiedParticle.setRotation(particle.getRotation())
classifiedParticle.setShift(particle.getShift())
if verbose:
print(classifiedParticle)
classifiedParticleList.append(classifiedParticle)
iteration += 1
clusterSizeSame = priorClassNumber == len(
classifiedParticleList.splitByClass(False))
clusterSizeSame = clusterSizeSame or criterion.getAllowClassCollapse()
return [classifiedParticleList, swapList]
def createClassificationResultDictionaries(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: A dictionary that maps new classnames to groundTruthClasses
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleList)
newClassesToGroundTruthMap = {
} #maps new class name to a ground truth class name
gtClassNames = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNames[particle.getClassName()] = False
if verbose:
print('gtClassNames : ', gtClassNames)
groundTruthParticleListXML = groundTruthParticleList.toXML()
for newClass in classifiedParticleList.splitByClass():
if verbose:
print('')
print('newClass : ', newClass)
particlesFromGroundTruth = ParticleList('/')
for particle in newClass:
#collect all particles that were assigned to newClass
particleName = particle.getFilename()
#gtParticle = groundTruthParticleList.getParticleByFilename(particleName)
particleXML = groundTruthParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' + str(particleName) +
'"]')
gtParticle = Particle('a')
gtParticle.fromXML(particleXML[0])
particlesFromGroundTruth.append(gtParticle)
if verbose:
print('len(particlesFromGroundTruth) : ',
len(particlesFromGroundTruth))
#sort classes according to size to descending order
sortedClasses = sorted(particlesFromGroundTruth.splitByClass(),
key=lambda x: len(x),
reverse=True)
classWasAssigned = False
classIndex = 0
if verbose:
print('len(sortedClasses) : ', len(sortedClasses))
while not classWasAssigned and classIndex < len(sortedClasses):
sortedClass = sortedClasses[classIndex]
className = sortedClass[0].getClassName()
if verbose:
print('className : ' + className)
print('len(sortedClass) : ', len(sortedClass))
classWasAssigned = not gtClassNames[className]
if verbose:
print('classWasAssigned : ', classWasAssigned)
if not classWasAssigned:
classIndex = classIndex + 1
else:
gtClassNames[className] = True
newClassesToGroundTruthMap[
newClass[0].getClassName()] = className
if verbose:
print('gtClassNames : ', gtClassNames)
print('newClassesToGroundTruthMap : ',
newClassesToGroundTruthMap)
return newClassesToGroundTruthMap
def assessClassification(classifiedParticleList,
groundTruthParticleList,
verbose=False):
"""
assessClassification: Comment in LB 31.1.2011
@param classifiedParticleList: list of classified particles
@param groundTruthParticleList: ground truth
@param verbose: If True, will print the class dictionaries generated. Default is false.
@return: [trueHits,falseHits,trueHitsPercent,falseHitsPercent,numberClusters,[clusterSizes]]
"""
from pytom.basic.structures import Particle, ParticleList
#from file if filename
if classifiedParticleList.__class__ == str:
classifiedParticleListFile = classifiedParticleList
classifiedParticleList = ParticleList('/')
classifiedParticleList.fromXMLFile(classifiedParticleListFile)
#from file if filename
if groundTruthParticleList.__class__ == str:
groundTruthParticleListFile = groundTruthParticleList
groundTruthParticleList = ParticleList('/')
groundTruthParticleList.fromXMLFile(groundTruthParticleListFile)
gtClassNamesAssigned = {} #maps if one class has been determined
for gtClass in groundTruthParticleList.splitByClass():
particle = gtClass[0]
gtClassNamesAssigned[particle.getClassName()] = False
if verbose:
print('GT Classes ', gtClassNamesAssigned)
gtClassesPerClass = {}
newClasses = classifiedParticleList.splitByClass()
newClassNamesAssigned = {}
numberClasses = len(newClasses)
classSizes = []
for i in range(len(newClasses)):
classSizes.append(len(newClasses[i]))
for newClass in newClasses:
newClassParticleList = ParticleList(newClass.getDirectory())
newClassNamesAssigned[newClass[0].getClassName()] = False
for particle in newClass:
pp = groundTruthParticleList[particle.getFilename()]
newClassParticleList.append(pp)
gtClassSizeDictionary = {}
for gtClass in newClassParticleList.splitByClass():
particle = gtClass[0]
gtParticle = groundTruthParticleList[particle.getFilename()]
gtClassSizeDictionary[gtParticle.getClassName()] = len(gtClass)
gtClassesPerClass[newClass[0].getClassName()] = [
newClassParticleList, gtClassSizeDictionary
]
if verbose:
print('Class distribution dictionary')
for k in list(gtClassesPerClass.keys()):
print(k, gtClassesPerClass[k])
gtToClassDictionary = {}
for gtName in list(gtClassNamesAssigned.keys()):
newClassIndex = 0
classSizeList = []
largestClass = -1
maxClassName = 'unknown'
assigned = False
for newClassName in list(gtClassesPerClass.keys()):
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
if verbose:
print('GT Name', gtName, ' New Class Name', newClassName)
print('GT Name Size', gtClassSizeDictionary)
try:
if verbose:
print(gtClassSizeDictionary[gtName])
if largestClass < gtClassSizeDictionary[
gtName] and not newClassNamesAssigned[newClassName]:
largestClass = gtClassSizeDictionary[gtName]
maxClassName = newClassName
if verbose:
print('SWAP')
except KeyError:
pass
gtToClassDictionary[gtName] = maxClassName
newClassNamesAssigned[maxClassName] = True
gtClassNamesAssigned[gtName] = True
for newClassName in list(gtClassesPerClass.keys()):
try:
l = gtClassesPerClass[newClassName]
gtClassSizeDictionary = l[1]
del gtClassSizeDictionary[gtName]
l[1] = gtClassSizeDictionary
gtClassesPerClass[newClassName] = l
except KeyError:
pass
if verbose:
print('GT to New Dictionary')
print(gtToClassDictionary)
trueHits = 0
falseHits = 0
classifiedParticleListXML = classifiedParticleList.toXML()
for gtParticle in groundTruthParticleList:
particleXML = classifiedParticleListXML.xpath(
'/ParticleList/Particle[@Filename="' +
str(gtParticle.getFilename()) + '"]')
particle = Particle('a')
particle.fromXML(particleXML[0])
if particle.getClassName() == gtToClassDictionary[
gtParticle.getClassName()]:
trueHits += 1
else:
falseHits += 1
return [
trueHits, falseHits,
float(trueHits) / len(classifiedParticleList),
float(falseHits) / len(classifiedParticleList), numberClasses,
classSizes
]
