def __init__(self,
particleList=None,
angleObject=None,
maskFile=None,
scoreObject=None,
startParticleNumber=0,
destinationDirectory='.',
preprocessing=None):
from pytom.tools.files import checkDirExists
from pytom.angles.angleList import AngleList
from pytom.basic.structures import ParticleList
self.particleList = particleList or ParticleList('/')
self.angleObject = angleObject or AngleList()
self.startParticleNumber = startParticleNumber
self.maskFile = maskFile or None
if preprocessing:
self.preprocessing = preprocessing
else:
from pytom.alignment.preprocessing import Preprocessing
self.preprocessing = Preprocessing()
if self.maskFile.__class__ == str:
from pytom.basic.structures import Mask
self.maskFile = Mask(self.maskFile)
self.score = scoreObject
if not checkDirExists(destinationDirectory):
raise Exception('Destination directory ' + destinationDirectory +
' does not exist.')
if not destinationDirectory[len(destinationDirectory) - 1] == '/':
destinationDirectory = destinationDirectory + '/'
self.destinationDirectory = destinationDirectory
except Exception as e:
print(e)
sys.exit()
if help is True:
print(helper)
sys.exit()
if not checkFileExists(particleList):
raise RuntimeError('ParticleList file ' + particleList +
' does not exist!')
if not checkFileExists(mask):
raise RuntimeError('Mask file ' + mask + ' does not exist!')
if not checkDirExists(destination):
raise RuntimeError('Destination directory ' + destination +
' does not exist!')
from pytom.cluster.mcoEXMXStructures import MCOEXMXJob
from pytom.basic.structures import ParticleList, Reference, Mask, Wedge, SampleInformation, PointSymmetry
from pytom.score.score import FLCFScore
from pytom.frontend.serverpages.createMCOEXMXJob import createRunscripts
from pytom.alignment.preprocessing import Preprocessing
p = ParticleList()
p.fromXMLFile(particleList)
m = Mask(mask)
w = Wedge([float(wedge1), float(wedge2)])
pre = Preprocessing(lowestFrequency=float(lowestFrequency),
highestFrequency=float(highestFrequency))
projectionDirectory = projectionDirectoryTemplate.replace(
'_CLOSEST_', '_{:04d}_'.format(markerIndex))
alignResultFile = os.path.join(projectionDirectory,
'alignmentResults.txt')
if not os.path.exists(alignResultFile):
alignResultFile = ''
else:
alignmentResults = loadstar(alignResultFile, dtype=datatypeAR)
projectionsFileNames = alignmentResults['FileName']
projectionDirectory = os.path.dirname(projectionsFileNames[0])
prefix = os.path.basename(
projectionsFileNames[0]).split('_')[0] + '_'
if checkFileExists(projectionList):
projections.fromXMLFile(projectionList)
elif checkDirExists(projectionDirectory):
projections.loadDirectory(projectionDirectory,
metafile=metafile,
prefix=prefix)
else:
raise RuntimeError(
'Neither projectionList existed nor the projectionDirectory you specified! Abort'
)
# transform the cropping offset
if len(projections) == 0:
print(markerIndex, projectionDirectory, metafile, prefix)
continue
tmp = projections[0]
sx = tmp.getXSize(
def distributeExpectation(particleLists,
iterationDirectory,
averagePrefix,
verbose=False,
symmetry=None):
"""
distributeExpectation: Distributes particle expectation (averaging) to multiple workers. Required by many algorithms such as MCOEXMX
@param particleLists: list of particleLists
@param iterationDirectory:
@param averagePrefix:
@param verbose:
@param symmetry:
"""
import pytom_mpi
from pytom.tools.files import checkDirExists
from pytom.parallel.alignmentMessages import ExpectationJobMsg, ExpectationResultMsg
from pytom.alignment.structures import ExpectationJob
from pytom.basic.structures import Reference, ReferenceList
from os import mkdir
if not pytom_mpi.isInitialised():
pytom_mpi.init()
mpi_myid = pytom_mpi.rank()
if not mpi_myid == 0:
raise RuntimeError(
'This function (distributeExpectation) can only be processed by mpi_id = 0! ID == '
+ str(mpi_myid) + ' Aborting!')
if not checkDirExists(iterationDirectory):
raise IOError('The iteration directory does not exist. ' +
iterationDirectory)
mpi_numberNodes = pytom_mpi.size()
if mpi_numberNodes <= 1:
raise RuntimeError(
'You must run clustering with openMPI on multiple CPUs')
listIterator = 0
referenceList = ReferenceList()
#distribute jobs to all nodes
for i in range(1, mpi_numberNodes):
if verbose:
print('Starting first job distribute step')
if listIterator < len(particleLists):
if not checkDirExists(iterationDirectory + 'class' +
str(listIterator) + '/'):
mkdir(iterationDirectory + 'class' + str(listIterator) + '/')
averageName = iterationDirectory + 'class' + str(
listIterator) + '/' + averagePrefix + '-' + str(
listIterator) + '.em'
if not symmetry.isOneFold():
newPl = symmetry.apply(particleLists[listIterator])
job = ExpectationJob(newPl, averageName)
else:
job = ExpectationJob(particleLists[listIterator], averageName)
newReference = Reference(averageName, particleLists[listIterator])
referenceList.append(newReference)
jobMsg = ExpectationJobMsg(0, str(i))
jobMsg.setJob(job)
pytom_mpi.send(str(jobMsg), i)
if verbose:
print(jobMsg)
listIterator = listIterator + 1
finished = False
#there are more jobs than nodes. continue distributing and collect results
receivedMsgCounter = 0
while not finished:
#listen and collect
mpi_msgString = pytom_mpi.receive()
if verbose:
print(mpi_msgString)
jobResultMsg = ExpectationResultMsg('', '')
jobResultMsg.fromStr(mpi_msgString)
receivedMsgCounter = receivedMsgCounter + 1
#send new job to free node
if listIterator < len(particleLists):
if not checkDirExists(iterationDirectory + 'class' +
str(listIterator) + '/'):
mkdir(iterationDirectory + 'class' + str(listIterator) + '/')
averageName = iterationDirectory + 'class' + str(
listIterator) + '/' + averagePrefix + '-' + str(
listIterator) + '.em'
job = ExpectationJob(particleLists[listIterator], averageName)
newReference = Reference(averageName, particleLists[listIterator])
referenceList.append(newReference)
jobMsg = ExpectationJobMsg(0, str(jobResultMsg.getSender()))
jobMsg.setJob(job)
pytom_mpi.send(str(jobMsg), i)
if verbose:
print(jobMsg)
listIterator = listIterator + 1
finished = listIterator >= len(
particleLists) and receivedMsgCounter == len(particleLists)
return referenceList
def mcoEXMX(mcoEMJob, doFinalize=True, verbose=False):
"""
mcoEXMX: Perfomrs kmeans clustering on particleList
@param mcoEMJob: The clustering job
@param doFinalize: Send finalize msgs to workers or not. Default is true
@param verbose: Default is false
"""
import pytom_mpi
if doFinalize:
pytom_mpi.init()
if pytom_mpi.rank() == 0:
from pytom.alignment.ExMaxAlignment import ExMaxManager
from pytom.tools.files import checkDirExists
from os import mkdir
from pytom.basic.plot import plotClassSizes
from builtins import min as minList
particleList = mcoEMJob.getParticleList()
if len(particleList) == 0:
raise RuntimeError('Particle list is empty! Abort!')
destinationDirectory = mcoEMJob.getDestinationDirectory()
numberIterations = mcoEMJob.getNumberIterations()
numberClasses = mcoEMJob.getNumberClasses()
exMaxJob = mcoEMJob.getExMaxJob()
if verbose:
print(mcoEMJob)
if not checkDirExists(destinationDirectory):
raise IOError('Destination directory ' + destinationDirectory +
' not found!')
try:
particleLists = particleList.splitByClass()
if len(particleLists) < 1 or (len(particleLists) == 1 and len(
particleLists[0]) == len(particleList)):
raise Exception()
except Exception:
from pytom.cluster.clusterFunctions import randomiseParticleListClasses
if numberClasses:
if verbose:
print('Randomising particle list')
particleList = randomiseParticleListClasses(
particleList, numberClasses)
particleList.toXMLFile(destinationDirectory +
'/RandomisedParticleList.xml')
particleLists = particleList.splitByClass()
else:
raise RuntimeError(
'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!'
)
initialParticleList = particleList
previousParticleList = initialParticleList
iteration = 0
converged = False
doAdaptiveResolution = mcoEMJob.getAdaptiveResolution()
if doAdaptiveResolution:
preProcessing = exMaxJob.getPreprocessing()
highestFrequency = preProcessing.getHighestFrequency()
resolutionList = [highestFrequency] * len(particleLists)
while iteration < numberIterations and (not converged):
if verbose:
print('Running iteration ' + str(iteration) + ' of ' +
str(numberIterations))
iterationDirectory = destinationDirectory + '/' + str(
iteration) + '/'
if not checkDirExists(iterationDirectory):
mkdir(iterationDirectory)
#referenceList = ReferenceList()
alignmentLists = [None] * len(particleLists)
#generate cluster centers
referenceList = distributeExpectation(
particleLists, iterationDirectory,
'clusterCenter' + str(iteration), verbose,
exMaxJob.getSymmetry())
for classIterator in range(len(particleLists)):
classDirectory = iterationDirectory + 'class' + str(
classIterator) + '/'
#determine distance for all particles
refinementDirectory = classDirectory + 'refinement/'
if verbose:
print(refinementDirectory)
if not checkDirExists(refinementDirectory):
mkdir(refinementDirectory)
exMaxJob.setParticleList(particleList)
exMaxJob.setReference(referenceList[classIterator])
exMaxJob.setDestination(refinementDirectory)
#use adaptive resolution -> update lowpass filter
if doAdaptiveResolution and len(
resolutionList
) > 0 and resolutionList[classIterator] > 0:
# preProcessing = exMaxJob.getPreprocessing()
# resolution = resolutionList[classIterator]
# preProcessing.setHighestFrequency(resolution)
# exMaxJob.setPreprocessing(preProcessing)
preProcessing = exMaxJob.getPreprocessing()
resolution = minList(resolutionList) * 1.1
preProcessing.setHighestFrequency(resolution)
exMaxJob.setPreprocessing(preProcessing)
#run refinement
manager = ExMaxManager(exMaxJob)
exMaxJob.toXMLFile(classDirectory + 'Job.xml')
manager.distributeAlignment(verbose)
alignmentLists[classIterator] = manager.getAlignmentList()
alignmentLists[classIterator].toXMLFile(iterationDirectory +
'AlignmentList' +
str(classIterator) +
'.xml')
#perform classification here
if verbose:
print('Classifying after iteration ' + str(iteration))
particleList = classifyParticleList(initialParticleList,
alignmentLists, verbose)
particleList.toXMLFile(iterationDirectory +
'classifiedParticles.xml')
particleLists = particleList.splitByClass()
difference = previousParticleList.classDifference(particleList)
converged = mcoEMJob.getEndThreshold() >= difference[3]
#determine resolution in each class
if doAdaptiveResolution:
resolutionList = [-1] * len(particleLists)
for classIterator in range(len(particleLists)):
classList = particleLists[classIterator]
if len(classList) == 1:
#if there is only one particle in that class, override resolution
print(
'Class ', classIterator,
' has only 1 particle! Will be assigned the lowest resolution determined.'
)
continue
className = classList[0].getClass()
v = classList[0].getVolume()
cubeSize = v.sizeX()
resolution = classList.determineResolution(
criterion=0.5,
numberBands=cubeSize / 2,
mask=exMaxJob.getMask(),
verbose=False,
plot='',
keepHalfsetAverages=False,
halfsetPrefix='class' + str(className),
parallel=True)
#resolution = [Resolution in Nyquist , resolution in band, numberBands]
resolutionList[classIterator] = resolution[1]
print('Resolution for class ', classIterator,
' determined to ',
resolution[1], ' pixels. Class size is ',
len(classList), ' particles')
#get lowest resolution determined for classes with more than 1 particle
min = 999999999999999
for classIterator in range(len(particleLists)):
if min >= resolutionList[classIterator] and resolutionList[
classIterator] >= 0:
min = resolutionList[classIterator]
#set resolution for all classes with only 1 particle to lowest resolution
for classIterator in range(len(particleLists)):
if resolutionList[classIterator] < 0:
resolutionList[classIterator] = min
#set up for next round!
previousParticleList = particleList
iteration = iteration + 1
if doFinalize:
manager.parallelEnd()
pytom_mpi.finalise()
return [particleList, alignmentLists]
else:
from pytom.cluster.mcoEXMXStructures import MCOEXMXWorker
worker = MCOEXMXWorker()
worker.setDoAlignment(mcoEMJob.getDoAlignment())
worker.setFRMBandwidth(mcoEMJob.getFRMBandwidth())
worker.parallelRun()
pytom_mpi.finalise()
def run(parameters, verbose=False):
"""
run: Answers a http request depending on the provided parameters.
Parameters can be
XML=FILENAME
DIR=DIRNAME
ALIG=NAME
@param parameters:
"""
from pytom.basic.structures import ParticleList
from pytom.tools.files import checkDirExists, checkFileExists
if verbose:
print("Parsing particleList request!")
splitParameters = parameters.split('&')
if splitParameters.__class__ == list:
if verbose:
print(splitParameters)
for i in range(len(splitParameters)):
parameter = splitParameters[i]
split = parameter.split('=')
keyword = split[0]
argument = split[1]
if verbose:
print('Keyword : ', keyword)
print('Arguments : ', argument)
if keyword == 'XML':
from pytom.tools.files import readStringFile, getPytomPath
import io
from lxml import etree
if not checkFileExists(argument):
raise IOError('File not found!')
pl = ParticleList('/')
pl.fromXMLFile(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif keyword == 'DIR':
from pytom.tools.files import checkDirExists, getPytomPath, readStringFile
import io
from lxml import etree
if not checkDirExists(argument):
raise IOError('File not found!')
pl = ParticleList(argument)
pl.loadDirectory()
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif keyword == 'ALIG':
from pytom.tools.files import checkDirExists, getPytomPath, readStringFile
import io
from lxml import etree
if not checkDirExists(argument):
raise IOError('File not found!')
pl = ParticleList(argument)
pl.fromAlignmentFile(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif splitParameters.__class__ == str:
if verbose:
print(splitParameters)
particleList = ParticleList()
try:
particleList.fromXMLFile(plFilename)
except:
from pytom.localization.structures import readParticleFile
particles = readParticleFile(plFilename)
particleList = ParticleList()
for particle in particles:
particleList.append(particle.toParticle())
particlePath = particleList[0].getFilename()
particleFolder = particlePath[0:particlePath.rfind('/')]
if not checkDirExists(particleFolder):
os.makedirs(particleFolder)
prog = FixedProgBar(0, len(particleList) - 1, '')
newParticleList = ParticleList()
vol = read(volFilename)
volX = vol.sizeX()
volY = vol.sizeY()
volZ = vol.sizeZ()
i = 0
for particle in particleList:
i = i + 1 # even if some particles are skipped, the progress shall be updated
prog.update(i)
def run(parameters, verbose=False):
"""
run: Answers a http request depending on the provided parameters.
Parameters can be
XML=FILENAME
DIR=DIRNAME
@param parameters:
"""
from pytom.reconstruction.reconstructionStructures import ProjectionList
from pytom.tools.files import checkDirExists, checkFileExists
from pytom.frontend.serverpages.serverMessages import ErrorMessage
splitParameters = parameters.split('&')
if splitParameters.__class__ == list:
if verbose:
print(splitParameters)
for i in range(len(splitParameters)):
parameter = splitParameters[i]
split = parameter.split('=')
keyword = split[0]
argument = split[1]
if verbose:
print('Keyword : ', keyword)
print('Arguments : ', argument)
if keyword == 'XML':
from pytom.tools.files import readStringFile, getPytomPath
import io
from lxml import etree
if not checkFileExists(argument):
print(ErrorMessage('File / directory not found!'))
return str(ErrorMessage('File / directory not found!'))
pl = ProjectionList()
pl.fromXMLFile(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ProjectionList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif keyword == 'DIR':
from pytom.tools.files import checkDirExists, getPytomPath, readStringFile
import io
from lxml import etree
if not checkDirExists(argument):
print(ErrorMessage('File / directory not found!'))
return str(ErrorMessage('File / directory not found!'))
pl = ProjectionList()
pl.loadDirectory(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ProjectionList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
def cutParticlesFromTomogram(particleList,
cubeSize,
sourceTomogram,
coordinateBinning=0,
binningFactorOut=0):
"""
cutParticlesFromTomogram: Cuts out particles from source tomogram. Source tomograms set for each particle will be used. If they dont exist, sourceTomogram will be set otherwise from Particle.getSource() .
@param particleList:
@type particleList: L{pytom.basic.structures.ParticleList}
@param cubeSize: Size of cut out cubes
@type cubeSize: int
@param sourceTomogram: The source tomogram (either file name or volume).
@type sourceTomogram: L{str} or L{pytom_volume.vol}
@param coordinateBinning: binning factor affecting coordinates. was template matching processed on binned data? use fractions (1/2 , 1/4) if tomogram is binned and coordinates are from unbinned...
@param binningFactorOut: binning factor for final cubes
@author: Thomas Hrabe
"""
from pytom.tools.files import checkDirExists
destinationDirectory = particleList.getDirectory()
if not checkDirExists(destinationDirectory):
raise RuntimeError('The destination directory ' +
destinationDirectory +
' does not exist. Create directory first.')
from pytom_volume import read
cubeRadius = cubeSize / 2
for particle in particleList:
#
pickPosition = particle.getPickPosition()
x = int(pickPosition.getX())
y = int(pickPosition.getY())
z = int(pickPosition.getZ())
originTomogram = pickPosition.getOriginFilename()
if originTomogram == '':
originTomogram = sourceTomogram
#
if coordinateBinning > 0:
x = x * coordinateBinning
y = y * coordinateBinning
z = z * coordinateBinning
#
if not originTomogram or originTomogram == '':
originTomogram = sourceTomogram
try:
subcube = read(originTomogram, x - cubeRadius, y - cubeRadius,
z - cubeRadius, cubeSize, cubeSize, cubeSize, 0, 0,
0, binningFactorOut, binningFactorOut,
binningFactorOut)
except Exception:
print(particle)
assert False
subcube.write(particle.getFilename())
def multiRef_EXMXAlign(multiRefJob, doFinalize=True, verbose=False):
"""
multiRef_EXMXAlign: Performs multi reference alignment on a particle list
@param multiRefJob: The multi reference alignment job
@param doFinalize: Send finalize msgs to workers or not. Default is true
@param verbose: Default is false
"""
import pytom_mpi
if doFinalize:
pytom_mpi.init()
if pytom_mpi.rank() == 0:
from pytom.alignment.ExMaxAlignment import ExMaxManager
from pytom.tools.files import checkDirExists
from os import mkdir
from pytom.basic.resolution import bandToAngstrom, angstromToBand, angleFromResolution
particleList = multiRefJob.getParticleList()
initialParticleList = particleList
previousParticleList = initialParticleList
destinationDirectory = multiRefJob.getDestinationDirectory()
numberIterations = multiRefJob.getNumberIterations()
numberClasses = multiRefJob.getNumberClasses()
exMaxJob = multiRefJob.getExMaxJob()
p = particleList[0]
pVol = p.getVolume()
cubeSize = pVol.sizeX()
preprocessing = exMaxJob.getPreprocessing()
sampleInfo = exMaxJob.getSampleInformation()
if verbose:
print(multiRefJob)
if not checkDirExists(destinationDirectory):
raise IOError('Destination directory ' + destinationDirectory +
' not found!')
try:
particleLists = particleList.splitByClass()
if len(particleLists) <= 1:
raise Exception()
except Exception:
from pytom.cluster.clusterFunctions import randomiseParticleListClasses
if numberClasses:
if verbose:
print('Randomizing particle list')
particleList = randomiseParticleListClasses(
particleList, numberClasses)
particleList.toXMLFile(destinationDirectory +
'/RandomisedParticleList.xml')
particleLists = particleList.splitByClass()
else:
raise RuntimeError(
'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!'
)
iteration = 0
converged = False
while iteration < numberIterations and (not converged):
if verbose:
print('Running iteration ' + str(iteration) + ' of ' +
str(numberIterations))
iterationDirectory = destinationDirectory + '/' + str(
iteration) + '/'
if not checkDirExists(iterationDirectory):
mkdir(iterationDirectory)
#determine resolution of all classes
maxRes = 0
minRes = 1000000
if not checkDirExists(iterationDirectory + 'resolution/'):
mkdir(iterationDirectory + 'resolution/')
for classIterator in range(len(particleLists)):
currentParticleList = particleLists[classIterator]
if len(currentParticleList) > 1:
[resNyquist, resolutionBand,
numberBands] = currentParticleList.determineResolution(
criterion=exMaxJob.getFSCCriterion(),
numberBands=cubeSize / 2,
mask=exMaxJob.getMask(),
keepHalfsetAverages=False,
halfsetPrefix=iterationDirectory + 'resolution/' +
'class' + str(classIterator) + '_fsc-',
verbose=verbose)
else:
continue
resolutionAngstrom = bandToAngstrom(resolutionBand,
sampleInfo.getPixelSize(),
numberBands, 1)
#resolutionAngstrom = bandToAngstrom(resolutionBand,sampleInfo.getPixelSize(),numberBands,exMaxJob.getBinning() )
if resolutionBand > maxRes:
maxRes = resolutionBand
if resolutionBand < minRes:
minRes = resolutionBand
if verbose:
print(
'Class ', classIterator, ' - current resolution :' +
str(resolutionAngstrom) + ' Angstrom')
#set highest frequency according to user specification
band = maxRes
if not multiRefJob.getUseMaxResolution():
band = minRes
if band == numberBands:
#determineResolution returns numberBands for filter if fsc result is invalid. in that case, use nyquist /2 as filter setting
print('Warning MultiRefAlignment.py: LL 114')
print(
'Warning: Resolution determined for all classes was invalid. Will use Nyquist/2 for current iteration'
)
band = numberBands / 2
preprocessing.setHighestFrequency(band)
exMaxJob.setPreprocessing(preprocessing)
alignmentLists = [None] * len(particleLists)
#generate cluster centers
referenceList = distributeExpectation(
particleLists, iterationDirectory,
'clusterCenter' + str(iteration), verbose,
exMaxJob.getSymmetry())
for classIterator in range(len(particleLists)):
classDirectory = iterationDirectory + 'class' + str(
classIterator) + '/'
#determine distance for all particles
refinementDirectory = classDirectory + 'refinement/'
if verbose:
print(refinementDirectory)
if not checkDirExists(refinementDirectory):
mkdir(refinementDirectory)
exMaxJob.setParticleList(particleList)
exMaxJob.setReference(referenceList[classIterator])
exMaxJob.setDestination(refinementDirectory)
#run refinement
manager = ExMaxManager(exMaxJob)
manager.distributeAlignment(verbose)
alignmentLists[classIterator] = manager.getAlignmentList()
alignmentLists[classIterator].toXMLFile(iterationDirectory +
'AlignmentList' +
str(classIterator) +
'.xml')
#perform classification here
if verbose:
print('Classifying after iteration ' + str(iteration))
particleList = classifyParticleList(initialParticleList,
alignmentLists, verbose)
particleList.toXMLFile(iterationDirectory +
'classifiedParticles.xml')
particleLists = particleList.splitByClass()
difference = previousParticleList.classDifference(particleList)
converged = multiRefJob.getEndThreshold() >= difference[3]
#set up for next round!
previousParticleList = particleList
iteration = iteration + 1
if doFinalize:
manager.parallelEnd()
pytom_mpi.finalise()
return [particleList, alignmentLists]
else:
from pytom.alignment.ExMaxAlignment import ExMaxWorker
worker = ExMaxWorker()
worker.parallelRun()
pytom_mpi.finalise()
def mcoAC(annealingJob, doFinalize=True, verbose=False):
"""
mcoAC: Performs mcoAC clustering on particleList
@param annealingJob:
@param doFinalize: Send finalize messages to workers or not. Default is true. Should be false when this process is integrated into another parallel process.
@param verbose: Default is false
"""
import pytom_mpi
if doFinalize:
pytom_mpi.init()
if pytom_mpi.rank() == 0:
from pytom.cluster.mcoEXMX import mcoEXMX
from pytom.tools.files import checkDirExists
from os import mkdir, system
particleList = annealingJob.getParticleList()
if len(particleList) == 0:
raise RuntimeError('Particle list is empty! Abort!')
initialParticleList = particleList
previousParticleList = initialParticleList
destinationDirectory = annealingJob.getDestinationDirectory()
numberIterations = annealingJob.getNumberIterations()
numberClasses = annealingJob.getNumberClasses()
if verbose:
print(annealingJob)
if not checkDirExists(destinationDirectory):
raise IOError('Destination directory ' + destinationDirectory +
' not found!')
try:
particleLists = particleList.splitByClass()
if len(particleLists) <= 1 or (len(particleLists) == 1 and len(
particleLists[0]) == len(particleList)):
raise Exception()
except Exception:
from pytom.cluster.clusterFunctions import randomiseParticleListClasses
if numberClasses:
if verbose:
print('Randomizing particle list')
particleList = randomiseParticleListClasses(
particleList, numberClasses)
particleList.toXMLFile(destinationDirectory +
'/RandomisedParticleList.xml')
particleLists = particleList.splitByClass()
else:
raise RuntimeError(
'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!'
)
iteration = 0
converged = False
bestScoreSum = None
bestParticleList = None
while (not annealingJob.cooledDown()) and (not converged):
if verbose:
print('Running iteration ' + str(iteration) + ' of ' +
str(numberIterations))
iterationDirectory = destinationDirectory + '/' + str(
iteration) + '/'
mcoEXMXDirectory = iterationDirectory + 'mcoEXMX/'
if not checkDirExists(iterationDirectory):
mkdir(iterationDirectory)
annealingJob.setDestinationDirectory(mcoEXMXDirectory)
annealingJob.setParticleList(previousParticleList)
annealingJob.setNumberIterations(annealingJob.getLocalIncrement())
annealingJob.toXMLFile(iterationDirectory + 'annealingJob.xml')
#run local refinement
[pl, alignmentLists] = mcoEXMX(annealingJob,
doFinalize=False,
verbose=verbose)
annealingJob.setNumberIterations(numberIterations)
#store currently best solution
if iteration == 0 or (bestScoreSum < pl.sumOfScores()
and len(pl.splitByClass()) > 1):
bestScoreSum = pl.sumOfScores()
bestParticleList = pl
bestParticleList.toXMLFile(iterationDirectory +
'currentBestParticleList.xml')
#perform classification here
[particleList, swapList
] = classifyParticleList(initialParticleList, alignmentLists,
annealingJob.getCriterion(),
annealingJob.getTemperature().copy(),
verbose)
#save iteration results to disk
particleList.toXMLFile(iterationDirectory +
'classifiedParticles.xml')
swapList.toXMLFile(iterationDirectory + 'swapList.xml')
#if not verbose mode, delete mcoEXMX files
if not verbose:
system('rm -rf ' + mcoEXMXDirectory)
#print number class swaps
difference = previousParticleList.classDifference(particleList)
converged = annealingJob.getEndThreshold() >= difference[3]
previousParticleList = particleList
#set up for new round
annealingJob.decreaseTemperature()
particleLists = particleList.splitByClass()
iteration = iteration + 1
print('Annealing iteration ' + str(iteration) + ' finished!')
if doFinalize:
from pytom.alignment.ExMaxAlignment import ExMaxManager
manager = ExMaxManager(annealingJob.getExMaxJob())
manager.parallelEnd()
pytom_mpi.finalise()
return bestParticleList
else:
from pytom.cluster.mcoEXMXStructures import MCOEXMXWorker
worker = MCOEXMXWorker()
worker.setDoAlignment(annealingJob.getDoAlignment())
worker.setFRMBandwidth(annealingJob.getFRMBandwidth())
worker.parallelRun(verbose=verbose)
pytom_mpi.finalise()
print(xmlsBasedOnClosestMarker)
for markerIndex, particleListXMLPath in xmlsBasedOnClosestMarker:
markerIndex = int(markerIndex)
projections = ProjectionList()
projectionDirectory = projectionDirectoryTemplate.replace(
'_CLOSEST_', '_{:04d}_'.format(markerIndex))
alignResultFile = os.path.join(projectionDirectory,
'alignmentResults.txt')
if not os.path.exists(alignResultFile):
alignResultFile = ''
else:
alignmentResults = loadstar(alignResultFile, dtype=datatypeAR)
projectionsFileNames = alignmentResults['FileName']
projectionDirectory = os.path.dirname(projectionsFileNames[0])
print(projectionDirectory, checkDirExists(projectionDirectory))
prefix = os.path.basename(
projectionsFileNames[0]).split('_')[0] + '_'
if checkFileExists(projectionList):
projections.fromXMLFile(projectionList)
elif checkDirExists(projectionDirectory):
projections.loadDirectory(projectionDirectory,
metafile=metafile,
prefix=prefix)
else:
raise RuntimeError(
'Neither projectionList existed nor the projectionDirectory you specified! Abort'
)
# transform the cropping offset
if len(projections) == 0: