def parallelRun(self,verbose=False): import pytom_mpi from pytom.parallel.messages import StatusMessage,MessageError from pytom.basic.exceptions import ParameterError from pytom.basic.structures import PyTomClassError end = False if not pytom_mpi.isInitialised(): pytom_mpi.init() mpi_id = pytom_mpi.rank() while not end: #listen for messages mpi_msgString = pytom_mpi.receive() if verbose: print(mpi_msgString) try: #wait for job and start processing msg = ReconstructionMessage() msg.fromStr(mpi_msgString) self.setJob(msg) self.run() resultMsg = StatusMessage(mpi_id,'0') resultMsg.setStatus('Finished') pytom_mpi.send(str(resultMsg),0) except (MessageError,PyTomClassError,ParameterError): try: #as StatusMessage and finish msg = StatusMessage('','') msg.fromStr(mpi_msgString) if msg.getStatus() == 'End': end = True except (MessageError,PyTomClassError,ParameterError): #print mpi_msgString #raise MessageError('Message unknown!') print('Error parsing message. Message either unknown or invalid.') assert False except: print('wild except')
def growingAverage(particleClassLists,score,angleObject,mask,destinationDirectory,preprocessing,verbose=False): import pytom_mpi if not pytom_mpi.isInitialised(): pytom_mpi.init() if pytom_mpi.size() > 1: mpi_myid = pytom_mpi.rank() if mpi_myid == 0: manager = GAManager(particleClassLists,score,angleObject,mask,destinationDirectory,preprocessing) manager.parallelGA(verbose) manager.parallelEnd() else: w = GAWorker(mpi_myid) w.run() else: print('Processing in sequential mode') manager = GAManager(particleClassLists,score,angleObject,mask,destinationDirectory,preprocessing) manager.sequentialGA(verbose) pytom_mpi.finalise()
def __init__(self): import pytom_mpi if not pytom_mpi.isInitialised(): pytom_mpi.init() self.mpi_id = pytom_mpi.rank() self.name = 'node_' + str(self.mpi_id) self.runtimes = 0
def __init__(self, suffix=''): import pytom_mpi if not pytom_mpi.isInitialised(): pytom_mpi.init() self.suffix = suffix self.mpi_id = pytom_mpi.rank() self.name = 'node_' + str(self.mpi_id) self.clean()
def __init__(self): if not pytom_mpi.isInitialised(): pytom_mpi.init() self.mpi_id = pytom_mpi.rank() self.num_workers = pytom_mpi.size()-1 self.node_name = 'node_' + str(self.mpi_id) if self.num_workers < 1: raise RuntimeError("Not enough nodes to parallelize the job!")
def __init__(self): import pytom_mpi if not pytom_mpi.isInitialised(): pytom_mpi.init() self._mpi_id = pytom_mpi.rank() self._numberWorkers = pytom_mpi.size() - 1 self._jobList = []
def printProcessStats(): """ printProcessStats: Prints current memory usage of this pytom process to screen. If in mpi mode, will display mpi_id, too """ import os, pytom_mpi if pytom_mpi.isInitialised(): print('Usage of MPI-ID (', pytom_mpi.rank(), ')') pid = os.getpid() psCommand = 'ps -v -p ' + str(pid) os.system(psCommand)
def splitAngles(self, job, verbose=True): ''' splitAngles: Distribute the job to the workers by splitting angles @param job: job to be done @type job: L{pytom.localization.peak_job.PeakJob} @param verbose: verbose mode @type verbose: boolean ''' import pytom_mpi mpi_myid = pytom_mpi.rank() if not mpi_myid == 0: raise RuntimeError( 'This function can only be processed by mpi_id = 0! ID == ' + str(mpi_myid) + ' Aborting!') # split the job into smaller ones rotationsPerWorker = job.rotations.numberRotations() // self.numWorkers if rotationsPerWorker == 0: raise RuntimeError("Not enough angles to split!") if verbose == True: print( '\n\nManager: distribute number of %d rotations to %d workers' % (job.rotations.numberRotations(), self.numWorkers)) for i in range(1, self.numWorkers + 1): # split the rotations if i != self.numWorkers: subRot = job.rotations[(i - 1) * rotationsPerWorker:i * rotationsPerWorker] else: # the last node will take all the rest of rotations subRot = job.rotations[(i - 1) * rotationsPerWorker:] self.jobInfo[i] = (i - 1) * rotationsPerWorker from pytom.angles.angleList import AngleList rot = AngleList(subRot) from pytom.localization.peak_job import PeakJob subJob = PeakJob(volume=job.volume, reference=job.reference, mask=job.mask, wedge=job.wedge, rotations=rot, score=job.score, jobID=i, dstDir=job.dstDir, bandpass=job.bandpass) subJob.send(0, i)
def end(self, verbose=False): if verbose == True: print(self.node_name + ': sending end messages to others') from pytom.parallel.messages import StatusMessage mpi_numberNodes = pytom_mpi.size() mpi_myid = pytom_mpi.rank() for i in range(1, mpi_numberNodes): msg = StatusMessage(str(mpi_myid),str(i)) msg.setStatus("End") pytom_mpi.send(str(msg),i) pytom_mpi.finalise()
def parallelEnd(self): """ parallelEnd : Sends status message = end to all workers. All workers will terminate upon receiving this message. @author: Thomas Hrabe """ import pytom_mpi from pytom.parallel.messages import StatusMessage if not pytom_mpi.isInitialised(): pytom_mpi.init() mpi_myid = pytom_mpi.rank() mpi_numberNodes = pytom_mpi.size() for i in range(1, mpi_numberNodes): msg = StatusMessage(mpi_myid.__str__(), i.__str__()) msg.setStatus("End") pytom_mpi.send(msg.__str__(), i)
def parallelEnd(self, verbose=True): """ parallelEnd : Sends status message = end to all workers. @param verbose: verbose mode @type verbose: boolean """ if verbose == True: print('Manager: sending end messages to workers') import pytom_mpi from pytom.parallel.messages import StatusMessage mpi_numberNodes = pytom_mpi.size() mpi_myid = pytom_mpi.rank() for i in range(1, mpi_numberNodes): msg = StatusMessage(str(mpi_myid), str(i)) msg.setStatus("End") pytom_mpi.send(str(msg), i)
def distributeAverage(particleList,averageName,showProgressBar = False,verbose=False,createInfoVolumes = False,sendEndMessage = False): """ distributeAverage : Distributes averaging to multiple nodes @param particleList: The particles @param averageName: Filename of new average @param verbose: Prints particle information. Disabled by default. @param createInfoVolumes: Create info data (wedge sum, inverted density) too? False by default. @return: A new Reference object @rtype: L{pytom.basic.structures.Reference} @author: Thomas Hrabe """ import pytom_mpi mpiInitialized = pytom_mpi.isInitialised() mpiAvailable = False if not mpiInitialized: try: pytom_mpi.init() if pytom_mpi.size() > 1: mpiAvailable = True except: print('Could not initialize MPI properly! Running in sequential mode!') if mpiAvailable: if pytom_mpi.rank() == 0: return _disrtibuteAverageMPI(particleList,averageName,showProgressBar,verbose,createInfoVolumes,sendEndMessage) else: from pytom.alignment.ExMaxAlignment import ExMaxWorker worker = ExMaxWorker() worker.parallelRun(False) else: print('MPI not available') return average(particleList,averageName,showProgressBar,verbose,createInfoVolumes)
ScriptOption(['-v', '--verbose'], 'Verbose', False, False), ScriptOption(['-h', '--help'], 'Help.', False, False) ]) verbose = False if len(sys.argv) == 1: print helper sys.exit() try: jobFile, verbose, helpme = parse_script_options(sys.argv[1:], helper) except Exception: #print e sys.exit() if helpme is True: print helper sys.exit() job = MCOACJob(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) job.fromXMLFile(jobFile) verbose = verbose is True try: mcoAC(job, True, verbose) except: if pytom_mpi.rank() == 0: from pytom.alignment.ExMaxAlignment import ExMaxManager manager = ExMaxManager(annealingJob.getExMaxJob()) manager.parallelEnd() pytom_mpi.finalise()
def parallelStart_splitVol(self, job, splitX, splitY, splitZ, verbose=True): """ """ self.parallelInit() import pytom_mpi mpi_myid = pytom_mpi.rank() if mpi_myid == 0: # manager # distribute the job to the workers and get the original size of the volume self.splitVolumes(job, splitX, splitY, splitZ, verbose) [vsizeX, vsizeY, vsizeZ] = self.jobInfo["originalSize"] # gather results and post processing from pytom_volume import vol volume = vol(vsizeX, vsizeY, vsizeZ) volume.setAll(0) orient = vol(vsizeX, vsizeY, vsizeZ) orient.setAll(0) i = 0 while i < splitX * splitY * splitZ: mpi_msgString = getMsgStr() msg = self.getResMsg(mpi_msgString) resFromWorker = self.resFromMsg(msg) if verbose == True: print("Manager: processing result from worker " + msg.getSender()) resV = resFromWorker.result.getVolume() resO = resFromWorker.orient.getVolume() jobID = resFromWorker.jobID [sizeX, sizeY, sizeZ] = self.jobInfo["splitSize"] [sub_start, start] = self.jobInfo[jobID] from pytom_volume import subvolume, putSubVolume sub_resV = subvolume(resV, sub_start[0], sub_start[1], sub_start[2], sizeX, sizeY, sizeZ) sub_resO = subvolume(resO, sub_start[0], sub_start[1], sub_start[2], sizeX, sizeY, sizeZ) putSubVolume(sub_resV, volume, start[0], start[1], start[2]) putSubVolume(sub_resO, orient, start[0], start[1], start[2]) i = i + 1 # write the final result back to the disk volume.write(self.name + '_res.em') orient.write(self.name + '_orient.em') # delete the temporary files on the disk import os files = os.listdir('.') for name in files: if 'job' in name and '.em' in name and not 'sum' in name and not 'sqr' in name: os.remove(name) # rename the result files name os.rename('node_0_res.em', 'scores{}{}.em'.format(suffix)) os.rename('node_0_orient.em', 'angles{}{}.em'.format(suffix)) # finishing, stop all workers self.parallelEnd(verbose) if verbose == True: print("Manager: end") else: # worker worker = PeakWorker() worker.parallelRun(verbose) pytom_mpi.finalise()
def parallelStart_splitAng(self, job, verbose=True): ''' parallelStart_splitAng: Start the parallel working for the job in the way of splitting angles @param job: job to be done @type job: L{pytom.localization.peak_job.PeakJob} @param verbose: verbose mode @type verbose: boolean ''' self.parallelInit() import pytom_mpi import os mpi_myid = pytom_mpi.rank() if mpi_myid == 0: # manager # distribute the job to the workers self.splitAngles(job, verbose) # gather results and post processing i = 0 volume = None orient = None while i < self.numWorkers: mpi_msgString = getMsgStr() msg = self.getResMsg(mpi_msgString) resFromWorker = self.resFromMsg(msg) if verbose == True: print("Manager: processing result from worker " + msg.getSender()) resV = resFromWorker.result.getVolume() resO = resFromWorker.orient.getVolume() jobID = resFromWorker.jobID # correct the orientation information # n = int(msg.getSender()) # rotationsPerWorker = job.rotations.numberRotations()/self.numWorkers # offset = (n-1)*rotationsPerWorker # print "Manager: Offset is %d and %d" % (offset, self.jobInfo[jobID]) offset = self.jobInfo[jobID] resO = resO + offset if volume == None or orient == None: from pytom_volume import vol volume = vol(resV.sizeX(), resV.sizeY(), resV.sizeZ()) orient = vol(resO.sizeX(), resO.sizeY(), resO.sizeZ()) volume.copyVolume(resV) orient.copyVolume(resO) else: # self.procResFromWorker(volume, orient, resV, resO) from pytom_volume import updateResFromVol updateResFromVol(volume, resV, orient, resO) i = i + 1 # write the final result back to the disk volume.write(self.name + '_res.em') orient.write(self.name + '_orient.em') # post processing the sum and sqr volume print("Start post processing the sum and sqr volume ...") sumList = [] sqrList = [] filenames = os.listdir('.') for name in filenames: if name[-6:] == 'sum.em': sumList.append(name) elif name[-6:] == 'sqr.em': sqrList.append(name) else: pass sumV = None sqrV = None from pytom_volume import read for name in sumList: v = read(name) if sumV: sumV = sumV + v else: sumV = v for name in sqrList: v = read(name) if sqrV: sqrV = sqrV + v else: sqrV = v sumV = sumV / job.rotations.numberRotations() sqrV = sqrV / job.rotations.numberRotations() sumV.write('node_0_sum.em') sqrV.write('node_0_sqr.em') # delete the temporary files on the disk import os files = os.listdir('.') for name in files: if 'job' in name and '.em' in name: os.remove(name) # rename the result files name os.rename('node_0_res.em', 'scores.em') os.rename('node_0_orient.em', 'angles.em') # finishing, stop all workers self.parallelEnd(verbose) if verbose == True: print("Manager: end") else: # worker worker = PeakWorker() worker.parallelRun(verbose)
def distributedCorrelationMatrix(job, verbose=False): """ distributedCorrelationMatrix: Performs calculation of correlation matrix either on multiple processes or sequentially. """ import pytom_mpi pytom_mpi.init() if pytom_mpi.size() > 1: mpi_myid = pytom_mpi.rank() if mpi_myid == 0: manager = CMManager(job) manager.distributeCalculation(mpi_myid, verbose) manager.parallelEnd() manager.saveMatrix() else: from pytom.parallel.clusterMessages import CorrelationVectorMessage, CorrelationVectorJobMessage from pytom.parallel.messages import StatusMessage, MessageError end = False while not end: mpi_msg = pytom_mpi.receive() if verbose: print(mpi_msg) try: msg = CorrelationVectorJobMessage() msg.fromStr(mpi_msg) worker = CMWorker(msg.getJob()) #worker.dumpMsg2Log('node'+str(mpi_myid)+'.log', msg.__str__()) resultVector = worker.run() resultMessage = CorrelationVectorMessage(mpi_myid, 0) resultMessage.setVector(resultVector) #worker.dumpMsg2Log('node'+mpi_myid.__str__()+'.log', resultMessage.__str__()) if verbose and False: print(resultMessage) pytom_mpi.send(resultMessage.__str__(), 0) except (MessageError, RuntimeError, IndexError): msg = StatusMessage('', '') msg.fromStr(mpi_msg) if msg.getStatus() == 'End': end = True print('Node ' + mpi_myid.__str__() + ' finished') else: print('Sequential Processing! Running on one machine only!') manager = CMManager(job) manager.calculateMatrix() manager.saveMatrix() pytom_mpi.finalise()
def mcoEXMX(mcoEMJob, doFinalize=True, verbose=False): """ mcoEXMX: Perfomrs kmeans clustering on particleList @param mcoEMJob: The clustering job @param doFinalize: Send finalize msgs to workers or not. Default is true @param verbose: Default is false """ import pytom_mpi if doFinalize: pytom_mpi.init() if pytom_mpi.rank() == 0: from pytom.alignment.ExMaxAlignment import ExMaxManager from pytom.tools.files import checkDirExists from os import mkdir from pytom.basic.plot import plotClassSizes from builtins import min as minList particleList = mcoEMJob.getParticleList() if len(particleList) == 0: raise RuntimeError('Particle list is empty! Abort!') destinationDirectory = mcoEMJob.getDestinationDirectory() numberIterations = mcoEMJob.getNumberIterations() numberClasses = mcoEMJob.getNumberClasses() exMaxJob = mcoEMJob.getExMaxJob() if verbose: print(mcoEMJob) if not checkDirExists(destinationDirectory): raise IOError('Destination directory ' + destinationDirectory + ' not found!') try: particleLists = particleList.splitByClass() if len(particleLists) < 1 or (len(particleLists) == 1 and len( particleLists[0]) == len(particleList)): raise Exception() except Exception: from pytom.cluster.clusterFunctions import randomiseParticleListClasses if numberClasses: if verbose: print('Randomising particle list') particleList = randomiseParticleListClasses( particleList, numberClasses) particleList.toXMLFile(destinationDirectory + '/RandomisedParticleList.xml') particleLists = particleList.splitByClass() else: raise RuntimeError( 'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!' ) initialParticleList = particleList previousParticleList = initialParticleList iteration = 0 converged = False doAdaptiveResolution = mcoEMJob.getAdaptiveResolution() if doAdaptiveResolution: preProcessing = exMaxJob.getPreprocessing() highestFrequency = preProcessing.getHighestFrequency() resolutionList = [highestFrequency] * len(particleLists) while iteration < numberIterations and (not converged): if verbose: print('Running iteration ' + str(iteration) + ' of ' + str(numberIterations)) iterationDirectory = destinationDirectory + '/' + str( iteration) + '/' if not checkDirExists(iterationDirectory): mkdir(iterationDirectory) #referenceList = ReferenceList() alignmentLists = [None] * len(particleLists) #generate cluster centers referenceList = distributeExpectation( particleLists, iterationDirectory, 'clusterCenter' + str(iteration), verbose, exMaxJob.getSymmetry()) for classIterator in range(len(particleLists)): classDirectory = iterationDirectory + 'class' + str( classIterator) + '/' #determine distance for all particles refinementDirectory = classDirectory + 'refinement/' if verbose: print(refinementDirectory) if not checkDirExists(refinementDirectory): mkdir(refinementDirectory) exMaxJob.setParticleList(particleList) exMaxJob.setReference(referenceList[classIterator]) exMaxJob.setDestination(refinementDirectory) #use adaptive resolution -> update lowpass filter if doAdaptiveResolution and len( resolutionList ) > 0 and resolutionList[classIterator] > 0: # preProcessing = exMaxJob.getPreprocessing() # resolution = resolutionList[classIterator] # preProcessing.setHighestFrequency(resolution) # exMaxJob.setPreprocessing(preProcessing) preProcessing = exMaxJob.getPreprocessing() resolution = minList(resolutionList) * 1.1 preProcessing.setHighestFrequency(resolution) exMaxJob.setPreprocessing(preProcessing) #run refinement manager = ExMaxManager(exMaxJob) exMaxJob.toXMLFile(classDirectory + 'Job.xml') manager.distributeAlignment(verbose) alignmentLists[classIterator] = manager.getAlignmentList() alignmentLists[classIterator].toXMLFile(iterationDirectory + 'AlignmentList' + str(classIterator) + '.xml') #perform classification here if verbose: print('Classifying after iteration ' + str(iteration)) particleList = classifyParticleList(initialParticleList, alignmentLists, verbose) particleList.toXMLFile(iterationDirectory + 'classifiedParticles.xml') particleLists = particleList.splitByClass() difference = previousParticleList.classDifference(particleList) converged = mcoEMJob.getEndThreshold() >= difference[3] #determine resolution in each class if doAdaptiveResolution: resolutionList = [-1] * len(particleLists) for classIterator in range(len(particleLists)): classList = particleLists[classIterator] if len(classList) == 1: #if there is only one particle in that class, override resolution print( 'Class ', classIterator, ' has only 1 particle! Will be assigned the lowest resolution determined.' ) continue className = classList[0].getClass() v = classList[0].getVolume() cubeSize = v.sizeX() resolution = classList.determineResolution( criterion=0.5, numberBands=cubeSize / 2, mask=exMaxJob.getMask(), verbose=False, plot='', keepHalfsetAverages=False, halfsetPrefix='class' + str(className), parallel=True) #resolution = [Resolution in Nyquist , resolution in band, numberBands] resolutionList[classIterator] = resolution[1] print('Resolution for class ', classIterator, ' determined to ', resolution[1], ' pixels. Class size is ', len(classList), ' particles') #get lowest resolution determined for classes with more than 1 particle min = 999999999999999 for classIterator in range(len(particleLists)): if min >= resolutionList[classIterator] and resolutionList[ classIterator] >= 0: min = resolutionList[classIterator] #set resolution for all classes with only 1 particle to lowest resolution for classIterator in range(len(particleLists)): if resolutionList[classIterator] < 0: resolutionList[classIterator] = min #set up for next round! previousParticleList = particleList iteration = iteration + 1 if doFinalize: manager.parallelEnd() pytom_mpi.finalise() return [particleList, alignmentLists] else: from pytom.cluster.mcoEXMXStructures import MCOEXMXWorker worker = MCOEXMXWorker() worker.setDoAlignment(mcoEMJob.getDoAlignment()) worker.setFRMBandwidth(mcoEMJob.getFRMBandwidth()) worker.parallelRun() pytom_mpi.finalise()
def multiRef_EXMXAlign(multiRefJob, doFinalize=True, verbose=False): """ multiRef_EXMXAlign: Performs multi reference alignment on a particle list @param multiRefJob: The multi reference alignment job @param doFinalize: Send finalize msgs to workers or not. Default is true @param verbose: Default is false """ import pytom_mpi if doFinalize: pytom_mpi.init() if pytom_mpi.rank() == 0: from pytom.alignment.ExMaxAlignment import ExMaxManager from pytom.tools.files import checkDirExists from os import mkdir from pytom.basic.resolution import bandToAngstrom, angstromToBand, angleFromResolution particleList = multiRefJob.getParticleList() initialParticleList = particleList previousParticleList = initialParticleList destinationDirectory = multiRefJob.getDestinationDirectory() numberIterations = multiRefJob.getNumberIterations() numberClasses = multiRefJob.getNumberClasses() exMaxJob = multiRefJob.getExMaxJob() p = particleList[0] pVol = p.getVolume() cubeSize = pVol.sizeX() preprocessing = exMaxJob.getPreprocessing() sampleInfo = exMaxJob.getSampleInformation() if verbose: print(multiRefJob) if not checkDirExists(destinationDirectory): raise IOError('Destination directory ' + destinationDirectory + ' not found!') try: particleLists = particleList.splitByClass() if len(particleLists) <= 1: raise Exception() except Exception: from pytom.cluster.clusterFunctions import randomiseParticleListClasses if numberClasses: if verbose: print('Randomizing particle list') particleList = randomiseParticleListClasses( particleList, numberClasses) particleList.toXMLFile(destinationDirectory + '/RandomisedParticleList.xml') particleLists = particleList.splitByClass() else: raise RuntimeError( 'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!' ) iteration = 0 converged = False while iteration < numberIterations and (not converged): if verbose: print('Running iteration ' + str(iteration) + ' of ' + str(numberIterations)) iterationDirectory = destinationDirectory + '/' + str( iteration) + '/' if not checkDirExists(iterationDirectory): mkdir(iterationDirectory) #determine resolution of all classes maxRes = 0 minRes = 1000000 if not checkDirExists(iterationDirectory + 'resolution/'): mkdir(iterationDirectory + 'resolution/') for classIterator in range(len(particleLists)): currentParticleList = particleLists[classIterator] if len(currentParticleList) > 1: [resNyquist, resolutionBand, numberBands] = currentParticleList.determineResolution( criterion=exMaxJob.getFSCCriterion(), numberBands=cubeSize / 2, mask=exMaxJob.getMask(), keepHalfsetAverages=False, halfsetPrefix=iterationDirectory + 'resolution/' + 'class' + str(classIterator) + '_fsc-', verbose=verbose) else: continue resolutionAngstrom = bandToAngstrom(resolutionBand, sampleInfo.getPixelSize(), numberBands, 1) #resolutionAngstrom = bandToAngstrom(resolutionBand,sampleInfo.getPixelSize(),numberBands,exMaxJob.getBinning() ) if resolutionBand > maxRes: maxRes = resolutionBand if resolutionBand < minRes: minRes = resolutionBand if verbose: print( 'Class ', classIterator, ' - current resolution :' + str(resolutionAngstrom) + ' Angstrom') #set highest frequency according to user specification band = maxRes if not multiRefJob.getUseMaxResolution(): band = minRes if band == numberBands: #determineResolution returns numberBands for filter if fsc result is invalid. in that case, use nyquist /2 as filter setting print('Warning MultiRefAlignment.py: LL 114') print( 'Warning: Resolution determined for all classes was invalid. Will use Nyquist/2 for current iteration' ) band = numberBands / 2 preprocessing.setHighestFrequency(band) exMaxJob.setPreprocessing(preprocessing) alignmentLists = [None] * len(particleLists) #generate cluster centers referenceList = distributeExpectation( particleLists, iterationDirectory, 'clusterCenter' + str(iteration), verbose, exMaxJob.getSymmetry()) for classIterator in range(len(particleLists)): classDirectory = iterationDirectory + 'class' + str( classIterator) + '/' #determine distance for all particles refinementDirectory = classDirectory + 'refinement/' if verbose: print(refinementDirectory) if not checkDirExists(refinementDirectory): mkdir(refinementDirectory) exMaxJob.setParticleList(particleList) exMaxJob.setReference(referenceList[classIterator]) exMaxJob.setDestination(refinementDirectory) #run refinement manager = ExMaxManager(exMaxJob) manager.distributeAlignment(verbose) alignmentLists[classIterator] = manager.getAlignmentList() alignmentLists[classIterator].toXMLFile(iterationDirectory + 'AlignmentList' + str(classIterator) + '.xml') #perform classification here if verbose: print('Classifying after iteration ' + str(iteration)) particleList = classifyParticleList(initialParticleList, alignmentLists, verbose) particleList.toXMLFile(iterationDirectory + 'classifiedParticles.xml') particleLists = particleList.splitByClass() difference = previousParticleList.classDifference(particleList) converged = multiRefJob.getEndThreshold() >= difference[3] #set up for next round! previousParticleList = particleList iteration = iteration + 1 if doFinalize: manager.parallelEnd() pytom_mpi.finalise() return [particleList, alignmentLists] else: from pytom.alignment.ExMaxAlignment import ExMaxWorker worker = ExMaxWorker() worker.parallelRun() pytom_mpi.finalise()
def distributeExpectation(particleLists, iterationDirectory, averagePrefix, verbose=False, symmetry=None): """ distributeExpectation: Distributes particle expectation (averaging) to multiple workers. Required by many algorithms such as MCOEXMX @param particleLists: list of particleLists @param iterationDirectory: @param averagePrefix: @param verbose: @param symmetry: """ import pytom_mpi from pytom.tools.files import checkDirExists from pytom.parallel.alignmentMessages import ExpectationJobMsg, ExpectationResultMsg from pytom.alignment.structures import ExpectationJob from pytom.basic.structures import Reference, ReferenceList from os import mkdir if not pytom_mpi.isInitialised(): pytom_mpi.init() mpi_myid = pytom_mpi.rank() if not mpi_myid == 0: raise RuntimeError( 'This function (distributeExpectation) can only be processed by mpi_id = 0! ID == ' + str(mpi_myid) + ' Aborting!') if not checkDirExists(iterationDirectory): raise IOError('The iteration directory does not exist. ' + iterationDirectory) mpi_numberNodes = pytom_mpi.size() if mpi_numberNodes <= 1: raise RuntimeError( 'You must run clustering with openMPI on multiple CPUs') listIterator = 0 referenceList = ReferenceList() #distribute jobs to all nodes for i in range(1, mpi_numberNodes): if verbose: print('Starting first job distribute step') if listIterator < len(particleLists): if not checkDirExists(iterationDirectory + 'class' + str(listIterator) + '/'): mkdir(iterationDirectory + 'class' + str(listIterator) + '/') averageName = iterationDirectory + 'class' + str( listIterator) + '/' + averagePrefix + '-' + str( listIterator) + '.em' if not symmetry.isOneFold(): newPl = symmetry.apply(particleLists[listIterator]) job = ExpectationJob(newPl, averageName) else: job = ExpectationJob(particleLists[listIterator], averageName) newReference = Reference(averageName, particleLists[listIterator]) referenceList.append(newReference) jobMsg = ExpectationJobMsg(0, str(i)) jobMsg.setJob(job) pytom_mpi.send(str(jobMsg), i) if verbose: print(jobMsg) listIterator = listIterator + 1 finished = False #there are more jobs than nodes. continue distributing and collect results receivedMsgCounter = 0 while not finished: #listen and collect mpi_msgString = pytom_mpi.receive() if verbose: print(mpi_msgString) jobResultMsg = ExpectationResultMsg('', '') jobResultMsg.fromStr(mpi_msgString) receivedMsgCounter = receivedMsgCounter + 1 #send new job to free node if listIterator < len(particleLists): if not checkDirExists(iterationDirectory + 'class' + str(listIterator) + '/'): mkdir(iterationDirectory + 'class' + str(listIterator) + '/') averageName = iterationDirectory + 'class' + str( listIterator) + '/' + averagePrefix + '-' + str( listIterator) + '.em' job = ExpectationJob(particleLists[listIterator], averageName) newReference = Reference(averageName, particleLists[listIterator]) referenceList.append(newReference) jobMsg = ExpectationJobMsg(0, str(jobResultMsg.getSender())) jobMsg.setJob(job) pytom_mpi.send(str(jobMsg), i) if verbose: print(jobMsg) listIterator = listIterator + 1 finished = listIterator >= len( particleLists) and receivedMsgCounter == len(particleLists) return referenceList
def parallelReconstruction(particleList, projectionList, cubeSize, binning, applyWeighting,verbose=False): """ parallelReconstruction """ import pytom_mpi from pytom.parallel.messages import StatusMessage if not pytom_mpi.isInitialised(): pytom_mpi.init() mpi_id = pytom_mpi.rank() if mpi_id == 0: firstDistribute = False numberWorkers = pytom_mpi.size() -1 #split particleList by number nodes splitSize = len(particleList) / numberWorkers pl = [] for i in range(0,len(particleList),splitSize): pl.append(particleList[i:i+splitSize]) for i in range(0,numberWorkers): msg = ReconstructionMessage(0,i+1,pl[i],projectionList,cubeSize, binning,applyWeighting) pytom_mpi.send(str(msg),i+1) finished = False msgCounter = 0 while not finished: mpi_msgString = pytom_mpi.receive() msg = StatusMessage(1,'0') msg.fromStr(mpi_msgString) if not msg.getStatus() == 'Finished': print('Worker ' + str(msg.getSender()) + ' sent status: ' + str(msg.getStatus())) msgCounter += 1 finished = msgCounter == numberWorkers for i in range(0,numberWorkers): msg = StatusMessage(mpi_id,'0') msg.setStatus('End') pytom_mpi.send(str(msg),i+1) else: worker = ReconstructionWorker() worker.parallelRun(verbose) pytom_mpi.finalise()
def mcoAC(annealingJob, doFinalize=True, verbose=False): """ mcoAC: Performs mcoAC clustering on particleList @param annealingJob: @param doFinalize: Send finalize messages to workers or not. Default is true. Should be false when this process is integrated into another parallel process. @param verbose: Default is false """ import pytom_mpi if doFinalize: pytom_mpi.init() if pytom_mpi.rank() == 0: from pytom.cluster.mcoEXMX import mcoEXMX from pytom.tools.files import checkDirExists from os import mkdir, system particleList = annealingJob.getParticleList() if len(particleList) == 0: raise RuntimeError('Particle list is empty! Abort!') initialParticleList = particleList previousParticleList = initialParticleList destinationDirectory = annealingJob.getDestinationDirectory() numberIterations = annealingJob.getNumberIterations() numberClasses = annealingJob.getNumberClasses() if verbose: print(annealingJob) if not checkDirExists(destinationDirectory): raise IOError('Destination directory ' + destinationDirectory + ' not found!') try: particleLists = particleList.splitByClass() if len(particleLists) <= 1 or (len(particleLists) == 1 and len( particleLists[0]) == len(particleList)): raise Exception() except Exception: from pytom.cluster.clusterFunctions import randomiseParticleListClasses if numberClasses: if verbose: print('Randomizing particle list') particleList = randomiseParticleListClasses( particleList, numberClasses) particleList.toXMLFile(destinationDirectory + '/RandomisedParticleList.xml') particleLists = particleList.splitByClass() else: raise RuntimeError( 'The particle list provided is not pre-classified and you did not set numberClasses for a random seed!' ) iteration = 0 converged = False bestScoreSum = None bestParticleList = None while (not annealingJob.cooledDown()) and (not converged): if verbose: print('Running iteration ' + str(iteration) + ' of ' + str(numberIterations)) iterationDirectory = destinationDirectory + '/' + str( iteration) + '/' mcoEXMXDirectory = iterationDirectory + 'mcoEXMX/' if not checkDirExists(iterationDirectory): mkdir(iterationDirectory) annealingJob.setDestinationDirectory(mcoEXMXDirectory) annealingJob.setParticleList(previousParticleList) annealingJob.setNumberIterations(annealingJob.getLocalIncrement()) annealingJob.toXMLFile(iterationDirectory + 'annealingJob.xml') #run local refinement [pl, alignmentLists] = mcoEXMX(annealingJob, doFinalize=False, verbose=verbose) annealingJob.setNumberIterations(numberIterations) #store currently best solution if iteration == 0 or (bestScoreSum < pl.sumOfScores() and len(pl.splitByClass()) > 1): bestScoreSum = pl.sumOfScores() bestParticleList = pl bestParticleList.toXMLFile(iterationDirectory + 'currentBestParticleList.xml') #perform classification here [particleList, swapList ] = classifyParticleList(initialParticleList, alignmentLists, annealingJob.getCriterion(), annealingJob.getTemperature().copy(), verbose) #save iteration results to disk particleList.toXMLFile(iterationDirectory + 'classifiedParticles.xml') swapList.toXMLFile(iterationDirectory + 'swapList.xml') #if not verbose mode, delete mcoEXMX files if not verbose: system('rm -rf ' + mcoEXMXDirectory) #print number class swaps difference = previousParticleList.classDifference(particleList) converged = annealingJob.getEndThreshold() >= difference[3] previousParticleList = particleList #set up for new round annealingJob.decreaseTemperature() particleLists = particleList.splitByClass() iteration = iteration + 1 print('Annealing iteration ' + str(iteration) + ' finished!') if doFinalize: from pytom.alignment.ExMaxAlignment import ExMaxManager manager = ExMaxManager(annealingJob.getExMaxJob()) manager.parallelEnd() pytom_mpi.finalise() return bestParticleList else: from pytom.cluster.mcoEXMXStructures import MCOEXMXWorker worker = MCOEXMXWorker() worker.setDoAlignment(annealingJob.getDoAlignment()) worker.setFRMBandwidth(annealingJob.getFRMBandwidth()) worker.parallelRun(verbose=verbose) pytom_mpi.finalise()