def interpretRequestParameters(parameters): """ interpretRequestParameters """ from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge from pytom.alignment.preprocessing import Preprocessing from pytom.frontend.serverpages.serverMessages import FileMessage particleList = ParticleList() if 'plXML' in parameters: particleList.fromXMLFile(parameters['plXML']) elif 'plDIR' in parameters: particleList = ParticleList(parameters['plDIR']) particleList.loadDirectory() else: raise RuntimeError('ParticleList parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(parameters['pixSize']) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(parameters['partDia']) else: raise RuntimeError('Particle diameter missing in request!') if 'wa1' in parameters: wedgeAngle1 = float(parameters['wa1']) else: raise RuntimeError('Wedge angle 1 parameter missing in request!') if 'wa2' in parameters: wedgeAngle2 = float(parameters['wa2']) else: raise RuntimeError('Wedge angle 2 parameter missing in request!') wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2]) if 'mask' in parameters: mask = Mask(parameters['mask']) else: raise RuntimeError('Mask parameter missing in request!') if not 'lowestF' in parameters: raise RuntimeError('Lowest frequency parameter missing in request!') if not 'highestF' in parameters: raise RuntimeError('Highest frequency parameter missing in request!') if not 'filtSm' in parameters: raise RuntimeError('Filter smooth parameter missing in request!') preprocessing = Preprocessing(float(parameters['lowestF']), float(parameters['highestF']), float(parameters['filtSm'])) score = None if 'score' in parameters: if parameters['score'] == 'xcf': from pytom.score.score import xcfScore as scoreClass elif parameters['score'] == 'nxcf': from pytom.score.score import nxcfScore as scoreClass elif parameters['score'] == 'flcf': from pytom.score.score import FLCFScore as scoreClass score = scoreClass() else: raise RuntimeError('Score parameter missing in request!') if 'iter' in parameters: iterations = int(parameters['iter']) else: raise RuntimeError('Number of iterations missing in request!') if 'binning' in parameters: binning = int(parameters['binning']) else: raise RuntimeError('Scaling parameter missing in request!') if 'classes' in parameters: numberClasses = float(parameters['classes']) else: raise RuntimeError('Number classes parameter missing in request!') if 'conv' in parameters: convergence = float(parameters['conv']) else: raise RuntimeError('Convergence parameter missing in request!') if 'dest' in parameters: destination = parameters['dest'] else: raise RuntimeError('Destination parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(float(parameters['pixSize'])) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(float(parameters['partDia'])) else: raise RuntimeError('Particle diameter missing in request!') from pytom.cluster.mcoEXMXStructures import MCOEXMXJob job = MCOEXMXJob(particleList, iterations, destination, mask, score, preprocessing, wedgeInfo, binning, sampleInfo, numberClasses, convergence, symmetry=None) jobXMLFile = '' if 'jobFile' in parameters: jobXMLFile = parameters['jobFile'] job.toXMLFile(jobXMLFile) jobRunFile = jobXMLFile[0:-3] createRunscripts(jobRunFile + 'sh', jobXMLFile) return FileMessage('MCOEXMXJob', jobXMLFile, 'created')
raise RuntimeError('Mask file ' + mask + ' does not exist!') if not checkDirExists(destination): raise RuntimeError('Destination directory ' + destination + ' does not exist!') from pytom.cluster.mcoEXMXStructures import MCOEXMXJob from pytom.basic.structures import ParticleList, Reference, Mask, Wedge, SampleInformation, PointSymmetry from pytom.score.score import FLCFScore from pytom.frontend.serverpages.createMCOEXMXJob import createRunscripts from pytom.alignment.preprocessing import Preprocessing p = ParticleList() p.fromXMLFile(particleList) m = Mask(mask) w = Wedge([float(wedge1), float(wedge2)]) pre = Preprocessing(lowestFrequency=float(lowestFrequency), highestFrequency=float(highestFrequency)) sample = SampleInformation(pixelSize=float(pixelSize), particleDiameter=float(diameter)) if symmetryN is None or symmetryAxisZ is None or symmetryAxisX is None: sym = None else: sym = PointSymmetry(nfold=int(symmetryN), z2=float(symmetryAxisZ), x=float(symmetryAxisX)) job = MCOEXMXJob(particleList=p,numberIterations=numberIterations,\ destinationDirectory=destination,mask=mask,score=FLCFScore(),preprocessing=pre,\ wedgeInfo=w,binning=int(binning),sampleInformation=sample,numberClasses=int(numberClasses),\
def create_RandomParticleList(reffile, pl_filename='pl.xml', pdir='./testparticles', nparticles=10): """ @param reffile: reference file @type reffile: C{str} @param nparticles: number of particles (default: 10) @type nparticles: C{int} @param pl_filename: particle list filename @type pl_filename: C{str} @param pdir: particle directory @type pdir: C{str} @return: particleList @rtype: L{pytom.basic.ParticleList} """ from pytom.basic.structures import Particle, ParticleList, Rotation, Shift, Wedge from pytom_volume import vol, rotate, shift, read from pytom.basic.transformations import general_transform_crop from pytom.basic.functions import initSphere from pytom.simulation.whiteNoise import add as addNoise import random from os import mkdir from pytom.score.score import FLCFScore as score try: mkdir(pdir) except FileExistsError: print('directory ' + pdir + ' existed already - using this one') random.seed(0) a = 0 wedge = Wedge(wedgeAngles=[30.0, 30.0], cutoffRadius=50.0) pl = ParticleList(directory='./') ref1 = read(reffile) ref2 = initSphere(sizeX=ref1.sizeX(), sizeY=ref1.sizeY(), sizeZ=ref1.sizeZ(), radius=45) #ref2.write('testData/mask_45.em') parts = {0: ref1, 1: ref2} for ii in range(0, nparticles): if not ii % 2: ref = read(reffile) else: ref = initSphere(sizeX=ref1.sizeX(), sizeY=ref1.sizeY(), sizeZ=ref1.sizeZ(), radius=30, smooth=30 / 10) rot = Rotation(random.uniform(0, 360), random.uniform(0, 360), random.uniform(0, 180)) shift = Shift(x=a * random.uniform(-5, 5), y=a * random.uniform(-5, 5), z=a * random.uniform(-5, 5)) rotvol = general_transform_crop(v=ref, rot=rot, shift=shift, scale=None, order=[0, 1, 2]) # add some noise noisy = addNoise(volume=rotvol, SNR=1) fname = pdir + '/particle_' + str(ii) + '.em' #noisy.write( fname) p = Particle(filename=fname, rotation=rot, shift=shift, wedge=wedge, className=0, pickPosition=None, score=score, sourceInfo=None) p.setScoreValue(0.0) wg = p.getWedge().getWedgeObject() wg.apply(noisy, Rotation(0, 0, 0)).write(fname) pl.append(particle=p) pl.setFileName(filename=pl_filename) pl.toXMLFile(filename=pl_filename) return pl
def interpretRequestParameters(parameters): """ interpretRequestParameters """ from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge from pytom.alignment.preprocessing import Preprocessing from pytom.frontend.serverpages.serverMessages import FileMessage particleList = ParticleList('.') if 'plXML' in parameters: particleList.fromXMLFile(parameters['plXML']) elif 'plDIR' in parameters: particleList = ParticleList(parameters['plDIR']) particleList.loadDirectory() else: raise RuntimeError('ParticleList parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(parameters['pixSize']) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(parameters['partDia']) else: raise RuntimeError('Particle diameter missing in request!') if 'wa1' in parameters: wedgeAngle1 = float(parameters['wa1']) else: raise RuntimeError('Wedge angle 1 parameter missing in request!') if 'wa2' in parameters: wedgeAngle2 = float(parameters['wa2']) else: raise RuntimeError('Wedge angle 2 parameter missing in request!') wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2]) if 'mask' in parameters: mask = Mask(parameters['mask']) else: raise RuntimeError('Mask parameter missing in request!') if not 'lowestF' in parameters: raise RuntimeError('Lowest frequency parameter missing in request!') if not 'highestF' in parameters: raise RuntimeError('Highest frequency parameter missing in request!') if not 'filtSm' in parameters: raise RuntimeError('Filter smooth parameter missing in request!') preprocessing = Preprocessing(float(parameters['lowestF']), float(parameters['highestF']), float(parameters['filtSm'])) score = None if 'score' in parameters: if parameters['score'] == 'xcf': from pytom.score.score import xcfScore as scoreClass elif parameters['score'] == 'nxcf': from pytom.score.score import nxcfScore as scoreClass elif parameters['score'] == 'flcf': from pytom.score.score import FLCFScore as scoreClass score = scoreClass() else: raise RuntimeError('Score parameter missing in request!') if 'binning' in parameters: binning = int(parameters['binning']) else: raise RuntimeError('Scaling parameter missing in request!') if 'classes' in parameters: numberClasses = float(parameters['classes']) else: raise RuntimeError('Number classes parameter missing in request!') if 'conv' in parameters: convergence = float(parameters['conv']) else: raise RuntimeError('Convergence parameter missing in request!') if 'dest' in parameters: destination = parameters['dest'] else: raise RuntimeError('Destination parameter missing in request!') sampleInfo = SampleInformation() if 'pixSize' in parameters: sampleInfo.setPixelSize(float(parameters['pixSize'])) else: raise RuntimeError('Pixelsize parameter missing in request!') if 'partDia' in parameters: sampleInfo.setParticleDiameter(float(parameters['partDia'])) else: raise RuntimeError('Particle diameter missing in request!') temperature = None if 'temp' in parameters: temperature = parameters['temp'] if 'stemp' in parameters: startTemperature = float(parameters['stemp']) else: raise RuntimeError( 'Start temperature parameter missing in request!') if 'astep' in parameters: annealingStep = float(parameters['astep']) else: raise RuntimeError('Annealing step parameter missing in request!') from pytom.cluster.mcoACStructures import SigmaTemperature if temperature == 'sigma': temperature = SigmaTemperature(startTemperature, annealingStep) else: raise RuntimeError('Temperature missing in request!') criterion = None if 'crit' in parameters: from pytom.cluster.mcoACStructures import MetropolisCriterion, ThresholdAcceptance criterion = parameters['crit'] if criterion == 'metropolis': criterion = MetropolisCriterion() elif criterion == 'threshold': criterion = ThresholdAcceptance() else: raise RuntimeError('Criterion missing in request!') if 'refin' in parameters: localSearchIncrement = float(parameters['refin']) else: raise RuntimeError('Number of refinement rounds missing in request!') from pytom.cluster.mcoACStructures import MCOACJob job = MCOACJob(particleList, destination, mask, score, preprocessing, wedgeInfo, binning, sampleInfo, numberClasses, temperature, criterion, convergence, localSearchIncrement, symmetry=None) jobXMLFile = '' if 'jobFile' in parameters: jobXMLFile = parameters['jobFile'] job.toXMLFile(jobXMLFile) jobRunFile = jobXMLFile[0:-3] createRunscripts(jobRunFile + 'sh', jobXMLFile) return FileMessage('MCOACJob', jobXMLFile, 'created')
class PeakJob(PyTomClass): """ PeakJob: stores all the infos needed for calculation of the peak score """ def __init__(self, volume='', reference='', mask='', wedge='', rotations='', score='', jobID=0, members=1, dstDir='./', bandpass=None): """ @param volume: target volume @type volume: L{pytom.localization.structures.Volume} @param reference: reference volume @type reference: L{pytom.basic.structures.Reference} @param mask: mask volume @type mask: L{pytom.basic.structures.Mask} @param wedge: wedge information @type wedge: L{pytom.basic.structures.WedgeInfo} @param rotations: rotation list @type rotations: L{pytom.angles.angle} @param score: score function @type score: L{pytom.score.score} @param jobID: job identification @type jobID: integer @param members: how many members are there available to accomplish this job (1 means only itself) @type members: integer @param dstDir: destination directory where the result is written to @type dstDir: string @param bandpass: bandpass object that will be applied to the reference @type bandpass: L{pytom.basic.structure.BandPassFilter} """ self.volume = volume self.reference = reference self.mask = mask self.wedge = wedge self.rotations = rotations self.score = score self.jobID = jobID self.members = members if dstDir[-1] == '/': self.dstDir = dstDir else: self.dstDir = dstDir + '/' self.bandpass = bandpass def copy(self, fromJob): self.volume = fromJob.volume self.reference = fromJob.reference self.mask = fromJob.mask self.wedge = fromJob.wedge self.rotations = fromJob.rotations self.score = fromJob.score self.jobID = fromJob.jobID self.members = fromJob.members self.dstDir = fromJob.dstDir self.bandpass = fromJob.bandpass def fromXML(self, xmlObj): """ fromXML : Assigns values to job attributes from XML object @param xmlObj: A xml object @author: chen """ from lxml.etree import _Element if xmlObj.__class__ != _Element: raise Exception('You must provide a valid XML object.') if xmlObj.tag == "JobDescription": jobDescription = xmlObj else: jobDescription = xmlObj.xpath('JobDescription') if len(jobDescription) == 0: raise Exception("This XML is not an JobDescription.") jobDescription = jobDescription[0] id = jobDescription.get('ID') if id != None and id != 'None': self.jobID = int(id) members = jobDescription.get('Members') if members != None and members != 'None': self.members = int(members) dstDir = jobDescription.get('Destination') if dstDir != None: if dstDir[-1] == '/': self.dstDir = dstDir else: self.dstDir = dstDir + '/' from pytom.score.score import fromXML as fromXMLScore from pytom.basic.structures import Mask, Reference, Wedge # from pytom.angles.angleList import AngleList from pytom.localization.structures import Volume e = jobDescription.xpath('Volume')[0] v = Volume() v.fromXML(e) self.volume = v ref = jobDescription.xpath('Reference')[0] self.reference = Reference('') self.reference.fromXML(ref) wedgeXML = jobDescription.xpath('Wedge') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('SingleTiltWedge') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('WedgeInfo') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('DoubleTiltWedge') assert len(wedgeXML) > 0 self.wedge = Wedge() self.wedge.fromXML(wedgeXML[0]) mask = jobDescription.xpath('Mask')[0] self.mask = Mask('') self.mask.fromXML(mask) score = jobDescription.xpath('Score') self.score = fromXMLScore(score[0]) rot = jobDescription.xpath('Angles')[0] from pytom.angles.angle import AngleObject ang = AngleObject() self.rotations = ang.fromXML(rot) # self.rotations = AngleList() # self.rotations.fromXML(rot) bp = jobDescription.xpath('BandPassFilter') if bp != []: bp = bp[0] from pytom.basic.structures import BandPassFilter self.bandpass = BandPassFilter(0, 0, 0) self.bandpass.fromXML(bp) else: self.bandpass = None def toXML(self): """ toXML : Compiles a XML file from job object @author: chen """ from lxml import etree jobElement = etree.Element("JobDescription", ID=str(self.jobID), Members=str(self.members), Destination=str(self.dstDir)) jobElement.append(self.volume.toXML()) jobElement.append(self.reference.toXML()) jobElement.append(self.mask.toXML()) jobElement.append(self.wedge.toXML()) jobElement.append(self.rotations.toXML()) jobElement.append(self.score.toXML()) if self.bandpass: jobElement.append(self.bandpass.toXML()) return jobElement def check(self): """ check: Performs check whether all settings are valid. Paths and Files exist @author: chen """ from pytom.tools.files import checkFileExists, checkDirExists returnValue = checkFileExists(self.volume.getFilename()) if not returnValue: raise IOError('File: ' + str(self.volume) + ' not found!') returnValue = checkFileExists(self.reference.getReferenceFilename()) if not returnValue: raise IOError('File: ' + str(self.reference) + ' not found!') returnValue = checkFileExists(self.mask.getFilename()) if not returnValue: raise IOError('File: ' + str(self.mask) + ' not found!') returnValue = checkDirExists(self.dstDir[:-1]) if not returnValue: raise IOError('Directory: ' + str(self.dstDir) + ' not found!') return returnValue def send(self, source, destination): """ send: Send the job-relevant message from source to destination @param source: source machine id gained from pytom_mpi @type source: int @param destination: destination machine id @type destination: int @author: chen """ from pytom.localization.peak_job_msg import PeakJobMsg # self.check() msg = PeakJobMsg(str(source), str(destination)) msg.setJob(self) import pytom_mpi print(f'destination: {destination}\ntype: {type(destination)}') pytom_mpi.send(str(msg), int(destination))
def fromXML(self, xmlObj): """ fromXML : Assigns values to job attributes from XML object @param xmlObj: A xml object @author: chen """ from lxml.etree import _Element if xmlObj.__class__ != _Element: raise Exception('You must provide a valid XML object.') if xmlObj.tag == "JobDescription": jobDescription = xmlObj else: jobDescription = xmlObj.xpath('JobDescription') if len(jobDescription) == 0: raise Exception("This XML is not an JobDescription.") jobDescription = jobDescription[0] id = jobDescription.get('ID') if id != None and id != 'None': self.jobID = int(id) members = jobDescription.get('Members') if members != None and members != 'None': self.members = int(members) dstDir = jobDescription.get('Destination') if dstDir != None: if dstDir[-1] == '/': self.dstDir = dstDir else: self.dstDir = dstDir + '/' from pytom.score.score import fromXML as fromXMLScore from pytom.basic.structures import Mask, Reference, Wedge # from pytom.angles.angleList import AngleList from pytom.localization.structures import Volume e = jobDescription.xpath('Volume')[0] v = Volume() v.fromXML(e) self.volume = v ref = jobDescription.xpath('Reference')[0] self.reference = Reference('') self.reference.fromXML(ref) wedgeXML = jobDescription.xpath('Wedge') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('SingleTiltWedge') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('WedgeInfo') if len(wedgeXML) == 0: wedgeXML = jobDescription.xpath('DoubleTiltWedge') assert len(wedgeXML) > 0 self.wedge = Wedge() self.wedge.fromXML(wedgeXML[0]) mask = jobDescription.xpath('Mask')[0] self.mask = Mask('') self.mask.fromXML(mask) score = jobDescription.xpath('Score') self.score = fromXMLScore(score[0]) rot = jobDescription.xpath('Angles')[0] from pytom.angles.angle import AngleObject ang = AngleObject() self.rotations = ang.fromXML(rot) # self.rotations = AngleList() # self.rotations.fromXML(rot) bp = jobDescription.xpath('BandPassFilter') if bp != []: bp = bp[0] from pytom.basic.structures import BandPassFilter self.bandpass = BandPassFilter(0, 0, 0) self.bandpass.fromXML(bp) else: self.bandpass = None
def _expectation(self): """ _expectation : Compares two particles. Does not do any alignemnt unless job enables FRMAlignment, this function overloads ExMaxWorker._expectation. @rtype: List of two objects @return: [endShift,endRotation] contains the final shift and final rotation. @author: Thomas Hrabe """ from pytom_volume import vol, shift, rotate, read from pytom.alignment.structures import Peak, MaximisationResult from pytom.basic.structures import Particle, Reference, Shift, Rotation, Wedge from pytom.tools.macros import volumesSameSize from pytom.alignment.alignmentFunctions import compareTwoVolumes from pytom.angles.angleList import OneAngleList from pytom.alignment.alignmentFunctions import FRMAlignmentWrapper try: from frm import frm_align_vol2 except ImportError: self._doAlignment = False if self._particle.__class__ == Particle: from pytom_volume import read particleFile = self._particle.getFilename() particle = read(particleFile, 0, 0, 0, 0, 0, 0, 0, 0, 0, self._binning, self._binning, self._binning) else: raise TypeError( 'Provided particle must be of type pytom.basic.structures.Particle' ) if self._scoreObject.__class__ == str: from pytom.score.score import Score string = self._scoreObject self._scoreObject = Score() self._scoreObject.fromStr(string) if self._reference.wasGeneratedBy( self._particle) and self._reference.hasPreWedge( ) and self._reference.hasWeighting(): #substract particle from reference if information is available [reference, self._referenceWeighting ] = self._reference.subtractParticle(self._particle, self._binning) referenceObject = self._reference else: if self._reference.__class__ == Reference: from pytom_volume import read referenceObject = self._reference referenceFile = self._reference.getReferenceFilename() reference = read(referenceFile, 0, 0, 0, 0, 0, 0, 0, 0, 0, self._binning, self._binning, self._binning) if self._referenceWeighting == str and len( self._referenceWeighting) > 0: self._referenceWeightingFile = self._referenceWeighting if binning == 1: from pytom_volume import read self._referenceWeighting = read(self._referenceWeighting) else: from pytom.basic.files import readSubvolumeFromFourierspaceFile self._referenceWeighting = readSubvolumeFromFourierspaceFile( self._referenceWeightingFile, reference.sizeX(), reference.sizeY(), reference.sizeZ()) self._mask.setBinning(self._binning) pScore = self._particle.getScore() if not pScore: pScore = self._scoreObject if self._doAlignment: #process FRM alignment for each particle and class peakPrior = self._scoreObject.getPeakPrior() newShift, newRotation, score = FRMAlignmentWrapper( particle, self._particle.getWedge(), reference, Wedge([0, 0]), self._frmBandwidth, int(self._preprocessing.getHighestFrequency()), self._mask, self._scoreObject.getPeakPrior()) self._scoreObject.setValue(score) result = MaximisationResult( self._particle, referenceObject, self._scoreObject, newShift, newRotation, OneAngleList(self._particle.getRotation())) else: #process classification without alignment only peakV = compareTwoVolumes( particle, reference, self._referenceWeighting, self._particle.getWedge(), OneAngleList(self._particle.getRotation()), self._particle.getShift(), self._scoreObject, self._mask, self._preprocessing, self._binning) self._scoreObject.setValue(peakV) result = MaximisationResult( self._particle, referenceObject, self._scoreObject, self._particle.getShift(), self._particle.getRotation(), OneAngleList(self._particle.getRotation())) return result