def interpretRequestParameters(parameters):
"""
interpretRequestParameters
"""
from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge
from pytom.alignment.preprocessing import Preprocessing
from pytom.frontend.serverpages.serverMessages import FileMessage
particleList = ParticleList()
if 'plXML' in parameters:
particleList.fromXMLFile(parameters['plXML'])
elif 'plDIR' in parameters:
particleList = ParticleList(parameters['plDIR'])
particleList.loadDirectory()
else:
raise RuntimeError('ParticleList parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(parameters['pixSize'])
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(parameters['partDia'])
else:
raise RuntimeError('Particle diameter missing in request!')
if 'wa1' in parameters:
wedgeAngle1 = float(parameters['wa1'])
else:
raise RuntimeError('Wedge angle 1 parameter missing in request!')
if 'wa2' in parameters:
wedgeAngle2 = float(parameters['wa2'])
else:
raise RuntimeError('Wedge angle 2 parameter missing in request!')
wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2])
if 'mask' in parameters:
mask = Mask(parameters['mask'])
else:
raise RuntimeError('Mask parameter missing in request!')
if not 'lowestF' in parameters:
raise RuntimeError('Lowest frequency parameter missing in request!')
if not 'highestF' in parameters:
raise RuntimeError('Highest frequency parameter missing in request!')
if not 'filtSm' in parameters:
raise RuntimeError('Filter smooth parameter missing in request!')
preprocessing = Preprocessing(float(parameters['lowestF']),
float(parameters['highestF']),
float(parameters['filtSm']))
score = None
if 'score' in parameters:
if parameters['score'] == 'xcf':
from pytom.score.score import xcfScore as scoreClass
elif parameters['score'] == 'nxcf':
from pytom.score.score import nxcfScore as scoreClass
elif parameters['score'] == 'flcf':
from pytom.score.score import FLCFScore as scoreClass
score = scoreClass()
else:
raise RuntimeError('Score parameter missing in request!')
if 'iter' in parameters:
iterations = int(parameters['iter'])
else:
raise RuntimeError('Number of iterations missing in request!')
if 'binning' in parameters:
binning = int(parameters['binning'])
else:
raise RuntimeError('Scaling parameter missing in request!')
if 'classes' in parameters:
numberClasses = float(parameters['classes'])
else:
raise RuntimeError('Number classes parameter missing in request!')
if 'conv' in parameters:
convergence = float(parameters['conv'])
else:
raise RuntimeError('Convergence parameter missing in request!')
if 'dest' in parameters:
destination = parameters['dest']
else:
raise RuntimeError('Destination parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(float(parameters['pixSize']))
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(float(parameters['partDia']))
else:
raise RuntimeError('Particle diameter missing in request!')
from pytom.cluster.mcoEXMXStructures import MCOEXMXJob
job = MCOEXMXJob(particleList,
iterations,
destination,
mask,
score,
preprocessing,
wedgeInfo,
binning,
sampleInfo,
numberClasses,
convergence,
symmetry=None)
jobXMLFile = ''
if 'jobFile' in parameters:
jobXMLFile = parameters['jobFile']
job.toXMLFile(jobXMLFile)
jobRunFile = jobXMLFile[0:-3]
createRunscripts(jobRunFile + 'sh', jobXMLFile)
return FileMessage('MCOEXMXJob', jobXMLFile, 'created')
raise RuntimeError('Mask file ' + mask + ' does not exist!')
if not checkDirExists(destination):
raise RuntimeError('Destination directory ' + destination +
' does not exist!')
from pytom.cluster.mcoEXMXStructures import MCOEXMXJob
from pytom.basic.structures import ParticleList, Reference, Mask, Wedge, SampleInformation, PointSymmetry
from pytom.score.score import FLCFScore
from pytom.frontend.serverpages.createMCOEXMXJob import createRunscripts
from pytom.alignment.preprocessing import Preprocessing
p = ParticleList()
p.fromXMLFile(particleList)
m = Mask(mask)
w = Wedge([float(wedge1), float(wedge2)])
pre = Preprocessing(lowestFrequency=float(lowestFrequency),
highestFrequency=float(highestFrequency))
sample = SampleInformation(pixelSize=float(pixelSize),
particleDiameter=float(diameter))
if symmetryN is None or symmetryAxisZ is None or symmetryAxisX is None:
sym = None
else:
sym = PointSymmetry(nfold=int(symmetryN),
z2=float(symmetryAxisZ),
x=float(symmetryAxisX))
job = MCOEXMXJob(particleList=p,numberIterations=numberIterations,\
destinationDirectory=destination,mask=mask,score=FLCFScore(),preprocessing=pre,\
wedgeInfo=w,binning=int(binning),sampleInformation=sample,numberClasses=int(numberClasses),\
def create_RandomParticleList(reffile,
pl_filename='pl.xml',
pdir='./testparticles',
nparticles=10):
"""
@param reffile: reference file
@type reffile: C{str}
@param nparticles: number of particles (default: 10)
@type nparticles: C{int}
@param pl_filename: particle list filename
@type pl_filename: C{str}
@param pdir: particle directory
@type pdir: C{str}
@return: particleList
@rtype: L{pytom.basic.ParticleList}
"""
from pytom.basic.structures import Particle, ParticleList, Rotation, Shift, Wedge
from pytom_volume import vol, rotate, shift, read
from pytom.basic.transformations import general_transform_crop
from pytom.basic.functions import initSphere
from pytom.simulation.whiteNoise import add as addNoise
import random
from os import mkdir
from pytom.score.score import FLCFScore as score
try:
mkdir(pdir)
except FileExistsError:
print('directory ' + pdir + ' existed already - using this one')
random.seed(0)
a = 0
wedge = Wedge(wedgeAngles=[30.0, 30.0], cutoffRadius=50.0)
pl = ParticleList(directory='./')
ref1 = read(reffile)
ref2 = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=45)
#ref2.write('testData/mask_45.em')
parts = {0: ref1, 1: ref2}
for ii in range(0, nparticles):
if not ii % 2:
ref = read(reffile)
else:
ref = initSphere(sizeX=ref1.sizeX(),
sizeY=ref1.sizeY(),
sizeZ=ref1.sizeZ(),
radius=30,
smooth=30 / 10)
rot = Rotation(random.uniform(0, 360), random.uniform(0, 360),
random.uniform(0, 180))
shift = Shift(x=a * random.uniform(-5, 5),
y=a * random.uniform(-5, 5),
z=a * random.uniform(-5, 5))
rotvol = general_transform_crop(v=ref,
rot=rot,
shift=shift,
scale=None,
order=[0, 1, 2])
# add some noise
noisy = addNoise(volume=rotvol, SNR=1)
fname = pdir + '/particle_' + str(ii) + '.em'
#noisy.write( fname)
p = Particle(filename=fname,
rotation=rot,
shift=shift,
wedge=wedge,
className=0,
pickPosition=None,
score=score,
sourceInfo=None)
p.setScoreValue(0.0)
wg = p.getWedge().getWedgeObject()
wg.apply(noisy, Rotation(0, 0, 0)).write(fname)
pl.append(particle=p)
pl.setFileName(filename=pl_filename)
pl.toXMLFile(filename=pl_filename)
return pl
def interpretRequestParameters(parameters):
"""
interpretRequestParameters
"""
from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge
from pytom.alignment.preprocessing import Preprocessing
from pytom.frontend.serverpages.serverMessages import FileMessage
particleList = ParticleList('.')
if 'plXML' in parameters:
particleList.fromXMLFile(parameters['plXML'])
elif 'plDIR' in parameters:
particleList = ParticleList(parameters['plDIR'])
particleList.loadDirectory()
else:
raise RuntimeError('ParticleList parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(parameters['pixSize'])
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(parameters['partDia'])
else:
raise RuntimeError('Particle diameter missing in request!')
if 'wa1' in parameters:
wedgeAngle1 = float(parameters['wa1'])
else:
raise RuntimeError('Wedge angle 1 parameter missing in request!')
if 'wa2' in parameters:
wedgeAngle2 = float(parameters['wa2'])
else:
raise RuntimeError('Wedge angle 2 parameter missing in request!')
wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2])
if 'mask' in parameters:
mask = Mask(parameters['mask'])
else:
raise RuntimeError('Mask parameter missing in request!')
if not 'lowestF' in parameters:
raise RuntimeError('Lowest frequency parameter missing in request!')
if not 'highestF' in parameters:
raise RuntimeError('Highest frequency parameter missing in request!')
if not 'filtSm' in parameters:
raise RuntimeError('Filter smooth parameter missing in request!')
preprocessing = Preprocessing(float(parameters['lowestF']),
float(parameters['highestF']),
float(parameters['filtSm']))
score = None
if 'score' in parameters:
if parameters['score'] == 'xcf':
from pytom.score.score import xcfScore as scoreClass
elif parameters['score'] == 'nxcf':
from pytom.score.score import nxcfScore as scoreClass
elif parameters['score'] == 'flcf':
from pytom.score.score import FLCFScore as scoreClass
score = scoreClass()
else:
raise RuntimeError('Score parameter missing in request!')
if 'binning' in parameters:
binning = int(parameters['binning'])
else:
raise RuntimeError('Scaling parameter missing in request!')
if 'classes' in parameters:
numberClasses = float(parameters['classes'])
else:
raise RuntimeError('Number classes parameter missing in request!')
if 'conv' in parameters:
convergence = float(parameters['conv'])
else:
raise RuntimeError('Convergence parameter missing in request!')
if 'dest' in parameters:
destination = parameters['dest']
else:
raise RuntimeError('Destination parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(float(parameters['pixSize']))
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(float(parameters['partDia']))
else:
raise RuntimeError('Particle diameter missing in request!')
temperature = None
if 'temp' in parameters:
temperature = parameters['temp']
if 'stemp' in parameters:
startTemperature = float(parameters['stemp'])
else:
raise RuntimeError(
'Start temperature parameter missing in request!')
if 'astep' in parameters:
annealingStep = float(parameters['astep'])
else:
raise RuntimeError('Annealing step parameter missing in request!')
from pytom.cluster.mcoACStructures import SigmaTemperature
if temperature == 'sigma':
temperature = SigmaTemperature(startTemperature, annealingStep)
else:
raise RuntimeError('Temperature missing in request!')
criterion = None
if 'crit' in parameters:
from pytom.cluster.mcoACStructures import MetropolisCriterion, ThresholdAcceptance
criterion = parameters['crit']
if criterion == 'metropolis':
criterion = MetropolisCriterion()
elif criterion == 'threshold':
criterion = ThresholdAcceptance()
else:
raise RuntimeError('Criterion missing in request!')
if 'refin' in parameters:
localSearchIncrement = float(parameters['refin'])
else:
raise RuntimeError('Number of refinement rounds missing in request!')
from pytom.cluster.mcoACStructures import MCOACJob
job = MCOACJob(particleList,
destination,
mask,
score,
preprocessing,
wedgeInfo,
binning,
sampleInfo,
numberClasses,
temperature,
criterion,
convergence,
localSearchIncrement,
symmetry=None)
jobXMLFile = ''
if 'jobFile' in parameters:
jobXMLFile = parameters['jobFile']
job.toXMLFile(jobXMLFile)
jobRunFile = jobXMLFile[0:-3]
createRunscripts(jobRunFile + 'sh', jobXMLFile)
return FileMessage('MCOACJob', jobXMLFile, 'created')
class PeakJob(PyTomClass):
"""
PeakJob: stores all the infos needed for calculation of the peak score
"""
def __init__(self,
volume='',
reference='',
mask='',
wedge='',
rotations='',
score='',
jobID=0,
members=1,
dstDir='./',
bandpass=None):
"""
@param volume: target volume
@type volume: L{pytom.localization.structures.Volume}
@param reference: reference volume
@type reference: L{pytom.basic.structures.Reference}
@param mask: mask volume
@type mask: L{pytom.basic.structures.Mask}
@param wedge: wedge information
@type wedge: L{pytom.basic.structures.WedgeInfo}
@param rotations: rotation list
@type rotations: L{pytom.angles.angle}
@param score: score function
@type score: L{pytom.score.score}
@param jobID: job identification
@type jobID: integer
@param members: how many members are there available to accomplish this job (1 means only itself)
@type members: integer
@param dstDir: destination directory where the result is written to
@type dstDir: string
@param bandpass: bandpass object that will be applied to the reference
@type bandpass: L{pytom.basic.structure.BandPassFilter}
"""
self.volume = volume
self.reference = reference
self.mask = mask
self.wedge = wedge
self.rotations = rotations
self.score = score
self.jobID = jobID
self.members = members
if dstDir[-1] == '/':
self.dstDir = dstDir
else:
self.dstDir = dstDir + '/'
self.bandpass = bandpass
def copy(self, fromJob):
self.volume = fromJob.volume
self.reference = fromJob.reference
self.mask = fromJob.mask
self.wedge = fromJob.wedge
self.rotations = fromJob.rotations
self.score = fromJob.score
self.jobID = fromJob.jobID
self.members = fromJob.members
self.dstDir = fromJob.dstDir
self.bandpass = fromJob.bandpass
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@author: chen
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise Exception('You must provide a valid XML object.')
if xmlObj.tag == "JobDescription":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('JobDescription')
if len(jobDescription) == 0:
raise Exception("This XML is not an JobDescription.")
jobDescription = jobDescription[0]
id = jobDescription.get('ID')
if id != None and id != 'None':
self.jobID = int(id)
members = jobDescription.get('Members')
if members != None and members != 'None':
self.members = int(members)
dstDir = jobDescription.get('Destination')
if dstDir != None:
if dstDir[-1] == '/':
self.dstDir = dstDir
else:
self.dstDir = dstDir + '/'
from pytom.score.score import fromXML as fromXMLScore
from pytom.basic.structures import Mask, Reference, Wedge
# from pytom.angles.angleList import AngleList
from pytom.localization.structures import Volume
e = jobDescription.xpath('Volume')[0]
v = Volume()
v.fromXML(e)
self.volume = v
ref = jobDescription.xpath('Reference')[0]
self.reference = Reference('')
self.reference.fromXML(ref)
wedgeXML = jobDescription.xpath('Wedge')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('SingleTiltWedge')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('WedgeInfo')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('DoubleTiltWedge')
assert len(wedgeXML) > 0
self.wedge = Wedge()
self.wedge.fromXML(wedgeXML[0])
mask = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(mask)
score = jobDescription.xpath('Score')
self.score = fromXMLScore(score[0])
rot = jobDescription.xpath('Angles')[0]
from pytom.angles.angle import AngleObject
ang = AngleObject()
self.rotations = ang.fromXML(rot)
# self.rotations = AngleList()
# self.rotations.fromXML(rot)
bp = jobDescription.xpath('BandPassFilter')
if bp != []:
bp = bp[0]
from pytom.basic.structures import BandPassFilter
self.bandpass = BandPassFilter(0, 0, 0)
self.bandpass.fromXML(bp)
else:
self.bandpass = None
def toXML(self):
"""
toXML : Compiles a XML file from job object
@author: chen
"""
from lxml import etree
jobElement = etree.Element("JobDescription",
ID=str(self.jobID),
Members=str(self.members),
Destination=str(self.dstDir))
jobElement.append(self.volume.toXML())
jobElement.append(self.reference.toXML())
jobElement.append(self.mask.toXML())
jobElement.append(self.wedge.toXML())
jobElement.append(self.rotations.toXML())
jobElement.append(self.score.toXML())
if self.bandpass:
jobElement.append(self.bandpass.toXML())
return jobElement
def check(self):
"""
check: Performs check whether all settings are valid. Paths and Files exist
@author: chen
"""
from pytom.tools.files import checkFileExists, checkDirExists
returnValue = checkFileExists(self.volume.getFilename())
if not returnValue:
raise IOError('File: ' + str(self.volume) + ' not found!')
returnValue = checkFileExists(self.reference.getReferenceFilename())
if not returnValue:
raise IOError('File: ' + str(self.reference) + ' not found!')
returnValue = checkFileExists(self.mask.getFilename())
if not returnValue:
raise IOError('File: ' + str(self.mask) + ' not found!')
returnValue = checkDirExists(self.dstDir[:-1])
if not returnValue:
raise IOError('Directory: ' + str(self.dstDir) + ' not found!')
return returnValue
def send(self, source, destination):
"""
send: Send the job-relevant message from source to destination
@param source: source machine id gained from pytom_mpi
@type source: int
@param destination: destination machine id
@type destination: int
@author: chen
"""
from pytom.localization.peak_job_msg import PeakJobMsg
# self.check()
msg = PeakJobMsg(str(source), str(destination))
msg.setJob(self)
import pytom_mpi
print(f'destination: {destination}\ntype: {type(destination)}')
pytom_mpi.send(str(msg), int(destination))
def fromXML(self, xmlObj):
"""
fromXML : Assigns values to job attributes from XML object
@param xmlObj: A xml object
@author: chen
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise Exception('You must provide a valid XML object.')
if xmlObj.tag == "JobDescription":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('JobDescription')
if len(jobDescription) == 0:
raise Exception("This XML is not an JobDescription.")
jobDescription = jobDescription[0]
id = jobDescription.get('ID')
if id != None and id != 'None':
self.jobID = int(id)
members = jobDescription.get('Members')
if members != None and members != 'None':
self.members = int(members)
dstDir = jobDescription.get('Destination')
if dstDir != None:
if dstDir[-1] == '/':
self.dstDir = dstDir
else:
self.dstDir = dstDir + '/'
from pytom.score.score import fromXML as fromXMLScore
from pytom.basic.structures import Mask, Reference, Wedge
# from pytom.angles.angleList import AngleList
from pytom.localization.structures import Volume
e = jobDescription.xpath('Volume')[0]
v = Volume()
v.fromXML(e)
self.volume = v
ref = jobDescription.xpath('Reference')[0]
self.reference = Reference('')
self.reference.fromXML(ref)
wedgeXML = jobDescription.xpath('Wedge')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('SingleTiltWedge')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('WedgeInfo')
if len(wedgeXML) == 0:
wedgeXML = jobDescription.xpath('DoubleTiltWedge')
assert len(wedgeXML) > 0
self.wedge = Wedge()
self.wedge.fromXML(wedgeXML[0])
mask = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(mask)
score = jobDescription.xpath('Score')
self.score = fromXMLScore(score[0])
rot = jobDescription.xpath('Angles')[0]
from pytom.angles.angle import AngleObject
ang = AngleObject()
self.rotations = ang.fromXML(rot)
# self.rotations = AngleList()
# self.rotations.fromXML(rot)
bp = jobDescription.xpath('BandPassFilter')
if bp != []:
bp = bp[0]
from pytom.basic.structures import BandPassFilter
self.bandpass = BandPassFilter(0, 0, 0)
self.bandpass.fromXML(bp)
else:
self.bandpass = None
def _expectation(self):
"""
_expectation : Compares two particles. Does not do any alignemnt unless job enables FRMAlignment, this function overloads ExMaxWorker._expectation.
@rtype: List of two objects
@return: [endShift,endRotation] contains the final shift and final rotation.
@author: Thomas Hrabe
"""
from pytom_volume import vol, shift, rotate, read
from pytom.alignment.structures import Peak, MaximisationResult
from pytom.basic.structures import Particle, Reference, Shift, Rotation, Wedge
from pytom.tools.macros import volumesSameSize
from pytom.alignment.alignmentFunctions import compareTwoVolumes
from pytom.angles.angleList import OneAngleList
from pytom.alignment.alignmentFunctions import FRMAlignmentWrapper
try:
from frm import frm_align_vol2
except ImportError:
self._doAlignment = False
if self._particle.__class__ == Particle:
from pytom_volume import read
particleFile = self._particle.getFilename()
particle = read(particleFile, 0, 0, 0, 0, 0, 0, 0, 0, 0,
self._binning, self._binning, self._binning)
else:
raise TypeError(
'Provided particle must be of type pytom.basic.structures.Particle'
)
if self._scoreObject.__class__ == str:
from pytom.score.score import Score
string = self._scoreObject
self._scoreObject = Score()
self._scoreObject.fromStr(string)
if self._reference.wasGeneratedBy(
self._particle) and self._reference.hasPreWedge(
) and self._reference.hasWeighting():
#substract particle from reference if information is available
[reference, self._referenceWeighting
] = self._reference.subtractParticle(self._particle,
self._binning)
referenceObject = self._reference
else:
if self._reference.__class__ == Reference:
from pytom_volume import read
referenceObject = self._reference
referenceFile = self._reference.getReferenceFilename()
reference = read(referenceFile, 0, 0, 0, 0, 0, 0, 0, 0, 0,
self._binning, self._binning, self._binning)
if self._referenceWeighting == str and len(
self._referenceWeighting) > 0:
self._referenceWeightingFile = self._referenceWeighting
if binning == 1:
from pytom_volume import read
self._referenceWeighting = read(self._referenceWeighting)
else:
from pytom.basic.files import readSubvolumeFromFourierspaceFile
self._referenceWeighting = readSubvolumeFromFourierspaceFile(
self._referenceWeightingFile, reference.sizeX(),
reference.sizeY(), reference.sizeZ())
self._mask.setBinning(self._binning)
pScore = self._particle.getScore()
if not pScore:
pScore = self._scoreObject
if self._doAlignment:
#process FRM alignment for each particle and class
peakPrior = self._scoreObject.getPeakPrior()
newShift, newRotation, score = FRMAlignmentWrapper(
particle, self._particle.getWedge(), reference, Wedge([0, 0]),
self._frmBandwidth,
int(self._preprocessing.getHighestFrequency()), self._mask,
self._scoreObject.getPeakPrior())
self._scoreObject.setValue(score)
result = MaximisationResult(
self._particle, referenceObject, self._scoreObject, newShift,
newRotation, OneAngleList(self._particle.getRotation()))
else:
#process classification without alignment only
peakV = compareTwoVolumes(
particle, reference, self._referenceWeighting,
self._particle.getWedge(),
OneAngleList(self._particle.getRotation()),
self._particle.getShift(), self._scoreObject, self._mask,
self._preprocessing, self._binning)
self._scoreObject.setValue(peakV)
result = MaximisationResult(
self._particle, referenceObject, self._scoreObject,
self._particle.getShift(), self._particle.getRotation(),
OneAngleList(self._particle.getRotation()))
return result