def distribute_job(self, job, verbose=False):
pl_filename = job["ParticleList"]
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(pl_filename)
nn = len(pl)
all_pairs = []
for i in range(nn):
for j in range(i+1, nn):
all_pairs.append((i, j))
n = len(all_pairs)
particlesPerNode = int(n/self.num_workers)
residual = n-particlesPerNode*self.num_workers
start_idx = 0
for i in range(1, self.num_workers+1):
if i < residual+1: # since i starts from 1
l = particlesPerNode+1
else:
l = particlesPerNode
sub_pairs = all_pairs[start_idx : start_idx+l]
start_idx += l
# construct the job
sub_job = {}
sub_job["Pairs"] = sub_pairs
sub_job["ParticleList"] = pl_filename
sub_job["Mask"] = job["Mask"]
sub_job["Frequency"] = job["Frequency"]
sub_job["Binning"] = job["Binning"]
self.send_job(sub_job, i)
if verbose:
print(self.node_name + ': distributed %d particles to node %d' % (len(sub_pairs), i))
def get_phase_flip_pl(self):
if self.phase_flip_pl_obj is None:
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(self.phase_flip_pl)
self.phase_flip_pl_obj = pl
return self.phase_flip_pl_obj
def start(self, job, outdir='./', verbose=False):
outdir = job["outdir"]
if self.mpi_id == 0:
import numpy as np
pl_filename = job["ParticleList"]
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(pl_filename)
n = len(pl)
correlation_matrix = np.ones((n, n))
# distribute the job
self.distribute_job(job, verbose)
# fill in the diagnal line
for i in range(n):
correlation_matrix[i][i] = 1
# gather the results
for i in range(self.num_workers):
result = self.get_result()
pairs = list(result.keys())
for pair in pairs:
correlation_matrix[pair[0]][pair[1]] = result[pair]
correlation_matrix[pair[1]][pair[0]] = result[pair]
# write the correlation matrix to the disk
np.savetxt(os.path.join(outdir, 'correlation_matrix.csv'), correlation_matrix, delimiter=',')
# send end signal to other nodes and terminate itself
self.end(verbose)
else:
# other nodes
self.run(verbose)
def writeCroppedParticles(particleListName, output, center, cubesize):
"""
@param particleListName: name of particle list
@type particleListName: str
@param output: Name of output particles
@type output: str
@param center: center of output particles in template orientation
@type center: list
@param cubesize: Size of output particles in pixel
@type cubesize: int
"""
from pytom.basic.structures import ParticleList, Particle, Shift
from pytom_volume import transformSpline as transform
from pytom_volume import subvolume, vol
pl = ParticleList()
pl.fromXMLFile(filename=particleListName)
#copy particle list for list of cropped particles
pl_new = pl.copy()
pvol = pl[0].getVolume()
sizeX = pvol.sizeX()
sizeY = pvol.sizeY()
sizeZ = pvol.sizeZ()
pvol_ali = vol(sizeX, sizeY, sizeZ)
subV = vol(cubesize, cubesize, cubesize)
sub_startX = center[0]-cubesize/2
sub_startY = center[1]-cubesize/2
sub_startZ = center[2]-cubesize/2
if (sub_startX < 0) or (sub_startY < 0) or (sub_startZ < 0):
raise ValueError('cubesize too large :(')
for (ipart, part) in enumerate(pl):
pvol_ali.setAll(0)
subV.setAll(0)
pvol = part.getVolume()
rot = part.getRotation()
rotinvert = rot.invert()
shiftV = part.getShift()
transform(pvol, pvol_ali, rotinvert[0], rotinvert[1], rotinvert[2],
sizeX/2, sizeY/2, sizeZ/2, -shiftV[0], -shiftV[1], -shiftV[2], 0, 0, 0)
# box out subvolume
subV = subvolume(pvol_ali, sub_startX, sub_startY, sub_startZ, cubesize, cubesize, cubesize)
transform(subV, subV, rot[0], rot[1], rot[2], cubesize/2, cubesize/2, cubesize/2, 0, 0, 0, 0, 0, 0)
fname = part.getFilename()
idx = fname.split('_')[-1].split('.')[0]
nfname = output+'_'+idx+'.em'
print("write file " + nfname)
subV.write(nfname)
pl_new[ipart].setFilename(newFilename=nfname)
pl_new[ipart].setShift(shift=Shift(0,0,0))
return pl_new
def cleanUp_RandomParticleList(pl_filename='pl.xml', pdir='./testparticles'):
"""
remove directories
"""
from os import remove, rmdir
from pytom.basic.structures import ParticleList
pl = ParticleList()
pl.fromXMLFile(filename=pl_filename)
for part in pl:
remove(part.getFilename())
rmdir(pdir)
remove(pl_filename)
def run(fname, outname, cores=6):
even = ParticleList()
even.fromXMLFile(fname)
aa = averageParallel(particleList=even,
averageName=outname,
showProgressBar=True,
verbose=False,
createInfoVolumes=False,
weighting=False,
norm=False,
setParticleNodesRatio=3,
cores=cores)
def extractClassesFromPL(pl_name, class_names, output):
from pytom.basic.structures import ParticleList
pl = ParticleList()
pl.fromXMLFile(pl_name)
pls = pl.splitByClass()
class_labels = class_names.split(',')
res = ParticleList()
for pp in pls:
if pp[0].getClass() in class_labels:
res += pp
if output: res.toXMLFile(output)
return res
def subTomoClust(particleListFilename,
classifiedParticleListFilename,
cccName,
neig,
nclass,
verbose=False):
"""
subtomogram clustering using CCC and kmeans
@param particleListFilename: particle list filename
@type particleListFilename: str
@param classifiedParticleListFilename: output particle list filename
@type classifiedParticleListFilename: str
@param cccName: Name of (Constrained) Correlation Coefficient (CCC) matrix
@type cccName: str
@param neig: number of eigenvectors
@type neig: int
@param nclass: number of classes
@type nclass: int
@author: FF Jan 2013
"""
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(particleListFilename)
if verbose:
print("Particle List read in")
ccc = readCCC(cccName)
if verbose:
print("CCC read in")
coeffs, eigvalues = SVD_analysis(ccc)
if verbose:
print("Eigen analysis done")
labels = kmeansCluster(coeff=coeffs, neig=neig, nclass=nclass)
if verbose:
print("kmeans clustering done")
for (ipart, part) in enumerate(pl):
part.setClass(className=str(labels[ipart]))
if verbose:
print("Class labels assigned")
pl.toXMLFile(classifiedParticleListFilename)
if verbose:
print("File written")
def get_size(particleList, directory):
tempPL = ParticleList()
tempPL.fromXMLFile(particleList)
tomoName = tempPL[0].getPickPosition().getOriginFilename(
) if tempPL[0].getPickPosition().getOriginFilename(
) else tempPL[0].getSourceInfo().getTomoName()
if not os.path.exists(tomoName):
tomoName = os.path.join(directory, tomoName)
if not os.path.exists(tomoName):
return 'Failed'
try:
dimx, dimy, dimz = read_size(tomoName)
except:
print('Failed')
return 'Failed'
return [dimx, dimy, dimz]
def averageClasses(particleListFilename, avName):
"""
write class averages of classified particles
@param particleListFilename: particle list filename
@type particleListFilename: str
@param avName: Name for class averages (<avName>_iclass.em)
@type avName: str
@author: FF
@date: Jan 2013
"""
from pytom.basic.structures import ParticleList
pl = ParticleList()
pl.fromXMLFile(particleListFilename)
pl.sortByClassLabel()
pls = pl.splitByClass()
for cl in pls:
className = cl[0].getClassName()
cl.average(avName + "_" + str(className) + '.em')
print(className, ' contains ', len(cl), ' particles')
def mirrorParticleList(particleList, outname, directory='./'):
sizes = get_size(particleList, directory)
if sizes == 'Failed':
print(
'Mirroring particle coordinates did not succeed. Please ensure the paths to the origin tomogram are correct'
)
return
dimx, dimy, dimz = sizes
tempPL = ParticleList()
tempPL.fromXMLFile(particleList)
for particle in tempPL:
pp = particle.getPickPosition()
pp.setX(dimx - pp.getX())
pp.setY(dimy - pp.getY())
pp.setZ(dimz - pp.getZ())
shift = particle.getShift()
shift.invert()
tempPL.toXMLFile(outname)
def readSetOfVolumes(volsXml, volSet, **kwargs):
""" Populate a Scipion set of volumes from a given
xml file from PyTom.
"""
# Add to the path the root to pytom
backupPath = list(sys.path)
addPyTomPaths()
from pytom.basic.structures import Particle, ParticleList, Wedge
from pytom.score.score import Score, PeakPrior
from pytom.frm.FRMAlignment import FRMScore
pl = ParticleList()
pl.fromXMLFile(volsXml)
xmlDir = os.path.dirname(volsXml)
for particle in pl:
volFn = os.path.join(xmlDir, particle.getFilename())
vol = em.Volume()
vol.setFileName(volFn)
vol.pytomInfo = readPyTomInfo(particle)
volSet.append(vol)
sys.path = backupPath
ScriptOption('-p', 'Particle list.', True, False),
ScriptOption('-c', 'Class label file.', True, False),
ScriptOption('-o', 'Output particle list.', True, False),
ScriptOption('-t', 'True positive class.', True, True),
ScriptOption(['-h', '--help'], 'Help.', False, True)
])
if len(sys.argv) == 1:
print(helper)
sys.exit()
try:
pl_filename, class_label_filename, output, tp_label, help = parse_script_options(
sys.argv[1:], helper)
except Exception as e:
print(e)
sys.exit()
if help is True:
print(helper)
sys.exit()
if tp_label is None:
tp_label = 1
else:
tp_label = int(tp_label)
from pytom.basic.structures import ParticleList
pl = ParticleList(".")
pl.fromXMLFile(pl_filename)
pl.setClassFromLocalizationClassList(class_label_filename)
new_pl = pl.particlesFromClass(tp_label)
new_pl.toXMLFile(output)
def run(self, verbose=False):
from pytom_volume import read, sum
from pytom.basic.filter import lowpassFilter
from pytom.basic.correlation import nxcc
from pytom.basic.structures import Rotation
from pytom.tools.ProgressBar import FixedProgBar
while True:
# get the job
job = self.get_job()
try:
pairs = job["Pairs"]
pl_filename = job["ParticleList"]
except:
if verbose:
print(self.node_name + ': end')
break # get some non-job message, break it
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(pl_filename)
if verbose:
prog = FixedProgBar(0, len(pairs)-1, self.node_name+':')
i = 0
# run the job
result = {}
last_filename = None
binning = int(job["Binning"])
mask = read(job["Mask"], 0, 0, 0, 0, 0, 0, 0, 0, 0, binning, binning, binning)
for pair in pairs:
if verbose:
prog.update(i)
i += 1
g = pl[pair[0]]
f = pl[pair[1]]
vf = f.getTransformedVolume(binning)
wf = f.getWedge().getWedgeObject()
wf_rotation = f.getRotation().invert()
# wf.setRotation(Rotation(-rotation[1],-rotation[0],-rotation[2]))
# wf_vol = wf.returnWedgeVolume(vf.sizeX(), vf.sizeY(), vf.sizeZ(), True, -rotation[1],-rotation[0],-rotation[2])
vf = lowpassFilter(vf, job["Frequency"], 0)[0]
if g.getFilename() != last_filename:
vg = g.getTransformedVolume(binning)
wg = g.getWedge().getWedgeObject()
wg_rotation = g.getRotation().invert()
# wg.setRotation(Rotation(-rotation[1],-rotation[0],-rotation[2]))
# wg_vol = wg.returnWedgeVolume(vg.sizeX(), vg.sizeY(), vg.sizeZ(), True, -rotation[1],-rotation[0],-rotation[2])
vg = lowpassFilter(vg, job["Frequency"], 0)[0]
last_filename = g.getFilename()
score = nxcc( wg.apply(vf, wg_rotation), wf.apply(vg, wf_rotation), mask)
# overlapped_wedge_vol = wf_vol * wg_vol
# scaling = float(overlapped_wedge_vol.numelem())/sum(overlapped_wedge_vol)
# score *= scaling
result[pair] = score
# send back the result
self.send_result(result)
pytom_mpi.finalise()
def polish_particles(particle_list_filename,
projection_directory,
averaged_subtomogram,
binning,
offset,
projections,
tilt_angles,
mpi,
fsc_path='',
peak_border=75,
outputDirectory='./',
create_graphics=False,
number_of_particles=0,
verbose=False):
"""
To polish a particle list based on (an) initial subtomogram(s).
:param particle_list_filename: the filename of the particlelist
:type particle_list_filename: str
:param projection_directory: the directory of the projections
:type projection_directory: str
:param averaged_subtomogram: to give a path to an averaged subtomogram to be used instead of subtomograms of all
particles separately
:type averaged_subtomogram: str
:param binning: the binning factor used
:type binning: int
:param offset: the offset used (x, y, z)
:type offset: list(int, int, int)
:param projections: a list with filenames of projections
:type projections: list(str)
:param tilt_angles: the list of tiltangles used
:type tilt_angles: list(int)
:param mpi: the instance of mpi which can be used for multi threading
:type mpi: mpi instance
:param create_graphics: to create plots of major parts of the algorithm, mainly used for debugging
and initial creation
:type create_graphics: bool
:param number_of_particles: to use a subset of the particles for the particle polishing
:type number_of_particles: int
:param skip_alignment: skips the alignment phase, does not do particle polishing
:type skip_alignment: bool
:return: nothing, it writes everything to disk
:returntype: void
"""
assert number_of_particles == -1 or number_of_particles > 0
assert binning > 0
assert vol_size > 0
assert vol_size % 2 == 0
assert isinstance(projections, list)
assert isinstance(vol_size, int)
assert isinstance(binning, int)
assert isinstance(offset, list) and len(offset) == 3
assert isinstance(offset[0], int) and isinstance(
offset[1], int) and isinstance(offset[2], int)
assert isinstance(tilt_angles, list)
assert isinstance(particle_list_filename, str)
assert isinstance(projection_directory, str)
assert isinstance(create_graphics, bool)
assert isinstance(averaged_subtomogram, str)
assert isinstance(number_of_particles, int)
assert isinstance(skip_alignment, bool)
import numpy as np
import os
from pytom.tompy.io import read_size, read
from pytom.basic.datatypes import fmtLAR, headerLocalAlignmentResults, LOCAL_ALIGNMENT_RESULTS
# load particle list
from pytom.basic.structures import ParticleList
particlelist = ParticleList()
particlelist.fromXMLFile(particle_list_filename)
particle_list_name = os.path.splitext(
os.path.basename(str(particle_list_filename)))[0]
if number_of_particles > 0:
particlelist = particlelist[:number_of_particles]
if verbose:
print(len(particlelist))
print("{:s}> Creating the input array".format(gettime()))
dimz = read_size(particlelist[0].getPickPosition().getOriginFilename(),
'z') * binning
vol_size = read_size(particlelist[0].getFilename(), 'x')
input_to_processes = []
# data = {}
# for projectioname in projections:
# data[projectioname] = read(projectioname)
#
# template = read(averaged_subtomogram)
input_to_processes = []
for particle_number, particle in enumerate(particlelist):
rot = (particle.getRotation().getZ1(), particle.getRotation().getX(),
particle.getRotation().getZ2())
# loop over tiltrange, take patch and cross correlate with reprojected subtomogram
for img, ang in zip(projections, tilt_angles):
input_to_processes.append([
averaged_subtomogram,
ang,
offset,
vol_size,
particle.getPickPosition().toVector(),
rot,
particle.getFilename(),
particle_number,
binning,
img,
create_graphics,
fsc_path,
dimz,
peak_border,
])
if verbose:
print(len(input_to_processes))
print("{:s}> Created the input array".format(gettime()))
if verbose: print("{:s}> Started on running the process".format(gettime()))
lists = list(zip(*input_to_processes))
if verbose: print(len(list(lists)))
output = mpi.parfor(
run_single_tilt_angle,
list(
zip(lists[0], lists[1], lists[2], lists[3], lists[4], lists[5],
lists[6], lists[7], lists[8], lists[9], lists[10], lists[11],
lists[12], lists[13]))) # Some problem internally 23/10/2019
results_file = os.path.join(outputDirectory,
f"resultsPolish_{particle_list_name}.txt")
np.savetxt(results_file,
np.array(output, dtype=LOCAL_ALIGNMENT_RESULTS),
fmt=fmtLAR,
header=headerLocalAlignmentResults)
if verbose: print("{:s}> Ran the processes".format(gettime()))
from pytom.tools.parse_script_options import parse_script_options
from pytom.basic.structures import ParticleList, SingleTiltWedge
helper = ScriptHelper(sys.argv[0].split('/')[-1], # script name
description='Set the same wedge to all the particles in the particle list.',
authors='Yuxiang Chen',
options=[ScriptOption(['-p'], 'Particle list', True, False),
ScriptOption(['-w'], 'Wedge Angle (degree)', True, False),
ScriptOption(['-o'], 'Output particle list', True, True),
ScriptOption(['-h', '--help'], 'Help.', False, True)])
if len(sys.argv) == 1:
print(helper)
sys.exit()
try:
pl_name, wedge_angle, output, bHelp = parse_script_options(sys.argv[1:], helper)
except:
sys.exit()
if bHelp is True:
print(helper)
sys.exit()
pl = ParticleList()
pl.fromXMLFile(pl_name)
w = SingleTiltWedge(int(wedge_angle))
pl.setWedgeAllParticles(w)
if output is None:
pl.toXMLFile(pl_name)
else:
pl.toXMLFile(output)
from pytom_volume import read
from sh.frm import frm_align_vol
from pytom.tools.maths import rotation_distance, euclidianDistance
from pytom.tools.timing import Timing
from pytom.basic.structures import ParticleList
from pytom.basic.structures import Shift, Rotation
pl = ParticleList('.')
pl.fromXMLFile('/fs/home/ychen/4Chen/first100.xml')
r = read('/fs/home/ychen/4Chen/avg_first100.em')
for pp in pl:
v = read(pp.getFilename())
pos, ang = frm_align_vol(v, [-60.0, 60.0], r, [8, 32], 10, mask=30)
pp.setShift(
Shift([
pos[0] - v.sizeX() / 2, pos[1] - v.sizeY() / 2,
pos[2] - v.sizeZ() / 2
]))
pp.setRotation(Rotation(ang))
pl.average('average.em', True)
##!/bin/bash
## This script merges desired classes from one or multiple particleLists of individual tomograms. It utilizes the pytom function pl.splitByClass
## Utrecht University, 19.11.2019, RE
#module load openmpi/2.1.1 pytom/0.971
from pytom.basic.structures import ParticleList
import os
## The variable nice_classes_list contains arrays specifying the tomogram and classes. Each array should contain the index of the desired tomograms in the first position, followed by the indices of the desired classes from that tomogram.
## In the example below, the first array [8,0,1,3,4] specifies the classes 0, 1, 3 and 4 within tomogram 8.
nice_classes_list = [[8,0,1,3,4], [9,0], [13,2], [14,0,1,2], [15,0,1,3], [16,0,1], [20,1], [21,3], [24,1], [27,2], [29,1], [30,1], [31,0], [32,1], [33,1], [34,1], [35,1], [36,1,4], [38,1], [43,3], [45,1], [46,1], [52,4], [53,1,3], [54,1], [79,3]]
pl=ParticleList()
## Below, adjust the path inside pl.fromXMLFile to point to the individual particle lists.
for nice_classes in nice_classes_list:
pl.fromXMLFile('tomogram' + str(nice_classes_list[nice_classes][0]) + '/subtomograms/subtomos_bin6/AC3D/classified_pl_iter9.xml')
lists=pl.splitByClass()
for i in range(len(nice_classes_list[nice_classes])-1)
ribos=ribos.append(lists[nice_classes_list[nice_classes][i + 1])
## specify the desired output path below
ribos.toXMLFile('AC3d_pickedClasses.xml')
def updatePL(fnames,
outnames,
directory='',
suffix='',
wedgeangles=[],
multiplypickpos=1,
multiplyshift=None,
new_center=[],
sizeSubtomo=64,
move_shift=False,
binSubtomo=1,
binRecon=1,
rotation=[],
anglelist='',
mirror=False,
tomogram_dir='./',
convention='zxz'):
if type(fnames) == str:
fnames = [fnames]
if type(outnames) == str:
outnames = [outnames]
try:
wedgelen = len(wedgeangles)
except:
wedgelen = 0
for n, xmlfile in enumerate(fnames):
tempPL = ParticleList()
tempPL.fromXMLFile(xmlfile)
for particle in tempPL:
# Update directory to particle
if directory:
filename = os.path.join(
directory, os.path.basename(particle.getFilename()))
particle.setFilename(filename)
# Add suffix to directory name in which particle is stored
# e.g. suffix = _bin3
# --> Subtomograms/tomogram_000/particle_1.em will become Subtomograms/tomogram_000_bin3/particle_1.em
if suffix:
filename = particle.getFilename()
filename = os.path.join(
os.path.dirname(filename) + suffix,
os.path.basename(filename))
particle.setFilename(filename)
# Update wedge angles of angle1 and angle2
if wedgelen > n + 1:
w = particle.getWedge()
w.setWedgeAngles(wedgeangles[n * 2:n * 2 + 2])
# Multiply pick position
if abs(multiplypickpos - 1) > 1E-3:
pp = particle.getPickPosition()
pp.scale(multiplypickpos)
# Shift is multiply by the respective binning factor.
if not (multiplyshift is None):
shift = particle.getShift()
shift.scale(multiplyshift)
# Randomize the angles of all particles in particle list.
if type(anglelist) == type(numpy.array([])):
cc = 180. / numpy.pi
import random
z1, z2, x = random.choice(anglelist)
particle.setRotation(rotation=Rotation(
z1=z1 * cc, z2=z2 * cc, x=x * cc, paradigm='ZXZ'))
shift = particle.getShift()
angles = particle.getRotation().toVector(convention=convention)
rot_particleList = R.from_euler(convention, angles, degrees=True)
if new_center:
new_center_vector = numpy.array(new_center) - sizeSubtomo // 2
new_center_vector_rotated = rot_particleList.apply(
new_center_vector)
shift.addVector(new_center_vector_rotated)
if move_shift == True:
pp = particle.getPickPosition()
shift.scale(binSubtomo / binRecon)
ss = shift.toVector()
pp.setX(pp.getX() + ss[0])
pp.setY(pp.getY() + ss[1])
pp.setZ(pp.getZ() + ss[2])
particle.setPickPosition(pp)
shift.scale(0.)
#print(particle)
# Combine rotations from particleList and rotation
if rotation:
rot_rotation = R.from_euler(convention, rotation, degrees=True)
combined_rotation = rot_particleList * rot_rotation
z1, x, z2 = combined_rotation.as_euler(convention,
degrees=True)
particle.setRotation(
rotation=Rotation(z1=z1, z2=z2, x=x, paradigm='ZXZ'))
if mirror:
# Update shifts
shift.scale(-1)
particle.setShift(shift)
# Update angles as well
rotationT = particle.getRotation()
rotationT.setZ1(-1 * rotationT.getZ1())
rotationT.setZ2(-1 * rotationT.getZ2())
rotationT.setX(-1 * rotationT.getX())
particle.setRotation(rotationT)
tempPL.toXMLFile(outnames[n])
if mirror:
mirrorParticleList(outnames[n],
outnames[n],
directory=tomogram_dir)
print('Command not right. Exit!')
sys.exit()
for o, a in opts:
if o in ("-h", "--help"):
print(usage)
sys.exit()
if o in ("-l"):
projDir = a
if o in ("-p"):
plFilename = a
if o in ("-b"):
binning = int(a)
if o in ("-s"):
size = int(a)
if o in ("-f"):
offset = [int(i) for i in a.split(",")]
from pytom.basic.structures import ParticleList
pl = ParticleList('.')
pl.fromXMLFile(plFilename)
pl.setCenterInPickVolume(
-offset[0], -offset[1],
-offset[2]) # thomas has changed the sign of it!!!
from pytom.reconstruction.reconstructionStructures import ProjectionList
projs = ProjectionList()
projs.loadDirectory(projDir)
projs.generateVolumes(pl, size, binning, False, True, False)
def interpretRequestParameters(parameters):
"""
interpretRequestParameters
"""
from pytom.basic.structures import ParticleList, SampleInformation, Reference, Mask, Wedge
from pytom.alignment.preprocessing import Preprocessing
from pytom.frontend.serverpages.serverMessages import FileMessage
particleList = ParticleList()
if 'plXML' in parameters:
particleList.fromXMLFile(parameters['plXML'])
elif 'plDIR' in parameters:
particleList = ParticleList(parameters['plDIR'])
particleList.loadDirectory()
else:
raise RuntimeError('ParticleList parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(parameters['pixSize'])
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(parameters['partDia'])
else:
raise RuntimeError('Particle diameter missing in request!')
if 'wa1' in parameters:
wedgeAngle1 = float(parameters['wa1'])
else:
raise RuntimeError('Wedge angle 1 parameter missing in request!')
if 'wa2' in parameters:
wedgeAngle2 = float(parameters['wa2'])
else:
raise RuntimeError('Wedge angle 2 parameter missing in request!')
wedgeInfo = Wedge([wedgeAngle1, wedgeAngle2])
if 'mask' in parameters:
mask = Mask(parameters['mask'])
else:
raise RuntimeError('Mask parameter missing in request!')
if not 'lowestF' in parameters:
raise RuntimeError('Lowest frequency parameter missing in request!')
if not 'highestF' in parameters:
raise RuntimeError('Highest frequency parameter missing in request!')
if not 'filtSm' in parameters:
raise RuntimeError('Filter smooth parameter missing in request!')
preprocessing = Preprocessing(float(parameters['lowestF']),
float(parameters['highestF']),
float(parameters['filtSm']))
score = None
if 'score' in parameters:
if parameters['score'] == 'xcf':
from pytom.score.score import xcfScore as scoreClass
elif parameters['score'] == 'nxcf':
from pytom.score.score import nxcfScore as scoreClass
elif parameters['score'] == 'flcf':
from pytom.score.score import FLCFScore as scoreClass
score = scoreClass()
else:
raise RuntimeError('Score parameter missing in request!')
if 'iter' in parameters:
iterations = int(parameters['iter'])
else:
raise RuntimeError('Number of iterations missing in request!')
if 'binning' in parameters:
binning = int(parameters['binning'])
else:
raise RuntimeError('Scaling parameter missing in request!')
if 'classes' in parameters:
numberClasses = float(parameters['classes'])
else:
raise RuntimeError('Number classes parameter missing in request!')
if 'conv' in parameters:
convergence = float(parameters['conv'])
else:
raise RuntimeError('Convergence parameter missing in request!')
if 'dest' in parameters:
destination = parameters['dest']
else:
raise RuntimeError('Destination parameter missing in request!')
sampleInfo = SampleInformation()
if 'pixSize' in parameters:
sampleInfo.setPixelSize(float(parameters['pixSize']))
else:
raise RuntimeError('Pixelsize parameter missing in request!')
if 'partDia' in parameters:
sampleInfo.setParticleDiameter(float(parameters['partDia']))
else:
raise RuntimeError('Particle diameter missing in request!')
from pytom.cluster.mcoEXMXStructures import MCOEXMXJob
job = MCOEXMXJob(particleList,
iterations,
destination,
mask,
score,
preprocessing,
wedgeInfo,
binning,
sampleInfo,
numberClasses,
convergence,
symmetry=None)
jobXMLFile = ''
if 'jobFile' in parameters:
jobXMLFile = parameters['jobFile']
job.toXMLFile(jobXMLFile)
jobRunFile = jobXMLFile[0:-3]
createRunscripts(jobRunFile + 'sh', jobXMLFile)
return FileMessage('MCOEXMXJob', jobXMLFile, 'created')
def extractParticleListsClosestToRefMarker(xmlfile,
markerfile,
binning_factor=8,
directory='./',
projDirTemplate=''):
from pytom.basic.structures import PickPosition, ParticleList
pL = ParticleList()
pL.fromXMLFile(os.path.join(directory, xmlfile))
dict_particle_lists = {}
for particle in pL:
tomogram = particle.getPickPosition().getOriginFilename().split(
'/')[-1].split('.')[0]
if not tomogram:
tomogram = particle.getSourceInfo().getTomoName().split('.')[0]
if not tomogram in dict_particle_lists.keys():
dict_particle_lists[tomogram] = ParticleList()
dict_particle_lists[tomogram].append(particle)
try:
markers = loadstar(markerfile, dtype=datatypeMR)
except:
correct_header_markerfile(markerfile)
markers = loadstar(markerfile, dtype=datatypeMR)
xmlsCM = []
for pl_key in dict_particle_lists.keys():
outLists = {}
for particle in dict_particle_lists[pl_key]:
x, y, z = particle.getPickPosition().toVector()
x *= binning_factor
y *= binning_factor
z *= binning_factor
closestMarkerIndex = determine_closest_marker(x, y, z, markers)
projectionDirectory = projDirTemplate.replace(
'_CLOSEST_', '_{:04d}_'.format(closestMarkerIndex))
markerPositionFile = f'{projectionDirectory}/markerLocations_irefmark_{closestMarkerIndex}.txt'
realignmarkers = loadstar(markerPositionFile, dtype=datatypeMR)
if not closestMarkerIndex in outLists.keys():
outLists[closestMarkerIndex] = ParticleList()
ox, oy = determineShiftXY(markers, realignmarkers)
oz = markers['OffsetZ'][closestMarkerIndex]
originFname = particle.getPickPosition().getOriginFilename()
print(x, y, z, ox, oy, oz)
pp = PickPosition(x=(x - ox) / binning_factor,
y=(y - oy) / binning_factor,
z=((z - oz) / binning_factor),
originFilename=originFname)
particle.setPickPosition(pp)
outLists[closestMarkerIndex].append(particle)
for markerIndex in outLists.keys():
outfname = '.tempCM_particleList_{}_refMarkerIndex_{}.xml'.format(
pl_key, markerIndex)
outfname = os.path.join(directory, outfname)
outLists[markerIndex].toXMLFile(outfname)
xmlsCM.append([markerIndex, outfname])
return xmlsCM
def run(parameters, verbose=False):
"""
run: Answers a http request depending on the provided parameters.
Parameters can be
XML=FILENAME
DIR=DIRNAME
ALIG=NAME
@param parameters:
"""
from pytom.basic.structures import ParticleList
from pytom.tools.files import checkDirExists, checkFileExists
if verbose:
print("Parsing particleList request!")
splitParameters = parameters.split('&')
if splitParameters.__class__ == list:
if verbose:
print(splitParameters)
for i in range(len(splitParameters)):
parameter = splitParameters[i]
split = parameter.split('=')
keyword = split[0]
argument = split[1]
if verbose:
print('Keyword : ', keyword)
print('Arguments : ', argument)
if keyword == 'XML':
from pytom.tools.files import readStringFile, getPytomPath
import io
from lxml import etree
if not checkFileExists(argument):
raise IOError('File not found!')
pl = ParticleList('/')
pl.fromXMLFile(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif keyword == 'DIR':
from pytom.tools.files import checkDirExists, getPytomPath, readStringFile
import io
from lxml import etree
if not checkDirExists(argument):
raise IOError('File not found!')
pl = ParticleList(argument)
pl.loadDirectory()
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif keyword == 'ALIG':
from pytom.tools.files import checkDirExists, getPytomPath, readStringFile
import io
from lxml import etree
if not checkDirExists(argument):
raise IOError('File not found!')
pl = ParticleList(argument)
pl.fromAlignmentFile(argument)
xsltString = readStringFile(
getPytomPath() + '/frontend/html/xslt/ParticleList.xsl')
xsltTransform = io.StringIO(xsltString)
transformed = pl.xsltTransform(xsltTransform)
return etree.tostring(transformed, pretty_print=True)
elif splitParameters.__class__ == str:
if verbose:
print(splitParameters)
def test(index=0):
from pytom.tompy.io import read
from pytom.tompy.transform import rotate3d
path_raw_projection = "/data2/dschulte/BachelorThesis/Data/VPP2/03_Tomographic_Reconstruction/tomogram_000/sorted/sorted_29.em"
path_aligned_projection = "/data2/dschulte/BachelorThesis/Data/VPP2/03_Tomographic_Reconstruction/tomogram_000/alignment/marker_0001_-60.0,60.0/sorted_aligned_30.em"
path_template = "/data2/dschulte/BachelorThesis/Data/VPP2/05_Subtomogram_Analysis/combo_reduced.em"
tilt_angle = -1.9989999533
raw_projection = read(path_raw_projection)
aligned_projection = read(path_aligned_projection)
template = read(path_template)
dim = aligned_projection.shape[0]
from pytom.basic.structures import ParticleList
particlelist1 = ParticleList()
particlelist1.fromXMLFile(
"/data2/dschulte/BachelorThesis/Data/VPP2/04_Particle_Picking/Picked_Particles/combined_reduced_extracted/particleList_combined_reduced_tomogram_010_WBP.xml"
)
align_results = (-15.3044300079, 3.7495634556, -184.3835906982,
1.0000053644) # -2
#align_results = (-1.2395387888, 4.9647006989, -184.3754882812, 1.0000000000) # 0
#pick_position = (278.12318382089245 * 8, 222.6395540890773 * 8, 268.97256780848085 * 8) #0
#pick_position = (381.21906883806 * 8, 153.61353397521387 * 8, 246.8315433927568 * 8) #74
#particle_rotation = (26.442828473505173, 44.44149544840194, 58.160958298848676) #0
#particle_rotation = (85.2456894956599, 30.815061362336394, 9.543300915975514) #74
pick_position = particlelist1[index].getPickPosition().toVector()
particle_rotation = (particlelist1[index].getRotation().getZ1(),
particlelist1[index].getRotation().getX(),
particlelist1[index].getRotation().getZ2())
print(pick_position, particle_rotation)
align_transformation, raw_position, aligned_position, template_transformation = combine_trans_projection(
align_results, pick_position, particle_rotation, tilt_angle, dim, 8,
template.shape[0])
d = 100
raw_patch = raw_projection[int(raw_position[0] - d):int(raw_position[0] +
d),
int(raw_position[1] - d):int(raw_position[1] +
d)].squeeze()
raw_patch = raw_patch / np.mean(raw_patch)
aligned_patch = aligned_projection[int(aligned_position[0] -
d):int(aligned_position[0] + d),
int(aligned_position[1] -
d):int(aligned_position[1] +
d)].squeeze()
aligned_patch = aligned_patch / (np.mean(aligned_patch))
aligned_raw = matrix_apply_to_2d(aligned_projection.squeeze(),
align_transformation)
aligned_raw_patch = aligned_raw[int(raw_position[0] -
d):int(raw_position[0] + d),
int(raw_position[1] -
d):int(raw_position[1] + d)].squeeze()
aligned_raw_patch = aligned_raw_patch / (np.mean(aligned_raw_patch))
transformed_template = matrix_apply_to_3d_3x3(template,
template_transformation)
print(np.mean(transformed_template), np.mean(template))
template_2d = transformed_template.sum(axis=2)
template_2d = template_2d / np.mean(template_2d)
print(template_2d.shape)
template_vol = rotate3d(template,
phi=particle_rotation[0],
the=particle_rotation[1],
psi=particle_rotation[2])
template_vol = rotate3d(template_vol, the=tilt_angle)
template_vol_2d = template_vol.sum(axis=2)
from pytom.reconstruction.reconstruct_local_alignment import normalised_cross_correlation_numpy, find_sub_pixel_max_value_2d
raw_cc = normalised_cross_correlation_numpy(raw_patch, template_2d)
rx, ry, _ = find_sub_pixel_max_value_2d(raw_cc)
aligned_cc = normalised_cross_correlation_numpy(aligned_patch,
template_vol_2d)
tx, ty, _ = find_sub_pixel_max_value_2d(aligned_cc)
import pylab as pl
import scipy
f, ax = pl.subplots(2, 3, figsize=(15, 10))
for i in range(2):
for j in range(3):
ax[i][j].axis('off')
ax[0][0].set_title('Raw Data Particle')
ax[0][0].imshow(scipy.ndimage.gaussian_filter(raw_patch, 3))
ax[0][1].set_title('Template Transformed to Raw Data')
ax[0][1].imshow(template_2d)
ax[0][2].set_title('Cross Correlation')
ax[0][2].imshow(raw_cc)
ax[0][2].text(0.05,
0.05,
'Peak\nx: {0:.2f}\ny: {0:.2f}'.format(rx, ry),
transform=ax[0][2].transAxes,
color='white')
ax[1][0].set_title('Aligned Data Particle')
ax[1][0].imshow(scipy.ndimage.gaussian_filter(aligned_patch, 3))
ax[1][1].set_title('Template Aligned to Aligned Data')
ax[1][1].imshow(template_vol_2d)
ax[1][2].set_title('Cross Correlation')
ax[1][2].imshow(aligned_cc)
ax[1][2].text(0.05,
0.05,
'Peak\nx: {0:.2f}\ny: {0:.2f}'.format(tx, ty),
transform=ax[1][2].transAxes,
color='white')
f.tight_layout()
pl.show()
print("binning = ", str(settings["binning"]))
print("mask = ", str(settings["mask"]))
print("fmask = ", str(settings["fmask"]))
print("niteration = ", str(settings["niteration"]))
print("dispersion = ", str(settings["dispersion"]))
print("threshold = ", str(settings["threshold"]))
print("noise = ", str(settings["noise"]))
print("noalign = ", str(settings["noalign"]))
print("outputdir = ", str(settings["output_directory"]))
print("noise = ", str(settings["noise"]))
print('----------------------------------------------------')
print()
# start the clustering
mpi.begin()
import time
tt = time.time()
try:
pl = ParticleList()
pl.fromXMLFile(options.filename)
classify(pl, settings)
except Exception as e:
print(e)
finally:
mpi.end()
print(f'total time: {time.time()-tt} sec')
def fromXML(self, xmlObj):
"""
read from xml file
@param xmlObj: xml object
@type xmlObj: L{lxml.etree.Element}
"""
from lxml.etree import _Element
if xmlObj.__class__ != _Element:
raise Exception('You must provide a valid XML object.')
if xmlObj.tag == "FRMJob":
jobDescription = xmlObj
else:
jobDescription = xmlObj.xpath('FRMJob')
if len(jobDescription) == 0:
raise Exception("This XML is not a FRMJob.")
jobDescription = jobDescription[0]
from pytom.basic.structures import ParticleList, Reference, Mask, SampleInformation, MultiSymmetries
pl = ParticleList('.')
particleList_element = jobDescription.xpath('ParticleList')
if len(particleList_element) > 0:
pl.fromXML(particleList_element[0])
else:
list_elements = jobDescription.xpath('ParticleListLocation')
for e in list_elements:
sub_pl = ParticleList()
sub_pl.fromXMLFile(e.get('Path'))
pl += sub_pl
self.particleList = pl
r = jobDescription.xpath('Reference')[0]
self.reference = Reference('')
self.reference.fromXML(r)
m = jobDescription.xpath('Mask')[0]
self.mask = Mask('')
self.mask.fromXML(m)
try:
si = jobDescription.xpath('SampleInformation')[0]
self.sampleInformation = SampleInformation()
self.sampleInformation.fromXML(si)
except:
self.sampleInformation = SampleInformation()
try:
syms = jobDescription.xpath('MultiSymmetries')[0]
self.symmetries = MultiSymmetries()
self.symmetries.fromXML(syms)
except:
self.symmetries = MultiSymmetries()
self.peak_offset = int(jobDescription.get('PeakOffset'))
self.bw_range = [
int(i)
for i in jobDescription.get('BandwidthRange')[1:-1].split(',')
]
self.freq = int(jobDescription.get('Frequency'))
self.destination = jobDescription.get('Destination')
self.max_iter = int(jobDescription.get('MaxIterations'))
self.r_score = jobDescription.get('RScore') == 'True'
self.weighting = jobDescription.get('WeightedAverage') == 'True'
self.bfactor = jobDescription.get('BFactor')
self.binning = int(jobDescription.get('binning'))
if jobDescription.get('AdaptiveResolution'):
adaptive_resolution = jobDescription.get('AdaptiveResolution')
if adaptive_resolution == '+1':
self.adaptive_res = False # always increase by 1
else:
self.adaptive_res = float(adaptive_resolution)
else:
self.adaptive_res = 0.0 # default, if not specified
if jobDescription.get('FSC'):
self.fsc_criterion = float(jobDescription.get('FSC'))
else:
self.fsc_criterion = 0.5 # default value
# for the constraint
try:
from sh_alignment.constrained_frm import AngularConstraint
con = jobDescription.xpath('AngularConstraint')
if len(con) != 0:
ac = AngularConstraint()
c = ac.fromXML(con[0])
self.constraint = c
else:
self.constraint = None
except:
self.constraint = None
sys.argv[1:], helper)
except Exception as e:
print(e)
sys.exit()
if help is True:
print(helper)
sys.exit()
try:
cores = int(cores)
except:
cores = 1
if not os.path.exists(plName):
print('Please provide an existing particle list')
sys.exit()
even = ParticleList()
even.fromXMLFile(plName)
averageParallel(particleList=even,
averageName=outname,
showProgressBar=showProgressBar,
verbose=verbose,
createInfoVolumes=createInfoVol,
weighting=weighting,
norm=norm,
setParticleNodesRatio=3,
cores=cores)
continue
tmp = projections[0]
sx = tmp.getXSize(
) # here should be the size of original projection!
sy = tmp.getYSize()
recOffset2 = [0, 0, 0]
recOffset2[0] = -sx / 2 + recOffset[0] * coordinateBinning
recOffset2[1] = -sy / 2 + recOffset[1] * coordinateBinning
recOffset2[2] = -sx / 2 + recOffset[2] * coordinateBinning
# set particle list in order to reconstruct subtomograms
particleList = ParticleList()
try:
particleList.fromXMLFile(particleListXMLPath)
except RuntimeError:
print('Error reading particleList XML file! Abort')
sys.exit()
from pytom.basic.structures import PickPosition
for particle in particleList:
pickPosition = particle.getPickPosition()
x = (pickPosition.getX() * coordinateBinning + recOffset2[0])
y = (pickPosition.getY() * coordinateBinning + recOffset2[1])
z = (pickPosition.getZ() * coordinateBinning + recOffset2[2])
particle.setPickPosition(PickPosition(x=x, y=y, z=z))
if alignResultFile:
print(f'Alignment result file is used.\n\n{alignResultFile}')
])
if len(sys.argv) == 1:
print(helper)
sys.exit()
try:
particleListName, volumeName, result, numberSymmetries, z1, x, z2, help = parse_script_options(
sys.argv[1:], helper)
except Exception as e:
print(e)
sys.exit()
if particleListName:
pl = ParticleList('.')
pl.fromXMLFile(particleListName)
elif volumeName:
volume = read(volumeName)
else:
raise RuntimeError(
'You must specify either a particle list or a volume file for symmetrization.'
)
if not z2:
z2 = 0
if not x:
x = 0
symmetry = PointSymmetry(numberSymmetries, z2, x)
if particleListName:
if not checkFileExists(mask):
raise RuntimeError('Mask file ' + mask + ' does not exist!')
if not checkDirExists(destination):
raise RuntimeError('Destination directory ' + destination +
' does not exist!')
from pytom.cluster.mcoEXMXStructures import MCOEXMXJob
from pytom.basic.structures import ParticleList, Reference, Mask, Wedge, SampleInformation, PointSymmetry
from pytom.score.score import FLCFScore
from pytom.frontend.serverpages.createMCOEXMXJob import createRunscripts
from pytom.alignment.preprocessing import Preprocessing
p = ParticleList()
p.fromXMLFile(particleList)
m = Mask(mask)
w = Wedge([float(wedge1), float(wedge2)])
pre = Preprocessing(lowestFrequency=float(lowestFrequency),
highestFrequency=float(highestFrequency))
sample = SampleInformation(pixelSize=float(pixelSize),
particleDiameter=float(diameter))
if symmetryN is None or symmetryAxisZ is None or symmetryAxisX is None:
sym = None
else:
sym = PointSymmetry(nfold=int(symmetryN),
z2=float(symmetryAxisZ),
x=float(symmetryAxisX))
job = MCOEXMXJob(particleList=p,numberIterations=numberIterations,\
print(helper)
sys.exit()
try:
volFilename, plFilename, cubeSize, help = parse_script_options(
sys.argv[1:], helper)
except:
sys.exit()
if help is True:
print(helper)
sys.exit()
cubeSize = int(cubeSize)
particleList = ParticleList()
try:
particleList.fromXMLFile(plFilename)
except:
from pytom.localization.structures import readParticleFile
particles = readParticleFile(plFilename)
particleList = ParticleList()
for particle in particles:
particleList.append(particle.toParticle())
particlePath = particleList[0].getFilename()
particleFolder = particlePath[0:particlePath.rfind('/')]
if not checkDirExists(particleFolder):
os.makedirs(particleFolder)
prog = FixedProgBar(0, len(particleList) - 1, '')
