def convolute(v, k, kernel_in_fourier=False):
"""
@param v: the volume to be convolute
@type v: L{pytom_volume.vol}
@param k: the convolution kernel in real space
@type k: L{pytom_volume.vol}
@param kernel_in_fourier: the given kernel is already in Fourier space or not (full size, shifted in the center! \
or reduced size). Default is False.
@type kernel_in_fourier: L{bool}
@return: Volume convoluted with kernel
@rtype: L{pytom_volume.vol}
"""
fv = fft(v)
if not kernel_in_fourier:
fk = fft(k)
res = fv*fk
else:
from pytom_volume import complexRealMult, fullToReduced
if k.sizeZ() == k.sizeX():
fk = fullToReduced(ftshift(k, inplace=False))
else:
fk = k
res = complexRealMult(fv, fk)
out = ifft(res)
out.shiftscale(0.0,1/float(out.sizeX()*out.sizeY()*out.sizeZ()))
return out
def convolutionCTF(volume, defocus, pixelSize=None, voltage=None, Cs=None, sigma=None):
"""
convolutionCTF:
@param volume: input volume to be convolved with CTF
@type volume: L{pytom_volume.vol}
@param defocus: Negative value = underfocus (in microns)
@param pixelSize: Size of pixels / voxels (in Anstroms)
@param voltage:
@param Cs:
@param sigma:
@return: CTF filtered volume
@rtype L{pytom_volume.vol}
@author: FF
"""
from pytom_volume import subvolume, complexRealMult
from pytom.basic.fourier import ftshift, fft, ifft
from pytom.basic.filter import volCTF
dimX = volume.sizeX()
dimY = volume.sizeY()
dimZ = volume.sizeZ()
ctf = volCTF(defocus, dimX, dimY, dimZ, pixelSize, voltage, Cs, sigma)
filterCTF = subvolume(ftshift(ctf,inplace=False), 0, 0, 0, dimX, dimY, (dimZ/2)+1)
filteredVolume = ifft( complexRealMult(fft(volume), filterCTF) )
return filteredVolume
def getWeightedProjectionCube(sourceVolume, thetaAngles):
from pytom_volume import vol, paste
from pytom_volume import complexRealMult
from pytom.basic.fourier import fft
from pytom.basic.fourier import ifft
from pytom.basic.filter import fourierFilterShift
from pytom.basic.filter import circleFilter
from pytom.basic.filter import rampFilter
dimX = sourceVolume.sizeX()
dimY = sourceVolume.sizeY()
imageCube = vol(dimX, dimY, len(thetaAngles))
imageCube.setAll(0.0)
weightSlice = fourierFilterShift(rampFilter(sourceVolume))
circleFilterRadius = dimX / 2
circleSlice = fourierFilterShift(
circleFilter(sourceVolume, circleFilterRadius))
for k in range(len(thetaAngles)):
angle = thetaAngles[k]
image = theta_vol_projection(sourceVolume, angle)
##filtered_img = ifft( complexRealMult(complexRealMult(complexRealMult(fft(image), filter_slice), circle_filter_slice), ctf) )
filteredImage = ifft(
complexRealMult(complexRealMult(fft(image), weightSlice),
circleSlice))
paste(filteredImage, imageCube, 0, 0, k)
#for i in range(dimX):
# for j in range(dimY):
# imageCube.setV(filteredImage.getV(i, j, 0), i, j, k)
return imageCube
def _disrtibuteAverageMPI(particleList,averageName,showProgressBar = False,verbose=False,
createInfoVolumes = False,setParticleNodesRatio = 3,sendEndMessage = False):
"""
_distributeAverageMPI : Distributes averaging to multiple MPI nodes.
@param particleList: The particles
@param averageName: Filename of new average
@param verbose: Prints particle information. Disabled by default.
@param createInfoVolumes: Create info data (wedge sum, inverted density) too? False by default.
@return: A new Reference object
@rtype: L{pytom.basic.structures.Reference}
@author: Thomas Hrabe
"""
import pytom_mpi
from pytom.alignment.structures import ExpectationJob
from pytom.parallel.parallelWorker import ParallelWorker
from pytom.parallel.alignmentMessages import ExpectationJobMsg
from pytom_volume import read,complexDiv,complexRealMult
from pytom.basic.fourier import fft,ifft
from pytom.basic.filter import lowpassFilter
from pytom.basic.structures import Reference
import os
import sys
numberOfNodes = pytom_mpi.size()
particleNodesRatio = float(len(particleList)) / float(numberOfNodes)
splitFactor = numberOfNodes
if particleNodesRatio < setParticleNodesRatio:
#make sure each node gets at least 20 particles.
splitFactor = len(particleList) / setParticleNodesRatio
splitLists = particleList.splitNSublists(splitFactor)
msgList = []
avgNameList = []
preList = []
wedgeList = []
for i in range(len(splitLists)):
plist = splitLists[i]
avgName = averageName + '_dist' +str(i) + '.em'
avgNameList.append(avgName)
preList.append(averageName + '_dist' +str(i) + '-PreWedge.em')
wedgeList.append(averageName + '_dist' +str(i) + '-WedgeSumUnscaled.em')
job = ExpectationJob(plist,avgName)
message = ExpectationJobMsg(0,i)
message.setJob(job)
msgList.append(message)
#distribute averaging
worker = ParallelWorker()
worker.fillJobList(msgList)
worker.parallelWork(True,sendEndMessage)
#collect results
result = read(preList[0])
wedgeSum = read(wedgeList[0])
for i in range(1,len(preList)):
r = read(preList[i])
result += r
w = read(wedgeList[i])
wedgeSum += w
result.write(averageName[:len(averageName)-3]+'-PreWedge.em')
wedgeSum.write(averageName[:len(averageName)-3] + '-WedgeSumUnscaled.em')
invert_WedgeSum( invol=wedgeSum, r_max=result.sizeX()/2-2., lowlimit=.05*len(particleList),
lowval=.05*len(particleList))
fResult = fft(result)
r = complexRealMult(fResult,wedgeSum)
result = ifft(r)
result.shiftscale(0.0,1/float(result.sizeX()*result.sizeY()*result.sizeZ()))
# do a low pass filter
result = lowpassFilter(result, result.sizeX()/2-2, (result.sizeX()/2-1)/10.)[0]
result.write(averageName)
# clean results
for i in range(0,len(preList)):
os.system('rm ' + avgNameList[i])
os.system('rm ' + preList[i])
os.system('rm ' + wedgeList[i])
return Reference(averageName,particleList)
def writeAlignedProjections(TiltSeries_,
weighting=None,
lowpassFilter=None,
binning=None,
verbose=False,
write_images=True):
"""write weighted and aligned projections to disk1
@param TiltSeries_: Tilt Series
@type TiltSeries_: reconstruction.TiltSeries
@param weighting: weighting (<0: analytical weighting, >1 exact weighting (value corresponds to object diameter in pixel AFTER binning)
@type weighting: float
@param lowpassFilter: lowpass filter (in Nyquist)
@type lowpassFilter: float
@param binning: binning (default: 1 = no binning). binning=2: 2x2 pixels -> 1 pixel, binning=3: 3x3 pixels -> 1 pixel, etc.
@author: FF
"""
import numpy
from pytom_numpy import vol2npy
from pytom.basic.files import read_em, write_em
from pytom.basic.functions import taper_edges
from pytom.basic.transformations import general_transform2d
from pytom.basic.fourier import ifft, fft
from pytom.basic.filter import filter as filterFunction, bandpassFilter
from pytom.basic.filter import circleFilter, rampFilter, exactFilter, fourierFilterShift, rotateFilter
from pytom_volume import complexRealMult, vol
import pytom_freqweight
from pytom.basic.transformations import resize
from pytom.gui.guiFunctions import fmtAR, headerAlignmentResults, datatypeAR
import os
if binning:
imdim = int(float(TiltSeries_._imdim) / float(binning) + .5)
else:
imdim = TiltSeries_._imdim
print('imdim', imdim)
sliceWidth = imdim
# pre-determine analytical weighting function and lowpass for speedup
if (weighting != None) and (weighting < -0.001):
w_func = fourierFilterShift(rampFilter(imdim, imdim))
print('start weighting')
# design lowpass filter
if lowpassFilter:
if lowpassFilter > 1.:
lowpassFilter = 1.
print("Warning: lowpassFilter > 1 - set to 1 (=Nyquist)")
# weighting filter: arguments: (angle, cutoff radius, dimx, dimy,
lpf = pytom_freqweight.weight(0.0, lowpassFilter * imdim / 2, imdim,
imdim // 2 + 1, 1,
lowpassFilter / 5. * imdim)
#lpf = bandpassFilter(volume=vol(imdim, imdim,1),lowestFrequency=0,highestFrequency=int(lowpassFilter*imdim/2),
# bpf=None,smooth=lowpassFilter/5.*imdim,fourierOnly=False)[1]
tilt_angles = []
for projection in TiltSeries_._ProjectionList:
tilt_angles.append(projection._tiltAngle)
tilt_angles = sorted(tilt_angles)
print(tilt_angles)
#q = numpy.matrix(abs(numpy.arange(-imdim//2, imdim//2)))
alignmentResults = numpy.zeros((len(TiltSeries_._ProjectionList)),
dtype=datatypeAR)
alignmentResults['TiltAngle'] = tilt_angles
for (ii, projection) in enumerate(TiltSeries_._ProjectionList):
alignmentResults['FileName'][ii] = os.path.join(
os.getcwd(), projection._filename)
transX = -projection._alignmentTransX / binning
transY = -projection._alignmentTransY / binning
rot = -(projection._alignmentRotation + 90.)
mag = projection._alignmentMagnification
alignmentResults['AlignmentTransX'][ii] = transX
alignmentResults['AlignmentTransY'][ii] = transY
alignmentResults['InPlaneRotation'][ii] = rot
alignmentResults['Magnification'][ii] = mag
if write_images:
if projection._filename.split('.')[-1] == 'st':
from pytom.basic.files import EMHeader, read
header = EMHeader()
header.set_dim(x=imdim, y=imdim, z=1)
idx = projection._index
if verbose:
print("reading in projection %d" % idx)
image = read(file=projection._filename,
subregion=[
0, 0, idx - 1, TiltSeries_._imdim,
TiltSeries_._imdim, 1
],
sampling=[0, 0, 0],
binning=[0, 0, 0])
if not (binning == 1) or (binning == None):
image = resize(volume=image, factor=1 / float(binning))[0]
else:
# read projection files
from pytom.basic.files import EMHeader, read, read_em_header
print(projection._filename)
image = read(projection._filename)
image = resize(volume=image, factor=1 / float(binning))[0]
if projection._filename[-3:] == '.em':
header = read_em_header(projection._filename)
else:
header = EMHeader()
header.set_dim(x=imdim, y=imdim, z=1)
if lowpassFilter:
filtered = filterFunction(volume=image,
filterObject=lpf,
fourierOnly=False)
image = filtered[0]
tiltAngle = projection._tiltAngle
header.set_tiltangle(tiltAngle)
# normalize to contrast - subtract mean and norm to mean
immean = vol2npy(image).mean()
image = (image - immean) / immean
# smoothen borders to prevent high contrast oscillations
image = taper_edges(image, imdim // 30)[0]
# transform projection according to tilt alignment
if projection._filename.split('.')[-1] == 'st':
newFilename = (TiltSeries_._alignedTiltSeriesName + "_" +
str(projection.getIndex()) + '.em')
else:
TiltSeries_._tiltSeriesFormat = 'mrc'
newFilename = (TiltSeries_._alignedTiltSeriesName + "_" +
str(projection.getIndex()) + '.' +
TiltSeries_._tiltSeriesFormat)
if verbose:
tline = ("%30s" % newFilename)
tline = tline + (" (tiltAngle=%6.2f)" % tiltAngle)
tline = tline + (": transX=%6.1f" % transX)
tline = tline + (", transY=%6.1f" % transY)
tline = tline + (", rot=%6.2f" % rot)
tline = tline + (", mag=%5.4f" % mag)
print(tline)
image = general_transform2d(v=image,
rot=rot,
shift=[transX, transY],
scale=mag,
order=[2, 1, 0],
crop=True)
# smoothen once more to avoid edges
image = taper_edges(image, imdim // 30)[0]
# analytical weighting
if (weighting != None) and (weighting < 0):
image = (ifft(complexRealMult(fft(image), w_func)) /
(image.sizeX() * image.sizeY() * image.sizeZ()))
elif (weighting != None) and (weighting > 0):
if abs(weighting - 2) < 0.001:
w_func = fourierFilterShift(
rotateFilter(tilt_angles, tiltAngle, imdim, imdim,
sliceWidth))
else:
w_func = fourierFilterShift(
exactFilter(tilt_angles, tiltAngle, imdim, imdim,
sliceWidth))
image = (ifft(complexRealMult(fft(image), w_func)) /
(image.sizeX() * image.sizeY() * image.sizeZ()))
header.set_tiltangle(tilt_angles[ii])
if newFilename.endswith('.mrc'):
data = copy.deepcopy(vol2npy(image))
mrcfile.new(newFilename,
data.T.astype(float32),
overwrite=True)
else:
write_em(filename=newFilename, data=image, header=header)
if verbose:
tline = ("%30s written ..." % newFilename)
outname = os.path.join(os.path.dirname(TiltSeries_._alignedTiltSeriesName),
'alignmentResults.txt')
numpy.savetxt(outname,
alignmentResults,
fmt=fmtAR,
header=headerAlignmentResults)
print('Alignment successful. See {} for results.'.format(outname))
def averageParallel(particleList,
averageName,
showProgressBar=False,
verbose=False,
createInfoVolumes=False,
weighting=None,
norm=False,
setParticleNodesRatio=3,
cores=6):
"""
compute average using parfor
@param particleList: The particles
@param averageName: Filename of new average
@param verbose: Prints particle information. Disabled by default.
@param createInfoVolumes: Create info data (wedge sum, inverted density) too? False by default.
@param weighting: weight particles by exp CC in average
@type weighting: bool
@param setParticleNodesRatio: minimum number of particles per node
@type setParticleNodesRatio: L{int}
@return: A new Reference object
@rtype: L{pytom.basic.structures.Reference}
@author: FF
"""
from pytom_volume import read, complexRealMult
from pytom.basic.fourier import fft, ifft
from pytom.basic.filter import lowpassFilter
from pytom.basic.structures import Reference
from pytom.alignment.alignmentFunctions import invert_WedgeSum
import os
splitLists = splitParticleList(particleList,
setParticleNodesRatio=setParticleNodesRatio,
numberOfNodes=cores)
splitFactor = len(splitLists)
avgNameList = []
preList = []
wedgeList = []
for ii in range(splitFactor):
avgName = averageName + '_dist' + str(ii) + '.em'
avgNameList.append(avgName)
preList.append(averageName + '_dist' + str(ii) + '-PreWedge.em')
wedgeList.append(averageName + '_dist' + str(ii) +
'-WedgeSumUnscaled.em')
#reference = average(particleList=plist, averageName=xxx, showProgressBar=True, verbose=False,
# createInfoVolumes=False, weighting=weighting, norm=False)
from multiprocessing import Process
procs = []
for i in range(splitFactor):
proc = Process(target=average,
args=(splitLists[i], avgNameList[i], showProgressBar,
verbose, createInfoVolumes, weighting, norm))
procs.append(proc)
proc.start()
import time
while procs:
procs = [proc for proc in procs if proc.is_alive()]
time.sleep(.1)
#averageList = mpi.parfor( average, list(zip(splitLists, avgNameList, [showProgressBar]*splitFactor,
# [verbose]*splitFactor, [createInfoVolumes]*splitFactor,
# [weighting]*splitFactor, [norm]*splitFactor)), verbose=True)
#collect results from files
unweiAv = read(preList[0])
wedgeSum = read(wedgeList[0])
os.system('rm ' + wedgeList[0])
os.system('rm ' + avgNameList[0])
os.system('rm ' + preList[0])
for ii in range(1, splitFactor):
av = read(preList[ii])
unweiAv += av
os.system('rm ' + preList[ii])
w = read(wedgeList[ii])
wedgeSum += w
os.system('rm ' + wedgeList[ii])
os.system('rm ' + avgNameList[ii])
if createInfoVolumes:
unweiAv.write(averageName[:len(averageName) - 3] + '-PreWedge.em')
wedgeSum.write(averageName[:len(averageName) - 3] +
'-WedgeSumUnscaled.em')
# convolute unweighted average with inverse of wedge sum
invert_WedgeSum(invol=wedgeSum,
r_max=unweiAv.sizeX() / 2 - 2.,
lowlimit=.05 * len(particleList),
lowval=.05 * len(particleList))
fResult = fft(unweiAv)
r = complexRealMult(fResult, wedgeSum)
unweiAv = ifft(r)
unweiAv.shiftscale(
0.0, 1 / float(unweiAv.sizeX() * unweiAv.sizeY() * unweiAv.sizeZ()))
# low pass filter to remove artifacts at fringes
unweiAv = lowpassFilter(volume=unweiAv,
band=unweiAv.sizeX() / 2 - 2,
smooth=(unweiAv.sizeX() / 2 - 1) / 10.)[0]
unweiAv.write(averageName)
return Reference(averageName, particleList)
def toProjectionStackFromAlignmentResultsFile(alignmentResultsFile,
weighting=None,
lowpassFilter=0.9,
binning=1,
circleFilter=False,
num_procs=1,
outdir='',
prefix='sorted_aligned'):
"""read image and create aligned projection stack, based on the results described in the alignmentResultFile.
@param alignmentResultsFile: result file generate by the alignment script.
@type datatypeAR: gui.guiFunction.datatypeAR
@param weighting: weighting (<0: analytical weighting, >1: exact weighting, 0/None: no weighting )
@type weighting: float
@param lowpassFilter: lowpass filter (in Nyquist)
@type lowpassFilter: float
@param binning: binning (default: 1 = no binning). binning=2: 2x2 pixels -> 1 pixel, binning=3: 3x3 pixels -> 1 pixel, etc.
@author: GvdS
"""
print('weighting: ', weighting)
import numpy
from pytom_numpy import vol2npy
from pytom.basic.files import read_em, write_em
from pytom.basic.functions import taper_edges
from pytom.basic.transformations import general_transform2d
from pytom.basic.fourier import ifft, fft
from pytom.basic.filter import filter as filterFunction, bandpassFilter
from pytom.basic.filter import circleFilter, rampFilter, exactFilter, fourierFilterShift, \
fourierFilterShift_ReducedComplex
from pytom_volume import complexRealMult, vol, paste
import pytom_freqweight
from pytom.basic.transformations import resize, rotate
from pytom.gui.guiFunctions import fmtAR, headerAlignmentResults, datatype, datatypeAR, loadstar
from pytom.reconstruction.reconstructionStructures import Projection, ProjectionList
from pytom_numpy import vol2npy
import mrcfile
from pytom.tompy.io import write, read_size
import os
print("Create aligned images from alignResults.txt")
alignmentResults = loadstar(alignmentResultsFile, dtype=datatypeAR)
imageList = alignmentResults['FileName']
tilt_angles = alignmentResults['TiltAngle']
imdim = int(read_size(imageList[0], 'x'))
if binning > 1:
imdim = int(float(imdim) / float(binning) + .5)
else:
imdim = imdim
sliceWidth = imdim
# pre-determine analytical weighting function and lowpass for speedup
if (weighting != None) and (float(weighting) < -0.001):
weightSlice = fourierFilterShift(rampFilter(imdim, imdim))
if circleFilter:
circleFilterRadius = imdim // 2
circleSlice = fourierFilterShift_ReducedComplex(
circleFilter(imdim, imdim, circleFilterRadius))
else:
circleSlice = vol(imdim, imdim // 2 + 1, 1)
circleSlice.setAll(1.0)
# design lowpass filter
if lowpassFilter:
if lowpassFilter > 1.:
lowpassFilter = 1.
print("Warning: lowpassFilter > 1 - set to 1 (=Nyquist)")
# weighting filter: arguments: (angle, cutoff radius, dimx, dimy,
lpf = pytom_freqweight.weight(0.0, lowpassFilter * imdim // 2, imdim,
imdim // 2 + 1, 1,
lowpassFilter / 5. * imdim)
# lpf = bandpassFilter(volume=vol(imdim, imdim,1),lowestFrequency=0,highestFrequency=int(lowpassFilter*imdim/2),
# bpf=None,smooth=lowpassFilter/5.*imdim,fourierOnly=False)[1]
projectionList = ProjectionList()
imageList = []
tilt_angles = []
for n, image in enumerate(alignmentResults['FileName']):
atx = alignmentResults['AlignmentTransX'][n]
aty = alignmentResults['AlignmentTransY'][n]
rot = alignmentResults['InPlaneRotation'][n]
mag = alignmentResults['Magnification'][n]
# print(image, alignmentResults['TiltAngle'][n])
# if abs(alignmentResults['TiltAngle'][n]) > 20:
# continue
tilt_angles.append(alignmentResults['TiltAngle'][n])
imageList.append(image)
projection = Projection(imageList[-1],
tiltAngle=tilt_angles[-1],
alignmentTransX=atx,
alignmentTransY=aty,
alignmentRotation=rot,
alignmentMagnification=mag)
projectionList.append(projection)
stack = vol(imdim, imdim, len(imageList))
stack.setAll(0.0)
phiStack = vol(1, 1, len(imageList))
phiStack.setAll(0.0)
thetaStack = vol(1, 1, len(imageList))
thetaStack.setAll(0.0)
offsetStack = vol(1, 2, len(imageList))
offsetStack.setAll(0.0)
for (ii, projection) in enumerate(projectionList):
if projection._filename.split('.')[-1] == 'st':
from pytom.basic.files import EMHeader, read
idx = projection._index
image = read(file=projection._filename,
subregion=[0, 0, idx - 1, imdim, imdim, 1],
sampling=[0, 0, 0],
binning=[0, 0, 0])
if not (binning == 1) or (binning == None):
image = resize(volume=image, factor=1 / float(binning))[0]
else:
# read projection files
from pytom.basic.files import EMHeader, read, read_em_header
image = read(str(projection._filename))
# image = rotate(image,180.,0.,0.)
image = resize(volume=image, factor=1 / float(binning))[0]
if lowpassFilter:
filtered = filterFunction(volume=image,
filterObject=lpf,
fourierOnly=False)
image = filtered[0]
tiltAngle = projection._tiltAngle
# normalize to contrast - subtract mean and norm to mean
immean = vol2npy(image).mean()
image = (image - immean) / immean
print(ii, immean, projection._filename)
# smoothen borders to prevent high contrast oscillations
image = taper_edges(image, imdim // 30)[0]
# transform projection according to tilt alignment
transX = projection._alignmentTransX / binning
transY = projection._alignmentTransY / binning
rot = float(projection._alignmentRotation)
mag = float(projection._alignmentMagnification)
image = general_transform2d(v=image,
rot=rot,
shift=[transX, transY],
scale=mag,
order=[2, 1, 0],
crop=True)
# smoothen once more to avoid edges
image = taper_edges(image, imdim // 30)[0]
# analytical weighting
if (weighting != None) and (weighting < 0):
# image = (ifft(complexRealMult(fft(image), w_func)) / (image.sizeX() * image.sizeY() * image.sizeZ()))
image = ifft(complexRealMult(
complexRealMult(fft(image), weightSlice), circleSlice),
scaling=True)
elif (weighting != None) and (weighting > 0):
weightSlice = fourierFilterShift(
exactFilter(tilt_angles, tiltAngle, imdim, imdim, sliceWidth))
# image = (ifft(complexRealMult(fft(image), w_func)) / (image.sizeX() * image.sizeY() * image.sizeZ()))
image = ifft(complexRealMult(
complexRealMult(fft(image), weightSlice), circleSlice),
scaling=True)
thetaStack(int(round(projection.getTiltAngle())), 0, 0, ii)
offsetStack(int(round(projection.getOffsetX())), 0, 0, ii)
offsetStack(int(round(projection.getOffsetY())), 0, 1, ii)
paste(image, stack, 0, 0, ii)
fname = '{}_{:02d}.mrc'.format(
prefix, int(imageList[ii].split('_')[-1].split('.')[0]))
if outdir:
import mrcfile
# write_em(os.path.join(outdir, fname.replace('mrc', 'em')), image)
write(os.path.join(outdir, fname),
vol2npy(image).copy().astype('float32'))
print('written file: ', fname)
return [stack, phiStack, thetaStack, offsetStack]