def __init__(self, maxSamples=3600, sleepTime=1.0):
self.maxSamples = maxSamples
self.sleepTime = sleepTime
self.cpuInfo = ne.cpu.info
self.cpuVirtualCores = len(self.cpuInfo)
self.cpuModel = self.cpuInfo[0]['model name']
self.cpuAdvertisedMHz = 1000.0 * np.float32(
self.cpuModel.split('@')[1].rstrip('GHz'))
self.cpuPowerManagement = self.cpuInfo[0]['power management']
self.cpuFlags = self.cpuInfo[0]['flags'].split(
) # Look for 'sse', 'sse2', 'avx', 'fma' for FFTW compilation
self.cpuCacheSize = np.float32(
self.cpuInfo[0]['cache size'][:-3]) # Should be kiloBytes
self.cpuCoresPerProcessor = np.int(ne.cpu.info[0]['cpu cores'])
self.cpuPhysicalCores = self.cpuCoresPerProcessor * countPhysicalProcessors(
)
if len(self.cpuInfo) == 2 * self.cpuPhysicalCores:
self.hyperthreading = True
else:
self.hyperthreading = False
self.zorroDefault = zorro.ImageRegistrator()
self.zorroDefault.diagWidth = 5
self.zorroDefault.CTFProgram = None
self.zorroDefault.filterMode = None
self.zorroDefault.doLazyFRC = False
self.zorroDefault.savePNG = False
self.resetStats()
def saveCtfImagesStar( self, outputName, zorroList = "*.dm4.log", physicalPixelSize=5.0, amplitudeContrast=0.08 ):
"""
Given a glob pattern, generate a list of zorro logs, or alternatively one can pass in a list. For each
zorro log, load it, extract the pertinant info (defocus, etc.). This is a file ready for particle
extraction, with imbedded Ctf information.
"""
import zorro
zorroList = glob.glob( zorroList )
headerDict = { b'MicrographName':1, b'CtfImage':2, b'DefocusU':3, b'DefocusV':4, b'DefocusAngle':5,
b'Voltage':6, b'SphericalAberration':7, b'AmplitudeContrast':8, b'Magnification':9,
b'DetectorPixelSize':10, b'CtfFigureOfMerit': 11 }
lookupDict = dict( zip( headerDict.values(), headerDict.keys() ) )
data = OrderedDict()
for header in headerDict:
data[header] = [None]*len(zorroList)
zorroReg = zorro.ImageRegistrator()
for J, zorroLog in enumerate(zorroList):
zorroReg.loadConfig( zorroLog, loadData=False )
data[b'MicrographName'][J] = zorroReg.files['sum']
data[b'CtfImage'][J] = os.path.splitext( zorroReg.files['sum'] )[0] + ".ctf:mrc"
# CTF4Results = [Micrograph number, DF1, DF2, Azimuth, Additional Phase shift, CC, max spacing fit-to]
data[b'DefocusU'][J] = zorroReg.CTF4Results[1]
data[b'DefocusV'][J] = zorroReg.CTF4Results[2]
data[b'DefocusAngle'][J] = zorroReg.CTF4Results[3]
data[b'CtfFigureOfMerit'][J] = zorroReg.CTF4Results[5]
data[b'Voltage'][J] = zorroReg.voltage
data[b'SphericalAberration'][J] = zorroReg.C3
data[b'AmplitudeContrast'][J] = amplitudeContrast
data[b'DetectorPixelSize'][J] = physicalPixelSize
data[b'Magnification'][J] = physicalPixelSize / (zorroReg.pixelsize * 1E-3)
with open( outputName, 'wb' ) as fh:
fh.write( b"\ndata_\n\nloop_\n")
for J in np.sort(lookupDict.keys()):
# print( "Column: " + "_rln" + lookupDict[J+1] + " #" + str(J+1) )
fh.write( b"_rln" + lookupDict[J] + b" #" + str(J) + b"\n")
lCnt = len( lookupDict )
for I in np.arange(0,len(zorroList)):
fh.write( b" ")
for J in np.arange(0,lCnt):
fh.write( str( data[lookupDict[J+1]][I] ) )
fh.write( b" " )
fh.write( b"\n" )
)
filenames = glob.glob(globPattern)
# Open the first one and grab the pixelsize
N = len(filenames)
CTFInfo = {}
CTFInfo['DefocusU'] = np.zeros([N], dtype='float32')
CTFInfo['DefocusV'] = np.zeros([N], dtype='float32')
CTFInfo['FinalResolution'] = np.zeros([N], dtype='float32')
CTFInfo['DefocusAngle'] = np.zeros([N], dtype='float32')
CTFInfo['CtfFigureOfMerit'] = np.zeros([N], dtype='float32')
# Better approach is probably to call this in batch mode. Then I get the all_micrographs_gctf.star file!
ctfReg = zorro.ImageRegistrator()
ctfReg.n_threads = 16
ctfReg.savePNG = True
ctfReg.files['figurePath'] = '../figs/'
ctfReg.plotDict["imageSum"] = False
ctfReg.plotDict["imageFirst"] = False
ctfReg.plotDict["FFTSum"] = False
ctfReg.plotDict["polarFFTSum"] = False
ctfReg.plotDict["corrTriMat"] = False
ctfReg.plotDict["shiftsTriMat"] = False
ctfReg.plotDict["peaksigTriMat"] = False
ctfReg.plotDict["errorTriMat"] = False
ctfReg.plotDict["translations"] = False
ctfReg.plotDict["pixRegError"] = False
ctfReg.plotDict["CTF4Diag"] = True
ctfReg.plotDict["logisticsCurve"] = False
hotpixMask = np.random.binomial(1, hotpixelRate, size=[N, N])
# NOT WHAT I WANT, need some variation frame-to-frame in hot pixel values...
hotpixImage = np.random.normal(loc=(ePerPixel +
hotpixelSigma * np.sqrt(ePerPixel)),
scale=hotpixelSigma * np.sqrt(ePerPixel),
size=hotpixMask.shape).astype('float32')
hotpixImage = np.clip(hotpixMask, 0, np.Inf)
hotpixImage *= (hotpixMask.astype('float32'))
ims(hotpixImage)
for J in xrange(0, M):
print("Applying hot pixel mask for image " + str(J))
noisyObject[J, :, :] = (noisyObject[J, :, :] * (~hotpixMask)) + hotpixImage
zorroReg = zorro.ImageRegistrator()
zorroReg.shapePadded = zorro.util.findValidFFTWDim([N * 1.1, N * 1.1])
zorroReg.images = noisyObject
zorroReg.stackName = 'Sim'
zorroReg.saveC = True
zorroReg.Bmode = 'opti'
zorroReg.triMode = 'diag'
zorroReg.weightMode = 'logistic'
zorroReg.peaksigThres = 5.5
zorroReg.diagWidth = 5
#zorroReg.Brad = 256
zorroReg.alignImageStack()
unblurReg = zorro.ImageRegistrator()
unblurReg.images = noisyObject
# -*- coding: utf-8 -*- """ Created on Mon Sep 7 10:25:34 2015 @author: Robert A. McLeod @email: [email protected] OR [email protected] """ import numpy as np import zorro import os, os.path, glob compress_ext = '.bz2' zorroDefault = zorro.ImageRegistrator() ##### IMPORTANT VARIABLES #### zorroDefault.diagWidth = 5 zorroDefault.n_threads = 16 zorroDefault.suppressOrigin = True zorroDefault.shapePadded = [4096, 4096] zorroDefault.doCompression = False zorroDefault.doDoseFilter = True zorroDefault.doLazyFRC = True zorroDefault.savePNG = True zorroDefault.doCTF = True ##### TYPICAL DEFAULT VARIABLES ##### zorroDefault.maxShift = 60 # This is only within diagWidth frames difference zorroDefault.weightMode = 'autologistic' zorroDefault.Bmode = 'opti' # zorroDefault.peaksigThres = 5.0 zorroDefault.subPixReg = 16 zorroDefault.triMode = 'diag' zorroDefault.shiftMethod = 'lanczos'
def main():
# Get command line arguments
from matplotlib import rc
rc('backend', qt4="PySide")
import sys, os
import zorro
import numpy as np
import time
# Usage:
# python `which zorro.py` -i Test.dm4 -c default.ini -o test.mrc
stackReg = zorro.ImageRegistrator()
configFile = None
inputFile = None
outputFile = None
try:
print("****Running Zorro-command-line on hostname: %s****" %
os.uname()[1])
except:
pass
for J in np.arange(0, len(sys.argv)):
# First argument is zorro.py
# Then we expect flag pairs
# sys.argv is a Python list
if sys.argv[J] == '-c':
configFile = sys.argv[J + 1]
J += 1
elif sys.argv[J] == '-i':
inputFile = sys.argv[J + 1]
J += 1
elif sys.argv[J] == '-o':
outputFile = sys.argv[J + 1]
J += 1
pass
if inputFile == None and configFile == None:
print("No input files, outputing template.zor")
stackReg.saveConfig(configNameIn="template.zor")
sys.exit()
if inputFile == None and not configFile == None:
stackReg.loadConfig(configNameIn=configFile, loadData=True)
stackReg.bench['total0'] = time.time()
if not inputFile == None and not configFile == None:
stackReg.loadConfig(configNameIn=configFile, loadData=False)
stackReg.bench['total0'] = time.time()
stackReg.files['stack'] = inputFile
stackReg.loadData()
if not outputFile == None:
stackReg.files['sum'] = outputFile
# Force use of 'Agg' for matplotlib. It's slower than Qt4Agg but doesn't crash on the cluster
stackReg.plotDict['backend'] = 'Agg'
if stackReg.triMode == 'refine':
# In the case of 'refine' we have to call 'diag' first if it hasn't already
# been performde.
if not bool(stackReg.errorDictList[-1]) and np.any(
stackReg.imageSum != None):
# Assume that
print(
"Zorro refine assuming that initial alignment has already been performed."
)
pass
else:
print("Zorro refine performing initial alignment.")
stackReg.triMode = 'diag'
stackReg.alignImageStack()
stackReg.loadData() # Only re-loads the stack
stackReg.triMode = 'refine'
# TODO: enable subZorro refinment.
stackReg.alignImageStack()
else:
# Execute the alignment as called for Zorro/UnBlur/etc.
stackReg.alignImageStack()
# Save everthing and do rounding/compression operations
stackReg.saveData() # Can be None
# Save plots
if stackReg.savePNG:
stackReg.plot()
stackReg.printProfileTimes()
stackReg.METAstatus = 'fini'
stackReg.saveConfig()
print("Zorro exiting")
sys.exit()
def orientGainRef(stackName,
gainRefName,
stackIsInAHole=True,
applyHotPixFilter=True,
doNoiseCorrelation=False,
relax=0.95,
n_threads=None):
"""
USAGE
Applies the gain reference over all possible orientations to determine which
is the best, quantitatively. Calculates the image standard deviation after
gain normalization (less is better) and the number of outlier pixels (less is
better) and the degree of correlated noise (less is better)
User should select a short flat-field stack (20 frames) in a hole. If no flat-field
image is available, pick a stack with no carbon or large amounts of ice, that
exhibited a large amount of drive.
"""
# ANALYSIS SCRIPT
zGain = zorro.ImageRegistrator()
zGain.loadData(gainRefName, target='sum')
zStack = zorro.ImageRegistrator()
zStack.loadData(stackName)
rawStack = np.copy(zStack.images)
#print( zGain.imageSum.shape )
#print( zStack.images.shape )
# Check if stack and gain is transposed
if zGain.imageSum.shape[0] == zGain.imageSum.shape[1]:
print("Square images")
orientList = [
[0, False, False],
[0, False, False],
[0, True, False],
[0, True, True],
[1, False, False],
[1, False, False],
[1, True, False],
[1, True, True],
]
elif (zGain.imageSum.shape[0] != zStack.images.shape[1]
and zGain.imageSum.shape[0] == zStack.images.shape[2]):
print("Rectangular image, rot90=1")
orientList = [
[1, False, False],
[1, False, False],
[1, True, False],
[1, True, True],
]
else:
print("Rectangular image, rot90=0")
orientList = [
[0, False, False],
[0, False, False],
[0, True, False],
[0, True, True],
]
# If the images are rectangular our life is easier as it cuts the number of
# possible orientations in half.
N = len(orientList)
stdArray = np.zeros(N)
outlierPixCount = np.zeros(N)
corrNoiseCoeff = np.zeros(N)
hotCutoff = np.zeros(N)
deadCutoff = np.zeros(N)
binnedSum = [None] * N
if bool(doNoiseCorrelation):
FFTMage = np.empty(zStack.images.shape[1:], dtype='complex64')
FFTConj = np.empty(zStack.images.shape[1:], dtype='complex64')
IFFTCorr = np.empty(zStack.images.shape[1:], dtype='complex64')
FFT2, IFFT2 = zorro.zorro_util.pyFFTWPlanner(FFTMage,
FFTConj,
n_threads=24)
normConst2 = np.float32(1.0 / np.size(FFTMage)**2)
for I, orient in enumerate(orientList):
gainRef = np.copy(zGain.imageSum)
if orient[0] > 0:
print("Rotating gain refernce by 90 degrees")
gainRef = np.rot90(gainRef, k=orient[0])
if orient[1] and orient[2]:
print("Rotating gain reference by 180 degrees")
gainRef = np.rot90(gainRef, k=2)
elif orient[1]:
print("Mirroring gain reference vertically")
gainRef = np.flipud(gainRef)
elif orient[2]:
print("Mirroring gain reference horizontally")
gainRef = np.fliplr(gainRef)
zStack.images = zStack.images * gainRef
if applyHotPixFilter:
zStack.hotpixInfo['relax'] = relax
zStack.hotpixFilter()
outlierPixCount[I] = zStack.hotpixInfo[
'guessDeadpix'] + zStack.hotpixInfo['guessHotpix']
hotCutoff[I] = zStack.hotpixInfo[u'cutoffUpper']
deadCutoff[I] = zStack.hotpixInfo[u'cutoffLower']
binnedSum[I] = util.squarekernel(np.sum(zStack.images, axis=0), k=3)
# zorro.zorro_plotting.ims( binnedSum[I] )
stdArray[I] = np.std(np.sum(zStack.images, axis=0))
if bool(stackIsInAHole) and bool(doNoiseCorrelation):
# Go through even-odd series
for J in np.arange(1, zStack.images.shape[0]):
print(
"(Orientation %d of %d) Compute Fourier correlation %d" %
(I, N, J))
if np.mod(J, 2) == 1:
FFT2.update_arrays(
zStack.images[J, :, :].astype('complex64'), FFTConj)
FFT2.execute()
else:
FFT2.update_arrays(
zStack.images[J, :, :].astype('complex64'), FFTMage)
FFT2.execute()
IFFT2.update_arrays(
nz.evaluate("normConst2*FFTMage*conj(FFTConj)"), IFFTCorr)
IFFT2.execute()
corrNoiseCoeff[I] += np.abs(IFFTCorr[0, 0])
elif bool(doNoiseCorrelation):
# Calculate phase correlations with a frame seperation of 6 frames to
# avoid signal correlation
frameSep = 6
for J in np.arange(0, zStack.images.shape[0] - frameSep):
print(
"(Orientation %d of %d) Compute Fourier correlation %d" %
(I, N, J))
FFT2.update_arrays(zStack.images[J, :, :].astype('complex64'),
FFTConj)
FFT2.execute()
FFT2.update_arrays(
zStack.images[J + frameSep, :, :].astype('complex64'),
FFTMage)
FFT2.execute()
IFFT2.update_arrays(
nz.evaluate("normConst2*FFTMage*conj(FFTConj)"), IFFTCorr)
IFFT2.execute()
corrNoiseCoeff[I] += np.real(IFFTCorr[0, 0])
pass
corrNoiseCoeff[I] /= normConst2
zStack.images = np.copy(rawStack)
#corrNoiseCoeff /= np.min( corrNoiseCoeff )
#stdArray /= np.min(stdArray)
bestIndex = np.argmin(stdArray)
nrows = 2
ncols = np.floor_divide(N + 1, 2)
plt.figure(figsize=(16, 9))
for I in np.arange(N):
plt.subplot(nrows * 100 + ncols * 10 + (I + 1))
clim = util.histClim(binnedSum[I], cutoff=1E-3)
plt.imshow(binnedSum[I],
cmap='gray',
norm=col.LogNorm(),
vmin=clim[0],
vmax=clim[1])
plt.axis('off')
if I == bestIndex:
textcolor = 'purple'
else:
textcolor = 'black'
title = "kRot: %d, VertFlip: %s, HorzFlip: %s \n" % (
orientList[I][0], orientList[I][1], orientList[I][2])
title += r"$\sigma: %.5g$" % stdArray[I]
if bool(applyHotPixFilter):
title += r"$, outliers: %d$" % outlierPixCount[I]
if bool(doNoiseCorrelation):
title += r"$, R_{noise}: %.5g$" % corrNoiseCoeff[I]
plt.title(title, fontdict={'fontsize': 14, 'color': textcolor})
plt.show(block=False)
return orientList[bestIndex]
def main():
# Get command line arguments
from matplotlib import rc
rc('backend', qt4="PySide")
import sys, os
import zorro
import numpy as np
import time
# Usage:
# python `which zorro.py` -i Test.dm4 -c default.ini -o test.mrc
stackReg = zorro.ImageRegistrator()
stackReg.bench['total0'] = time.time()
configFile = None
inputFile = None
outputFile = None
try:
print("****Running Zorro-command-line on hostname: %s****" %
os.uname()[1])
except:
pass
for J in np.arange(0, len(sys.argv)):
# First argument is zorro.py
# Then we expect flag pairs
# sys.argv is a Python list
if sys.argv[J] == '-c':
configFile = sys.argv[J + 1]
J += 1
elif sys.argv[J] == '-i':
inputFile = sys.argv[J + 1]
J += 1
elif sys.argv[J] == '-o':
outputFile = sys.argv[J + 1]
J += 1
pass
if inputFile == None and configFile == None:
print("No input files, outputing template.zor")
stackReg.saveConfig(configNameIn="template.zor")
sys.exit()
if inputFile == None and not configFile == None:
stackReg.loadConfig(configNameIn=configFile, loadData=True)
if not inputFile == None and not configFile == None:
stackReg.loadConfig(configNameIn=configFile, loadData=False)
stackReg.files['stack'] = inputFile
stackReg.loadData()
if not outputFile == None:
stackReg.files['sum'] = outputFile
# Force use of 'Agg' for matplotlib. It's slower than Qt4Agg but doesn't crash on the cluster
stackReg.plotDict['backend'] = 'Agg'
# Execute the alignment
stackReg.alignImageStack()
# Save everthing and do rounding/compression operations
stackReg.saveData() # Can be None
# Save plots
if stackReg.savePNG:
stackReg.plot()
stackReg.bench['total1'] = time.time()
stackReg.printProfileTimes()
stackReg.METAstatus = 'fini'
stackReg.saveConfig()
print("Zorro exiting")
sys.exit()