def __init__(self, ctrlNs, printQueue, rawDataRingBuf, fftDataRingBuf):
self.log = logging.getLogger("Main.FFTWorker")
logSetup.initLogging(printQ=printQueue)
self.log.info("FFT Worker Starting up")
self.ctrlNs = ctrlNs
self.printQueue = printQueue
self.rawDataRingBuf = rawDataRingBuf
self.fftDataRingBuf = fftDataRingBuf
self.fftChunkSize = s.FFT_CHUNK_SIZE
self.outputSize = self.fftChunkSize // 2 + 1
self.chunksPerAcq = int(SignalHound.rawSweepArrSize /
self.fftChunkSize)
self.overlap = s.FFT_OVERLAP
self.window = np.hamming(self.fftChunkSize)
inArr = pyfftw.n_byte_align_empty(self.fftChunkSize,
16,
dtype=np.float32)
outArr = pyfftw.n_byte_align_empty(self.outputSize,
16,
dtype=np.complex64)
self.log.info("Choosing maximally optimized transform")
self.fftFunc = pyfftw.FFTW(inArr,
outArr,
flags=('FFTW_PATIENT',
"FFTW_DESTROY_INPUT"))
self.log.info("Optimized transform selected. Run starting")
self.run()
def test_update_data_with_unaligned_original(self):
in_shape = self.input_shapes['2d']
out_shape = self.output_shapes['2d']
input_dtype_alignment = self.get_input_dtype_alignment()
axes=(-1,)
a, b = self.create_test_arrays(in_shape, out_shape)
# Offset from 16 byte aligned to guarantee it's not
# 16 byte aligned
a__ = n_byte_align_empty(
numpy.prod(in_shape)*a.itemsize + input_dtype_alignment,
16, dtype='int8')
a_ = a__[input_dtype_alignment:].view(dtype=self.input_dtype).reshape(*in_shape)
a_[:] = a
b__ = n_byte_align_empty(
numpy.prod(out_shape)*b.itemsize + input_dtype_alignment,
16, dtype='int8')
b_ = b__[input_dtype_alignment:].view(dtype=self.output_dtype).reshape(*out_shape)
b_[:] = b
fft, ifft = self.run_validate_fft(a_, b_, axes,
force_unaligned_data=True)
self.run_validate_fft(a, b_, axes, fft=fft, ifft=ifft)
self.run_validate_fft(a_, b, axes, fft=fft, ifft=ifft)
self.run_validate_fft(a_, b_, axes, fft=fft, ifft=ifft)
def pre_process(self):
in_pData = self.get_plugin_in_datasets()[0]
sino_shape = list(in_pData.get_shape())
width1 = sino_shape[1] + 2*self.pad
height1 = sino_shape[0] + 2*self.pad
v0 = np.abs(self.parameters['vvalue'])
u0 = np.abs(self.parameters['uvalue'])
n = np.abs(self.parameters['nvalue'])
# Create filter
centerx = np.ceil(width1/2.0)-1.0
centery = np.int16(np.ceil(height1/2.0)-1)
self.row1 = centery - v0
self.row2 = centery + v0+1
listx = np.arange(width1)-centerx
filtershape = 1.0/(1.0 + np.power(listx/u0, 2*n))
filtershapepad2d = np.zeros((self.row2 - self.row1, filtershape.size))
filtershapepad2d[:] = np.float64(filtershape)
self.filtercomplex = filtershapepad2d + filtershapepad2d*1j
a = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
b = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
c = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
d = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
self.fft_object = pyfftw.FFTW(a, b, axes=(0, 1))
self.ifft_object = pyfftw.FFTW(c, d, axes=(0, 1),
direction='FFTW_BACKWARD')
def __init__(self, nbeads, natoms):
"""Initializes nm_trans.
Args:
nbeads: The number of beads.
natoms: The number of atoms.
"""
self.nbeads = nbeads
self.natoms = natoms
try:
import pyfftw
info("Import of PyFFTW successful", verbosity.medium)
self.qdummy = pyfftw.n_byte_align_empty((nbeads, 3*natoms), 16, 'float32')
self.qnmdummy = pyfftw.n_byte_align_empty((nbeads//2+1, 3*natoms), 16, 'complex64')
self.fft = pyfftw.FFTW(self.qdummy, self.qnmdummy, axes=(0,), direction='FFTW_FORWARD')
self.ifft = pyfftw.FFTW(self.qnmdummy, self.qdummy, axes=(0,), direction='FFTW_BACKWARD')
except ImportError: #Uses standard numpy fft library if nothing better
#is available
info("Import of PyFFTW unsuccessful, using NumPy library instead", verbosity.medium)
self.qdummy = np.zeros((nbeads,3*natoms), dtype='float32')
self.qnmdummy = np.zeros((nbeads//2+1,3*natoms), dtype='complex64')
def dummy_fft(self):
self.qnmdummy = np.fft.rfft(self.qdummy, axis=0)
def dummy_ifft(self):
self.qdummy = np.fft.irfft(self.qnmdummy, n=self.nbeads, axis=0)
self.fft = lambda: dummy_fft(self)
self.ifft = lambda: dummy_ifft(self)
def test_call_with_normalisation_precision(self):
'''The normalisation should use a double precision scaling.
'''
# Should be the case for double inputs...
_input_array = n_byte_align_empty((256, 512), 16,
dtype='complex128')
ifft = FFTW(self.output_array, _input_array,
direction='FFTW_BACKWARD')
ref_output = ifft(normalise_idft=False).copy()/numpy.float64(ifft.N)
test_output = ifft(normalise_idft=True).copy()
self.assertTrue(numpy.alltrue(ref_output == test_output))
# ... and single inputs.
_input_array = n_byte_align_empty((256, 512), 16,
dtype='complex64')
ifft = FFTW(numpy.array(self.output_array, _input_array.dtype),
_input_array,
direction='FFTW_BACKWARD')
ref_output = ifft(normalise_idft=False).copy()/numpy.float64(ifft.N)
test_output = ifft(normalise_idft=True).copy()
self.assertTrue(numpy.alltrue(ref_output == test_output))
def ShiftConvNumbaFFT(h, N, M, xdtype=np.complex_, powerof2=True):
# implements Doppler filter:
# y[n, p] = SUM_k (exp(2*pi*j*n*(k - (L-1))/N) * h[k]) * x[p - k]
# = SUM_k (exp(-2*pi*j*n*k/N) * s*[k]) * x[p - (L-1) + k]
L = len(h)
outlen = M + L - 1
nfft = outlen
if powerof2:
nfft = pow2(nfft)
dopplermat = np.exp(2*np.pi*1j*np.arange(N)[:, np.newaxis]*(np.arange(L) - (L - 1))/N)
dopplermat.astype(np.result_type(h.dtype, np.complex64)) # cast to complex type with precision of h
hbank = h*dopplermat
# speed not critical here, just use numpy fft
hbankpad = zero_pad(hbank, nfft)
H = np.fft.fft(hbankpad) / nfft # divide by nfft b/c FFTW's ifft does not do this
xcdtype = np.result_type(xdtype, np.complex64) # cast to complex type with precision of x
xpad = pyfftw.n_byte_align(np.zeros(nfft, xcdtype), 16)
X = pyfftw.n_byte_align(np.zeros(nfft, xcdtype), 16)
xfft = pyfftw.FFTW(xpad, X, threads=_THREADS)
ydtype = np.result_type(H.dtype, xcdtype)
Y = pyfftw.n_byte_align_empty(H.shape, 16, ydtype)
y = pyfftw.n_byte_align_empty(H.shape, 16, ydtype)
ifft = pyfftw.FFTW(Y, y, direction='FFTW_BACKWARD', threads=_THREADS)
xtype = numba.__getattribute__(str(np.dtype(xdtype)))
#htype = numba.__getattribute__(str(H.dtype))
#xctype = numba.__getattribute__(str(X.dtype))
#ytype = numba.__getattribute__(str(Y.dtype))
#@jit(argtypes=[htype[:, ::1], xctype[::1], ytype[:, ::1], xtype[::1]])
#def fun(H, X, Y, x):
#xpad[:M] = x
#xfft.execute() # input is xpad, output is X
#Y[:, :] = H*X # need expression optimized by numba but that writes into Y
#ifft.execute() # input is Y, output is y
#yc = np.array(y)[:, :outlen] # need a copy, which np.array provides
#return yc
#@dopplerbank_dec(h, N, M, nfft=nfft, H=H)
#def shiftconv_numba_fft(x):
#return fun(H, X, Y, x)
#@jit(argtypes=[xtype[::1]])
@jit
def shiftconv_numba_fft(x):
xpad[:M] = x
xfft.execute() # input is xpad, output is X
Y[:, :] = X*H # need expression optimized by numba but that writes into Y
ifft.execute() # input is Y, output is y
yc = np.array(y[:, :outlen]) # need a copy, which np.array provides
return yc
shiftconv_numba_fft = dopplerbank_dec(h, N, M, nfft=nfft, H=H)(shiftconv_numba_fft)
return shiftconv_numba_fft
def test_flags(self):
'''Test to see if the flags are correct
'''
fft = FFTW(self.input_array, self.output_array)
self.assertEqual(fft.flags, ('FFTW_MEASURE',))
fft = FFTW(self.input_array, self.output_array,
flags=('FFTW_DESTROY_INPUT', 'FFTW_UNALIGNED'))
self.assertEqual(fft.flags, ('FFTW_DESTROY_INPUT', 'FFTW_UNALIGNED'))
# Test an implicit flag
_input_array = n_byte_align_empty(256, 16, dtype='complex64')
_output_array = n_byte_align_empty(256, 16, dtype='complex64')
# These are guaranteed to be misaligned (due to dtype size == 8)
input_array = _input_array[:-1]
output_array = _output_array[:-1]
u_input_array = _input_array[1:]
u_output_array = _output_array[1:]
fft = FFTW(input_array, u_output_array)
self.assertEqual(fft.flags, ('FFTW_MEASURE', 'FFTW_UNALIGNED'))
fft = FFTW(u_input_array, output_array)
self.assertEqual(fft.flags, ('FFTW_MEASURE', 'FFTW_UNALIGNED'))
fft = FFTW(u_input_array, u_output_array)
self.assertEqual(fft.flags, ('FFTW_MEASURE', 'FFTW_UNALIGNED'))
def generate_wisdom(self):
for each_dtype in (numpy.complex128, numpy.complex64,
numpy.clongdouble):
a = n_byte_align_empty((1,1024), 16, each_dtype)
b = n_byte_align_empty(a.shape, 16, dtype=a.dtype)
fft = FFTW(a,b)
def dfm3(input, angles, Npad):
input = np.double(input)
(Nx, Ny, Nproj) = input.shape
angles = np.double(angles)
pad_pre = int(np.ceil((Npad - Ny) / 2.0))
pad_post = int(np.floor((Npad - Ny) / 2.0))
# Initialization
Nz = Ny // 2 + 1
w = np.zeros((Nx, Ny, Nz)) #store weighting factors
v = pyfftw.n_byte_align_empty((Nx, Ny, Nz), 16, dtype='complex128')
v = np.zeros(v.shape) + 1j * np.zeros(v.shape)
recon = pyfftw.n_byte_align_empty((Nx, Ny, Ny), 16, dtype='float64')
recon_fftw_object = pyfftw.FFTW(v,
recon,
direction='FFTW_BACKWARD',
axes=(0, 1, 2))
p = pyfftw.n_byte_align_empty((Nx, Npad), 16, dtype='float64')
pF = pyfftw.n_byte_align_empty((Nx, Npad // 2 + 1), 16, dtype='complex128')
p_fftw_object = pyfftw.FFTW(p, pF, axes=(0, 1))
dk = np.double(Ny) / np.double(Npad)
for a in range(0, Nproj):
ang = angles[a] * np.pi / 180
projection = input[:, :, a] #2D projection image
p = np.lib.pad(projection, ((0, 0), (pad_pre, pad_post)),
'constant',
constant_values=(0, 0)) #pad zeros
p = np.fft.ifftshift(p)
p_fftw_object.update_arrays(p, pF)
p_fftw_object()
probjection_f = pF.copy()
if ang < 0:
probjection_f = np.conj(pF.copy())
probjection_f[1:, :] = np.flipud(probjection_f[1:, :])
ang = np.pi + ang
# Bilinear extrapolation
for i in range(0, np.int(np.ceil(Npad / 2)) + 1):
ky = i * dk
#kz = 0
ky_new = np.cos(ang) * ky #new coord. after rotation
kz_new = np.sin(ang) * ky
sy = abs(np.floor(ky_new) - ky_new) #calculate weights
sz = abs(np.floor(kz_new) - kz_new)
for b in range(1, 5): #bilinear extrapolation
pz, py, weight = bilinear(kz_new, ky_new, sz, sy, Ny, b)
if (py >= 0 and py < Ny and pz >= 0 and pz < Nz / 2 + 1):
w[:, py, pz] = w[:, py, pz] + weight
v[:, py, pz] = v[:, py, pz] + weight * probjection_f[:, i]
v[w != 0] = v[w != 0] / w[w != 0]
recon_F = v.copy()
recon_fftw_object.update_arrays(v, recon)
recon_fftw_object()
recon = np.fft.fftshift(recon)
return (recon, recon_F)
def radial_average(tiltseries,kr_cutoffs):
(Nx,Ny,Nproj) = tiltseries.shape
f = pyfftw.n_byte_align_empty((Nx,Ny/2+1),16,dtype='complex128')
r = pyfftw.n_byte_align_empty((Nx,Ny),16,dtype='float64')
p_fftw_object = pyfftw.FFTW(r,f,axes=(0,1))
Ir = np.zeros(kr_cutoffs.size); I = np.zeros(kr_cutoffs.size)
kx = np.fft.fftfreq(Nx)
ky = np.fft.fftfreq(Ny)
ky = ky[0:int(np.ceil(Ny/2)+1)]
kX,kY = np.meshgrid(ky,kx)
kR = np.sqrt(kY**2+kX**2)
for a in range(0,Nproj):
r = tiltseries[:,:,a].copy().astype('float64')
p_fftw_object.update_arrays(r,f); p_fftw_object.execute()
shell = kR<=kr_cutoffs[0]
I[0] = np.sum(np.absolute(f[shell]))
I[0] = I[0]/np.sum(shell)
for j in range(1,kr_cutoffs.size):
shell = np.logical_and(kR>kr_cutoffs[j-1],kR<=kr_cutoffs[j])
I[j] = np.sum(np.absolute(f[shell]))
I[j] = I[j]/np.sum(shell)
Ir = Ir + I
Ir = Ir/Nproj
return Ir
def __init__(self, ctrlNs, printQueue, rawDataRingBuf, fftDataRingBuf):
self.log = logging.getLogger("Main.FFTWorker")
logSetup.initLogging(printQ = printQueue)
self.log.info("FFT Worker Starting up")
self.ctrlNs = ctrlNs
self.printQueue = printQueue
self.rawDataRingBuf = rawDataRingBuf
self.fftDataRingBuf = fftDataRingBuf
self.fftChunkSize = s.FFT_CHUNK_SIZE
self.outputSize = self.fftChunkSize//2 + 1
self.chunksPerAcq = int(SignalHound.rawSweepArrSize/self.fftChunkSize)
self.overlap = s.FFT_OVERLAP
self.window = np.hamming(self.fftChunkSize)
inArr = pyfftw.n_byte_align_empty(self.fftChunkSize, 16, dtype=np.float32)
outArr = pyfftw.n_byte_align_empty(self.outputSize, 16, dtype=np.complex64)
self.log.info("Choosing maximally optimized transform")
self.fftFunc = pyfftw.FFTW(inArr, outArr, flags=('FFTW_PATIENT', "FFTW_DESTROY_INPUT"))
self.log.info("Optimized transform selected. Run starting")
self.run()
def radial_average(tiltseries, kr_cutoffs):
(Nx, Ny, Nproj) = tiltseries.shape
f = pyfftw.n_byte_align_empty((Nx, Ny // 2 + 1), 16, dtype='complex128')
r = pyfftw.n_byte_align_empty((Nx, Ny), 16, dtype='float64')
p_fftw_object = pyfftw.FFTW(r, f, axes=(0, 1))
Ir = np.zeros(kr_cutoffs.size)
I = np.zeros(kr_cutoffs.size)
kx = np.fft.fftfreq(Nx)
ky = np.fft.fftfreq(Ny)
ky = ky[0:int(np.ceil(Ny / 2) + 1)]
kX, kY = np.meshgrid(ky, kx)
kR = np.sqrt(kY**2 + kX**2)
for a in range(0, Nproj):
r = tiltseries[:, :, a].copy().astype('float64')
p_fftw_object.update_arrays(r, f)
p_fftw_object.execute()
shell = kR <= kr_cutoffs[0]
I[0] = np.sum(np.absolute(f[shell]))
I[0] = I[0] / np.sum(shell)
for j in range(1, kr_cutoffs.size):
shell = np.logical_and(kR > kr_cutoffs[j - 1], kR <= kr_cutoffs[j])
I[j] = np.sum(np.absolute(f[shell]))
I[j] = I[j] / np.sum(shell)
Ir = Ir + I
Ir = Ir / Nproj
return Ir
def dfm3(input,angles,Npad):
# input: aligned data
# angles: projection angles
# N_pad: size of padded projection.
input = np.double(input)
(Nx,Ny,Nproj) = input.shape
angles = np.double(angles)
cen = np.floor(Ny/2.0)
cen_pad = np.floor(Npad/2.0)
pad_pre = np.ceil((Npad-Ny)/2.0); pad_post = np.floor((Npad-Ny)/2.0)
# Initialization
Nz = Ny
w = np.zeros((Nx,Ny,np.int(Nz/2+1))) #store weighting factors
v = pyfftw.n_byte_align_empty((Nx,Ny,Nz/2+1),16,dtype='complex128')
v = np.zeros(v.shape) + 1j*np.zeros(v.shape)
recon = pyfftw.n_byte_align_empty((Nx,Ny,Nz),16,dtype='float64')
recon_fftw_object = pyfftw.FFTW(v,recon,direction='FFTW_BACKWARD',axes=(0,1,2))
p = pyfftw.n_byte_align_empty((Nx,Npad),16,dtype='float64')
pF = pyfftw.n_byte_align_empty((Nx,Npad/2+1),16,dtype='complex128')
p_fftw_object = pyfftw.FFTW(p,pF,axes=(0,1))
dk = np.double(Ny)/np.double(Npad)
for a in range(0, Nproj):
#print angles[a]
ang = angles[a]*np.pi/180
projection = input[:,:,a] #2D projection image
p = np.lib.pad(projection,((0,0),(pad_pre,pad_post)),'constant',constant_values=(0,0)) #pad zeros
p = np.fft.ifftshift(p)
p_fftw_object.update_arrays(p,pF)
p_fftw_object()
probjection_f = pF.copy()
if ang<0:
probjection_f = np.conj(pF.copy())
probjection_f[1:,:] = np.flipud(probjection_f[1:,:])
ang = np.pi+ang
# Bilinear extrapolation
for i in range(0, np.int(np.ceil(Npad/2))+1):
ky = i*dk; #kz = 0;
ky_new = np.cos(ang)*ky #new coord. after rotation
kz_new = np.sin(ang)*ky
sy = abs(np.floor(ky_new) - ky_new) #calculate weights
sz = abs(np.floor(kz_new) - kz_new)
for b in range(1,5): #bilinear extrapolation
pz,py,weight = bilinear(kz_new,ky_new,sz,sy,Ny,b)
if (py>=0 and py<Ny and pz>=0 and pz<Nz/2+1):
w[:,py,pz] = w[:,py,pz] + weight
v[:,py,pz] = v[:,py,pz] + weight * probjection_f[:,i]
v[w!=0] = v[w!=0]/w[w!=0]
recon_F = v.copy()
recon_fftw_object.update_arrays(v,recon)
recon_fftw_object()
recon = np.fft.fftshift(recon)
return recon.astype(np.float32)
def pre_process(self):
in_pData = self.get_plugin_in_datasets()[0]
sino_shape = list(in_pData.get_shape())
width1 = sino_shape[1] + 2 * self.pad
height1 = sino_shape[0] + 2 * self.pad
v0 = np.abs(self.parameters['vvalue'])
u0 = np.abs(self.parameters['uvalue'])
n = np.abs(self.parameters['nvalue'])
# Create filter
centerx = np.ceil(width1 / 2.0) - 1.0
centery = np.int16(np.ceil(height1 / 2.0) - 1)
self.row1 = centery - v0
self.row2 = centery + v0 + 1
listx = np.arange(width1) - centerx
filtershape = 1.0 / (1.0 + np.power(listx / u0, 2 * n))
filtershapepad2d = np.zeros((self.row2 - self.row1, filtershape.size))
filtershapepad2d[:] = np.float64(filtershape)
self.filtercomplex = filtershapepad2d + filtershapepad2d * 1j
a = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
b = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
c = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
d = pyfftw.n_byte_align_empty((height1, width1), 16, 'complex128')
self.fft_object = pyfftw.FFTW(a, b, axes=(0, 1))
self.ifft_object = pyfftw.FFTW(c,
d,
axes=(0, 1),
direction='FFTW_BACKWARD')
def __init__(self,
n = 1364,
N = 2048,
iter0 = 138000,
nfiles = 64,
fft_threads = 32,
out_threads = 4,
src_dir = './',
dst_dir = './',
src_format = 'K{0:0>6}QNP{1:0>3}',
dst_format = 'RMHD_{0}_t{1:0>4x}_z{2:0>7x}'):
self.src_format = src_format
self.dst_format = dst_format
self.src_dir = src_dir
self.dst_dir = dst_dir
self.n = n
self.N = N
self.iter0 = iter0
self.nfiles = nfiles
self.kdata = pyfftw.n_byte_align_empty(
(self.N//2+1, 2, self.N, self.N),
pyfftw.simd_alignment,
dtype = np.complex64)
self.rrdata = pyfftw.n_byte_align_empty(
(self.N, self.N, self.N, 2),
pyfftw.simd_alignment,
dtype = np.float32)
self.rzdata = pyfftw.n_byte_align_empty(
((self.N//8) * (self.N//8) * (self.N//8),
8*8*8*2),
pyfftw.simd_alignment,
dtype = np.float32)
if type(self.dst_dir) == type([]):
self.zdir = np.array(
range(0,
self.rzdata.shape[0],
self.rzdata.shape[0] // len(self.dst_dir)))
self.cubbies_per_file = self.rzdata.shape[0] // self.nfiles
if (os.path.isfile('fftw_wisdom.pickle.gz')):
pyfftw.import_wisdom(
pickle.load(gzip.open('fftw_wisdom.pickle.gz', 'rb')))
print('about to initialize the fftw plan, which can take a while')
self.plan = pyfftw.FFTW(
self.kdata.transpose(3, 2, 0, 1), self.rrdata,
axes = (0, 1, 2),
direction = 'FFTW_BACKWARD',
flags = ('FFTW_MEASURE',
'FFTW_DESTROY_INPUT'),
threads = fft_threads)
print('finalized fftw initialization')
bla = pyfftw.export_wisdom()
pickle.dump(bla, gzip.open('fftw_wisdom.pickle.gz', 'wb'))
self.fft_threads = fft_threads
self.out_threads = out_threads
self.shuffle_lib = np.ctypeslib.load_library(
'libzshuffle.so',
os.path.abspath(os.path.join(
os.path.expanduser('~'), 'repos/RMHD_converter/C-shuffle')))
return None
def __init__(self,npa,dt,decimation,window=None):
n = len(npa)
print("Length n is",n)
fftpg = 10*np.log10(n)
print("FFT Processing Gain {0:.1f} dB".format(fftpg))
filterpg = 10*np.log10(decimation)
print("Filter Processing Gain {0:.1f} dB".format(filterpg))
totalpg=fftpg+filterpg
print("Total Processing Gain {0:.1f} dB".format(totalpg))
fftia = pyfftw.n_byte_align_empty(n, 16, 'complex64')
fftoa = pyfftw.n_byte_align_empty(n, 16, 'complex64')
fft = pyfftw.FFTW(fftia,fftoa) ##,flags=('FFTW_ESTIMATE',),planning_timelimit=60.0)
ra = np.real(npa)
ia = np.imag(npa)
bitsused = int(np.ceil(np.log2(2*np.max([np.max(ra),np.abs(np.min(ra)),np.max(ia),np.abs(np.min(ia))]))))
print("Input bits in use",bitsused)
#print("Real max={0:.1f} min={1:.1f}".format(np.max(ra),np.min(ra)))
#print("Imag max={0:.1f} min={1:.1f}".format(np.max(ia),np.min(ia)))
if window:
w = window(n)
fftia[:] = w * npa
else:
fftia[:] = npa
fft()
self.sa = np.abs(fftoa)
## Scale amplitude for window
if window:
scale = 1.0/np.sum(window(10000)/10000.0)
print("Scaling postwindow by",scale)
self.sa = scale * self.sa
self.sa = np.concatenate( [self.sa[int(n/2):n],self.sa[0:int(n/2)]] )
## Result is vrms
##print("Converting to dBFS")
maxv = self.sa.max()
self.sa = 20.0*np.log10(self.sa/maxv)
w = int(n/4)
print("Quarter noise floor means: {0:.1f} {1:.1f} {2:.1f} {3:.1f}".format(
np.mean(self.sa[:w]),np.mean(self.sa[w:2*w]),np.mean(self.sa[2*w:3*w]),np.mean(self.sa[3*w:])))
self.mhz2bin = len(self.sa) * 1e6 * dt * decimation
self.bin2mhz = 1.0/self.mhz2bin
print("Spectrum Array length is",len(self.sa))
def init_pyfftw(N):
a = pyfftw.n_byte_align_empty(N, simd_alignment, 'complex128')
b = pyfftw.n_byte_align_empty(N, simd_alignment, 'complex128')
c = pyfftw.n_byte_align_empty(N, simd_alignment, 'complex128')
fft = pyfftw.FFTW(a, b, threads=2, effort=effort[0])
ifft = pyfftw.FFTW(b, c, direction='FFTW_BACKWARD', threads=2,
effort=effort[0])
return fft, ifft
def __init__(self,npa,dt,window=None):
n = len(npa)
self.complexinput = npa.dtype == 'complex64'
if self.complexinput:
fftia = pyfftw.n_byte_align_empty(n, 16, 'complex64')
fftoa = pyfftw.n_byte_align_empty(n, 16, 'complex64')
else:
fftia = pyfftw.n_byte_align_empty(n, 16, 'float32')
fftoa = pyfftw.n_byte_align_empty(int(n/2) + 1, 16, 'complex64')
fft = pyfftw.FFTW(fftia,fftoa,flags=('FFTW_ESTIMATE',),planning_timelimit=60.0)
maxv = abs(npa).max()
print("Max value is",maxv)
if window:
w = window(n)
fftia[:] = w * (npa/maxv)
else:
fftia[:] = npa/maxv
fft()
self.sa = np.abs(fftoa)
## Scale amplitude for window
if window:
scale = 1.0/np.sum(window(10000)/10000.0)
print("Scaling postwindow by",scale)
self.sa = scale * self.sa
if self.complexinput:
self.sa = np.concatenate( [self.sa[int(n/2):n],self.sa[0:int(n/2)]] )
else:
## 2.0 To get magnitude in terms of original V since half of spectrum is returned
self.sa = 2* self.sa
## Result is vrms
print("Converting to dBFS")
maxv = self.sa.max()
print("Max is",maxv)
print(self.sa)
self.sa = 20.0*np.log10(self.sa/maxv)
if self.complexinput:
self.mhz2bin = len(self.sa) * 1e6 * dt
else:
self.mhz2bin = len(self.sa) * 1e6 * 2 * dt
self.bin2mhz = 1.0/self.mhz2bin
print("Spectrum Array length is",len(self.sa))
def __init__(self,gridsize):
self.gridsize = gridsize
self.a = pyfftw.n_byte_align_empty(gridsize,16,'complex128')
self.b = pyfftw.n_byte_align_empty(gridsize,16,'complex128')
self.fftw = pyfftw.FFTW(self.a,self.b,direction='FFTW_FORWARD')
self.c = pyfftw.n_byte_align_empty(gridsize,16,'complex128')
self.d = pyfftw.n_byte_align_empty(gridsize,16,'complex128')
self.ifftw = pyfftw.FFTW(self.c,self.d,direction='FFTW_BACKWARD')
def seqDatab_to_four(seqList,fftPlan,inverseVariable,method=1,n=0): four_seq_datab = []; for i in range(len(seqList)): if n==0: four_seq_datab.append(pyfftw.n_byte_align_empty(len(seqList[i]),1,'complex')); else: four_seq_datab.append(pyfftw.n_byte_align_empty(n,1,'complex')); four_seq_datab[i][:] = seq_to_four(seqList[i],fftPlan,inverseVariable,method,n) return four_seq_datab
def do_it(params):
DIROUT = params[0]
FILEINs = params[1]
t_now = params[2]
NW = params[3]
K = params[4]
PVAL = params[5]
FS = params[6]
NFFT = params[7]
NWINDOWS_PER_WORKER = params[8]
tapers = params[9]
worker = params[10]
t_offset_tic = params[11]
FILE_TYPE = params[12]
FILE_LEN_TIC = params[13]
NCHANNELS = len(FILEINs)
sig=stats.f.ppf((1-PVAL/NFFT),2,2*K-2)
fft_in = pyfftw.n_byte_align_empty((NFFT,K,NCHANNELS), 16, 'float32')
fft_out = pyfftw.n_byte_align_empty((NFFT//2+1,K,NCHANNELS), 16, 'complex64')
fft_plan = pyfftw.FFTW(fft_in, fft_out, axes=(0,), flags=('FFTW_PATIENT',))
fft_in2 = pyfftw.n_byte_align_empty(NFFT, 16, 'float32')
fft_out2 = pyfftw.n_byte_align_empty(NFFT//2+1, 16, 'complex64')
fft_plan2 = pyfftw.FFTW(fft_in2, fft_out2, flags=('FFTW_PATIENT',))
NSAMPLES = NFFT//2*(NWINDOWS_PER_WORKER+1)
dd = np.empty([NSAMPLES, NCHANNELS], dtype=np.float32)
for i in range(0,NCHANNELS):
tmp=np.empty(0)
tmp2=(t_now+worker*NWINDOWS_PER_WORKER)*(NFFT//2)+t_offset_tic
filei=os.path.join(DIROUT,FILEINs[i])
if tmp2<FILE_LEN_TIC:
if FILE_TYPE==1:
fid=open(filei,'rb')
fid.seek(tmp2*4,0);
tmp = fid.read(NSAMPLES*4)
fid.close()
tmp = np.array(struct.unpack(str(int(len(tmp)/4))+'f', tmp), dtype=np.float32)
if FILE_TYPE==2:
rate, fid = wavfile.read(filei,mmap=True)
tmp=fid[(tmp2+1):min(tmp2+NSAMPLES, FILE_LEN_TIC)]
if len(tmp) < NSAMPLES:
tmp = np.concatenate((tmp, np.zeros(NSAMPLES-len(tmp), dtype=np.float32)))
dd[:,i] = tmp
idx=list()
for j in range(0,NWINDOWS_PER_WORKER):
ddd=dd[np.array(range(0,NFFT))+j*NFFT//2,:]
#F, A, f, sig, sd = ftest(ddd, tapers, FS, PVAL, fft_in, fft_out, fft_plan)
F = ftest(ddd, tapers, PVAL, fft_in, fft_out, fft_plan)
for l in range(0,NCHANNELS):
tmp=[i+1 for (i,v) in enumerate(F[1:-1,l]) if v>sig]
for m in range(0,len(tmp)):
freq,amp = brown_puckette(ddd[:,l],tmp[m],FS, fft_in2, fft_out2, fft_plan2)
idx.append((j+worker*NWINDOWS_PER_WORKER, freq, amp, l))
return idx
def __init__(self,
n=1364,
N=2048,
iter0=138000,
nfiles=64,
fft_threads=32,
out_threads=4,
src_dir='./',
dst_dir='./',
src_format='K{0:0>6}QNP{1:0>3}',
dst_format='RMHD_{0}_t{1:0>4x}_z{2:0>7x}'):
self.src_format = src_format
self.dst_format = dst_format
self.src_dir = src_dir
self.dst_dir = dst_dir
self.n = n
self.N = N
self.iter0 = iter0
self.nfiles = nfiles
self.kdata = pyfftw.n_byte_align_empty(
(self.N // 2 + 1, 2, self.N, self.N),
pyfftw.simd_alignment,
dtype=np.complex64)
self.rrdata = pyfftw.n_byte_align_empty((self.N, self.N, self.N, 2),
pyfftw.simd_alignment,
dtype=np.float32)
self.rzdata = pyfftw.n_byte_align_empty(
((self.N // 8) * (self.N // 8) * (self.N // 8), 8 * 8 * 8 * 2),
pyfftw.simd_alignment,
dtype=np.float32)
if type(self.dst_dir) == type([]):
self.zdir = np.array(
range(0, self.rzdata.shape[0],
self.rzdata.shape[0] // len(self.dst_dir)))
self.cubbies_per_file = self.rzdata.shape[0] // self.nfiles
if (os.path.isfile('fftw_wisdom.pickle.gz')):
pyfftw.import_wisdom(
pickle.load(gzip.open('fftw_wisdom.pickle.gz', 'rb')))
print('about to initialize the fftw plan, which can take a while')
self.plan = pyfftw.FFTW(self.kdata.transpose(3, 2, 0, 1),
self.rrdata,
axes=(0, 1, 2),
direction='FFTW_BACKWARD',
flags=('FFTW_MEASURE', 'FFTW_DESTROY_INPUT'),
threads=fft_threads)
print('finalized fftw initialization')
bla = pyfftw.export_wisdom()
pickle.dump(bla, gzip.open('fftw_wisdom.pickle.gz', 'wb'))
self.fft_threads = fft_threads
self.out_threads = out_threads
self.shuffle_lib = np.ctypeslib.load_library(
'libzshuffle.so',
os.path.abspath(
os.path.join(os.path.expanduser('~'),
'repos/RMHD_converter/C-shuffle')))
return None
def test_is_n_byte_aligned(self):
a = n_byte_align_empty(100, 16)
self.assertTrue(is_n_byte_aligned(a, 16))
a = n_byte_align_empty(100, 5)
self.assertTrue(is_n_byte_aligned(a, 5))
a = n_byte_align_empty(100, 16, dtype="float32")[1:]
self.assertFalse(is_n_byte_aligned(a, 16))
self.assertTrue(is_n_byte_aligned(a, 4))
def test_is_n_byte_aligned(self):
a = n_byte_align_empty(100, 16)
self.assertTrue(is_n_byte_aligned(a, 16))
a = n_byte_align_empty(100, 5)
self.assertTrue(is_n_byte_aligned(a, 5))
a = n_byte_align_empty(100, 16, dtype='float32')[1:]
self.assertFalse(is_n_byte_aligned(a, 16))
self.assertTrue(is_n_byte_aligned(a, 4))
def __init__(self, shape, dtype, threads=1, inverse=False):
self.threads = threads
if inverse:
self.arr_in = pyfftw.n_byte_align_empty(shape[0], pyfftw.simd_alignment, dtype=dtype)
self.fftw = pyfftw.builders.irfft2(self.arr_in, shape[1],
threads=threads, avoid_copy=True)
else:
self.arr_in = pyfftw.n_byte_align_empty(shape, pyfftw.simd_alignment, dtype=dtype)
self.fftw = pyfftw.builders.rfft2(self.arr_in, overwrite_input=True,
threads=threads, avoid_copy=True)
def test_call_with_unaligned(self):
'''Make sure the right thing happens with unaligned data.
'''
input_array = (numpy.random.randn(*self.input_array.shape)
+ 1j*numpy.random.randn(*self.input_array.shape))
output_array = self.fft(
input_array=n_byte_align(input_array.copy(), 16)).copy()
input_array = n_byte_align(input_array, 16)
output_array = n_byte_align(output_array, 16)
# Offset by one from 16 byte aligned to guarantee it's not
# 16 byte aligned
a = n_byte_align(input_array.copy(), 16)
a__ = n_byte_align_empty(
numpy.prod(a.shape)*a.itemsize+1, 16, dtype='int8')
a_ = a__[1:].view(dtype=a.dtype).reshape(*a.shape)
a_[:] = a
# Create a different second array the same way
b = n_byte_align(output_array.copy(), 16)
b__ = n_byte_align_empty(
numpy.prod(b.shape)*a.itemsize+1, 16, dtype='int8')
b_ = b__[1:].view(dtype=b.dtype).reshape(*b.shape)
b_[:] = a
# Set up for the first array
fft = FFTW(input_array, output_array)
a_[:] = a
output_array = fft().copy()
# Check a_ is not aligned...
self.assertRaisesRegex(ValueError, 'Invalid input alignment',
self.fft.update_arrays, *(a_, output_array))
# and b_ too
self.assertRaisesRegex(ValueError, 'Invalid output alignment',
self.fft.update_arrays, *(input_array, b_))
# But it should still work with the a_
fft(a_)
# However, trying to update the output will raise an error
self.assertRaisesRegex(ValueError, 'Invalid output alignment',
self.fft.update_arrays, *(input_array, b_))
# Same with SIMD off
fft = FFTW(input_array, output_array, flags=('FFTW_UNALIGNED',))
fft(a_)
self.assertRaisesRegex(ValueError, 'Invalid output alignment',
self.fft.update_arrays, *(input_array, b_))
def initialize(self,N1,N2,N3): ## Inverse transforms of uhat,vhat,what are of the truncated padded variable. ## Input is complex truncate,output is real untruncated self.invalT = pyfftw.n_byte_align_empty((int(3./2.*N1),int(3./2.*N2),int(3./4.*N3+1)), 16, 'complex128') self.outvalT= pyfftw.n_byte_align_empty((int(3./2.*N1),int(3./2.*N2),int(3./2*N3 )), 16, 'float64') self.ifftT_obj = pyfftw.FFTW(self.invalT,self.outvalT,axes=(0,1,2,),direction='FFTW_BACKWARD',threads=8) ## Fourier transforms of padded vars like u*u. ## Input is real full, output is imag truncated self.inval = pyfftw.n_byte_align_empty((int(3./2.*N1),int(3./2.*N2),int(3./2.*N3) ), 16, 'float64') self.outval= pyfftw.n_byte_align_empty((int(3./2.*N1),int(3./2.*N2),int(3./4*N3+1)), 16, 'complex128') self.fft_obj = pyfftw.FFTW(self.inval,self.outval,axes=(0,1,2,),direction='FFTW_FORWARD', threads=8)
def seqDatab_to_four(seqList, fftPlan, inverseVariable, method=1, n=0):
four_seq_datab = []
for i in range(len(seqList)):
if n == 0:
four_seq_datab.append(
pyfftw.n_byte_align_empty(len(seqList[i]), 1, 'complex'))
else:
four_seq_datab.append(pyfftw.n_byte_align_empty(n, 1, 'complex'))
four_seq_datab[i][:] = seq_to_four(seqList[i], fftPlan,
inverseVariable, method, n)
return four_seq_datab
def setUp(self):
self.input_array = n_byte_align_empty((256, 512), 16,
dtype='complex128')
self.output_array = n_byte_align_empty((256, 512), 16,
dtype='complex128')
self.fft = FFTW(self.input_array, self.output_array)
self.output_array[:] = (numpy.random.randn(*self.output_array.shape)
+ 1j*numpy.random.randn(*self.output_array.shape))
def __init__(self, A):
"""Set up arrays and FFTs for convolution.
Parameters
----------
A : ndarray (3-d)
PSF, assumed to be centered in the array at the
"reference wavelength."
"""
self.nw, self.ny, self.nx = A.shape
# The attribute `fftconv` stores the Fourier-space array
# necessary to convolve another array by the PSF. This is done
# by mulitiplying the input array by `fftconv` in fourier
# space.
#
# We shift the PSF so that instead of being exactly centered
# in the array, it is exactly centered on the lower left
# pixel. (For convolution in Fourier space, the (0, 0)
# element of the kernel is effectively the "center.")
# Note that this shifting is different than simply
# creating the PSF centered at the lower left pixel to begin
# with, due to wrap-around.
#
#`ifft2(fftconv).real` would be the PSF in
# real space, shifted to be centered on the lower-left pixel.
shift = -(self.ny - 1) / 2., -(self.nx - 1) / 2.
fshift = fft_shift_phasor_2d((self.ny, self.nx), shift)
fftconv = fft2(A) * fshift
# align on SIMD boundary.
self.fftconv = pyfftw.n_byte_align(fftconv,
pyfftw.simd_alignment,
dtype=np.complex128)
# set up input and output arrays for FFTs.
self.fftin = pyfftw.n_byte_align_empty(A.shape,
pyfftw.simd_alignment,
dtype=np.complex128)
self.fftout = pyfftw.n_byte_align_empty(A.shape,
pyfftw.simd_alignment,
dtype=np.complex128)
# Set up forward and backward FFTs.
self.fft = pyfftw.FFTW(self.fftin, self.fftout, axes=(1, 2), threads=1)
self.ifft = pyfftw.FFTW(self.fftout,
self.fftin,
axes=(1, 2),
threads=1,
direction='FFTW_BACKWARD')
self.fftnorm = 1. / (self.ny * self.nx)
def test_update_data_with_alignment_error(self):
in_shape = self.input_shapes['2d']
out_shape = self.output_shapes['2d']
byte_error = 1
axes=(-1,)
a, b = self.create_test_arrays(in_shape, out_shape)
a = n_byte_align(a, 16)
b = n_byte_align(b, 16)
fft, ifft = self.run_validate_fft(a, b, axes)
a, b = self.create_test_arrays(in_shape, out_shape)
# Offset from 16 byte aligned to guarantee it's not
# 16 byte aligned
a__ = n_byte_align_empty(
numpy.prod(in_shape)*a.itemsize+byte_error,
16, dtype='int8')
a_ = (a__[byte_error:]
.view(dtype=self.input_dtype).reshape(*in_shape))
a_[:] = a
b__ = n_byte_align_empty(
numpy.prod(out_shape)*b.itemsize+byte_error,
16, dtype='int8')
b_ = (b__[byte_error:]
.view(dtype=self.output_dtype).reshape(*out_shape))
b_[:] = b
with self.assertRaisesRegex(ValueError, 'Invalid output alignment'):
self.run_validate_fft(a, b_, axes, fft=fft, ifft=ifft,
create_array_copies=False)
with self.assertRaisesRegex(ValueError, 'Invalid input alignment'):
self.run_validate_fft(a_, b, axes, fft=fft, ifft=ifft,
create_array_copies=False)
# Should also be true for the unaligned case
fft, ifft = self.run_validate_fft(a, b, axes,
force_unaligned_data=True)
with self.assertRaisesRegex(ValueError, 'Invalid output alignment'):
self.run_validate_fft(a, b_, axes, fft=fft, ifft=ifft,
create_array_copies=False)
with self.assertRaisesRegex(ValueError, 'Invalid input alignment'):
self.run_validate_fft(a_, b, axes, fft=fft, ifft=ifft,
create_array_copies=False)
def __init__(self, psf, adr_refract, dtype=np.float64, fftw_threads=1):
self.shape = psf.shape
self.nw, self.ny, self.nx = psf.shape
spatial_shape = self.ny, self.nx
nbyte = pyfftw.simd_alignment
self.dtype = dtype
if dtype == np.float32:
self.complex_dtype = np.complex64
if dtype == np.float64:
self.complex_dtype = np.complex128
else:
raise ValueError("dtype must be float32 or float64")
# The attribute `fftconv` stores the Fourier-space array
# necessary to convolve another array by the PSF. This is done
# by mulitiplying the input array by `fftconv` in fourier
# space.
#
# We shift the PSF so that instead of being exactly centered
# in the array, it is exactly centered on the lower left
# pixel. (For convolution in Fourier space, the (0, 0)
# element of the kernel is effectively the "center.")
# Note that this shifting is different than simply
# creating the PSF centered at the lower left pixel to begin
# with, due to wrap-around.
#
# `ifft2(fftconv).real` would be the PSF in
# real space, shifted to be centered on the lower-left pixel.
shift = -(self.ny - 1) / 2.0, -(self.nx - 1) / 2.0
fshift = fft_shift_phasor_2d(spatial_shape, shift)
self.fftconv = fft2(psf) * fshift
self.fftconv = pyfftw.n_byte_align(self.fftconv, nbyte, dtype=self.complex_dtype)
# Check that ADR has the correct shape.
assert adr_refract.shape == (2, self.nw)
# Further shift the PSF by the ADR.
for i in range(self.nw):
shift = adr_refract[0, i], adr_refract[1, i]
fshift = fft_shift_phasor_2d(spatial_shape, shift)
self.fftconv[i, :, :] *= fshift
# set up input and output arrays for performing forward and reverse
# FFTs.
self.fftin = pyfftw.n_byte_align_empty(self.shape, nbyte, dtype=self.complex_dtype)
self.fftout = pyfftw.n_byte_align_empty(self.shape, nbyte, dtype=self.complex_dtype)
self.fft = pyfftw.FFTW(self.fftin, self.fftout, axes=(1, 2), threads=fftw_threads)
self.ifft = pyfftw.FFTW(self.fftout, self.fftin, axes=(1, 2), threads=fftw_threads, direction="FFTW_BACKWARD")
self.fftnorm = 1.0 / (self.ny * self.nx)
def __init__(self, nbeads, natoms, open_paths=None):
"""Initializes nm_trans.
Args:
nbeads: The number of beads.
natoms: The number of atoms.
"""
self.nbeads = nbeads
self.natoms = natoms
if open_paths is None:
open_paths = []
self._open = open_paths
# for atoms with open path we still use the matrix transformation
self._b2o_nm = mk_o_nm_matrix(nbeads)
self._o_nm2b = self._b2o_nm.T
try:
import pyfftw
info("Import of PyFFTW successful", verbosity.medium)
self.qdummy = pyfftw.n_byte_align_empty((nbeads, 3 * natoms), 16,
"float32")
self.qnmdummy = pyfftw.n_byte_align_empty(
(nbeads // 2 + 1, 3 * natoms), 16, "complex64")
self.fft = pyfftw.FFTW(self.qdummy,
self.qnmdummy,
axes=(0, ),
direction="FFTW_FORWARD")
self.ifft = pyfftw.FFTW(self.qnmdummy,
self.qdummy,
axes=(0, ),
direction="FFTW_BACKWARD")
except ImportError: # Uses standard numpy fft library if nothing better
# is available
info(
"Import of PyFFTW unsuccessful, using NumPy library instead",
verbosity.medium,
)
self.qdummy = np.zeros((nbeads, 3 * natoms), dtype="float32")
self.qnmdummy = np.zeros((nbeads // 2 + 1, 3 * natoms),
dtype="complex64")
def dummy_fft(self):
self.qnmdummy = np.fft.rfft(self.qdummy, axis=0)
def dummy_ifft(self):
self.qdummy = np.fft.irfft(self.qnmdummy,
n=self.nbeads,
axis=0)
self.fft = lambda: dummy_fft(self)
self.ifft = lambda: dummy_ifft(self)
def _initializeFFTW(sh):
"""Initialize the forward and inverse transforms"""
sh = tuple(sh)
d = len(sh)
ax = range(d)
inp = pyfftw.n_byte_align_empty(sh, 16, np.float64)
outp = pyfftw.n_byte_align_empty(sh[:-1] + (sh[-1] / 2 + 1, ), 16,
np.complex128)
ffter = pyfftw.FFTW(inp, outp, axes=ax)
iffter = pyfftw.FFTW(outp, inp, axes=ax, direction='FFTW_BACKWARD')
pyfftw.interfaces.cache.enable()
return (ffter, iffter)
def __init__(self, shape):
have_wisdom = self.load_wisdom()
self.floatbuf = pyfftw.n_byte_align_empty(shape, pyfftw.simd_alignment,
dtype=np.float32)
self._rfft2 = pyfftw.builders.rfft2(self.floatbuf,
planner_effort='FFTW_MEASURE')
self.complexbuf = pyfftw.n_byte_align_empty(shape,
pyfftw.simd_alignment,
dtype=np.complex64)
self._irfft2 = pyfftw.builders.irfft2(self.complexbuf,
planner_effort='FFTW_MEASURE')
if not have_wisdom:
self.save_wisdom()
def __init__(self, A):
"""Set up arrays and FFTs for convolution.
Parameters
----------
A : ndarray (3-d)
PSF, assumed to be centered in the array at the
"reference wavelength."
"""
self.nw, self.ny, self.nx = A.shape
# The attribute `fftconv` stores the Fourier-space array
# necessary to convolve another array by the PSF. This is done
# by mulitiplying the input array by `fftconv` in fourier
# space.
#
# We shift the PSF so that instead of being exactly centered
# in the array, it is exactly centered on the lower left
# pixel. (For convolution in Fourier space, the (0, 0)
# element of the kernel is effectively the "center.")
# Note that this shifting is different than simply
# creating the PSF centered at the lower left pixel to begin
# with, due to wrap-around.
#
#`ifft2(fftconv).real` would be the PSF in
# real space, shifted to be centered on the lower-left pixel.
shift = -(self.ny - 1) / 2., -(self.nx - 1) / 2.
fshift = fft_shift_phasor_2d((self.ny, self.nx), shift)
fftconv = fft2(A) * fshift
# align on SIMD boundary.
self.fftconv = pyfftw.n_byte_align(fftconv, pyfftw.simd_alignment,
dtype=np.complex128)
# set up input and output arrays for FFTs.
self.fftin = pyfftw.n_byte_align_empty(A.shape,
pyfftw.simd_alignment,
dtype=np.complex128)
self.fftout = pyfftw.n_byte_align_empty(A.shape,
pyfftw.simd_alignment,
dtype=np.complex128)
# Set up forward and backward FFTs.
self.fft = pyfftw.FFTW(self.fftin, self.fftout, axes=(1, 2),
threads=1)
self.ifft = pyfftw.FFTW(self.fftout, self.fftin, axes=(1, 2),
threads=1, direction='FFTW_BACKWARD')
self.fftnorm = 1. / (self.ny * self.nx)
def __init__(self, gridsize):
self.gridsize = gridsize
self.corrin = pyfftw.n_byte_align_empty(gridsize * 2, 16, 'complex128')
self.corrtransfer = pyfftw.n_byte_align_empty(gridsize * 2, 16,
'complex128')
self.fft = pyfftw.FFTW(self.corrin,
self.corrtransfer,
direction='FFTW_FORWARD')
self.backout = pyfftw.n_byte_align_empty(gridsize * 2, 16,
'complex128')
self.ifft = pyfftw.FFTW(self.corrtransfer,
self.backout,
direction='FFTW_BACKWARD')
def __init__(self, shape):
have_wisdom = self.load_wisdom()
self.floatbuf = pyfftw.n_byte_align_empty(shape,
pyfftw.simd_alignment,
dtype=np.float32)
self._rfft2 = pyfftw.builders.rfft2(self.floatbuf,
planner_effort='FFTW_MEASURE')
self.complexbuf = pyfftw.n_byte_align_empty(shape,
pyfftw.simd_alignment,
dtype=np.complex64)
self._irfft2 = pyfftw.builders.irfft2(self.complexbuf,
planner_effort='FFTW_MEASURE')
if not have_wisdom:
self.save_wisdom()
def _FFT(_delta, fft='pyfftw', Ngrid=360, silent=True):
''' run FFT
'''
if fft == 'pyfftw':
delta = pyfftw.n_byte_align_empty((Ngrid, Ngrid, Ngrid),
16,
dtype='complex64')
elif fft == 'fortran':
delta = np.zeros((Ngrid, Ngrid, Ngrid), dtype='complex64', order='F')
delta.real = _delta[::2, :, :]
delta.imag = _delta[1::2, :, :]
if not silent: print('positions assigned to grid')
# FFT delta (checked with fortran code, more or less matches)
if fft == 'pyfftw':
fftw_ob = pyfftw.builders.ifftn(delta, planner_effort='FFTW_ESTIMATE')
ifft_delta = np.zeros((Ngrid, Ngrid, Ngrid),
dtype='complex64',
order='F')
ifft_delta[:, :, :] = fftw_ob(normalise_idft=False)
elif fft == 'fortran':
ifft_delta = np.zeros((Ngrid, Ngrid, Ngrid),
dtype=np.complex64,
order='F')
fEstimate.ffting(delta, N, Ngrid)
ifft_delta[:, :, :] = delta[:, :, :]
return ifft_delta
def fftw():
inp = pyfftw.n_byte_align_empty(2048, 8, 'float64')
out = pyfftw.n_byte_align_empty(1024+1, 16, 'complex128')
a = pyfftw.n_byte_align_empty(2048, 16, 'float64')
b = pyfftw.n_byte_align_empty(1024+1, 16, 'complex128')
a[:] = numpy.random.randn(2048)
fft_factory = pyfftw.FFTW(inp, out, direction='FFTW_FORWARD', flags=('FFTW_MEASURE', ), threads=1)
fft_factory(a, b)
c = numpy.fft.rfft(a)
print numpy.allclose(b, c)
print b
print c
def seq_convert(sequence, method=1, n=0):
seqLen = len(sequence)
if n == 0:
vecLen = seqLen
else:
vecLen = n
if method == 1:
intSeq = pyfftw.n_byte_align_empty(vecLen, 1, 'complex')
#intSeq = numpy.empty(seqLen,complex);
for i, let in enumerate(sequence):
if i < vecLen:
if let == 'A' or let == 'a':
intSeq[i] = 1j
elif let == 'T' or let == 't':
intSeq[i] = -1j
elif let == 'C' or let == 'c':
intSeq[i] = 1
elif let == 'G' or let == 'g':
intSeq[i] = -1
else:
intSeq[i] = 0
for i in range(seqLen, vecLen):
intSeq[i] = 0
return intSeq
def __init__(self, eqn, grid, damping='default'):
"""
Initialise a solver.
Parameters:
eqn: Equation object specifying the system of equations to solve.
grid: Grid object that specifies the grid on which to solve the
equations
damping: An array to use for damping. Default is a tanh function]
that drops from 1.0 to 0 over the last 10% of each dimension. If
None, no damping will be applied.
None, no damping will be applied.
"""
al = pyfftw.simd_alignment
self.grid = deepcopy(grid)
self.eqn = deepcopy(eqn)
N = grid.N
self.x, self.y = grid.getSpatialGrid()
self.psis = [pyfftw.n_byte_align_empty((N, N), al, 'complex128') for i
in xrange(eqn.N_fields)]
if damping == 'default':
self.damping = defaultDamping(grid)
else:
self.damping = damping
def test_call_with_different_striding(self):
"""Test the input update with different strides to internal array.
"""
input_array_shape = self.input_array.shape + (2,)
internal_array_shape = self.internal_array.shape
internal_array = n_byte_align(
numpy.random.randn(*internal_array_shape) + 1j * numpy.random.randn(*internal_array_shape), simd_alignment
)
fft = utils._FFTWWrapper(
internal_array,
self.output_array,
input_array_slicer=self.input_array_slicer,
FFTW_array_slicer=self.FFTW_array_slicer,
)
test_output_array = fft().copy()
new_input_array = n_byte_align_empty(input_array_shape, simd_alignment, dtype=internal_array.dtype)
new_input_array[:] = 0
new_input_array[:, :, 0][self.input_array_slicer] = internal_array[self.FFTW_array_slicer]
new_output = fft(new_input_array[:, :, 0]).copy()
# Test the test!
self.assertTrue(new_input_array[:, :, 0].strides != internal_array.strides)
self.assertTrue(numpy.alltrue(test_output_array == new_output))
def __init__(self, eqn, grid, damping='default'):
"""
Initialise a solver.
Parameters:
eqn: Equation object specifying the system of equations to solve.
grid: Grid object that specifies the grid on which to solve the
equations
damping: An array to use for damping. Default is a tanh function]
that drops from 1.0 to 0 over the last 10% of each dimension. If
None, no damping will be applied.
None, no damping will be applied.
"""
al = pyfftw.simd_alignment
self.grid = deepcopy(grid)
self.eqn = deepcopy(eqn)
N = grid.N
self.x, self.y = grid.getSpatialGrid()
self.psis = [
pyfftw.n_byte_align_empty((N, N), al, 'complex128')
for i in xrange(eqn.N_fields)
]
if damping == 'default':
self.damping = defaultDamping(grid)
else:
self.damping = damping
def seq_convert(sequence,method=1,n=0): seqLen = len(sequence) if n==0: vecLen = seqLen else: vecLen=n if method == 1: intSeq = pyfftw.n_byte_align_empty(vecLen, 1, 'complex') #intSeq = numpy.empty(seqLen,complex); for i,let in enumerate(sequence): if i<vecLen: if let == 'A' or let=='a': intSeq[i] = 1j elif let=='T' or let == 't': intSeq[i]=-1j elif let=='C' or let == 'c': intSeq[i] = 1 elif let =='G' or let == 'g': intSeq[i] = -1 else: intSeq[i] = 0 for i in range(seqLen,vecLen): intSeq[i] = 0 return intSeq
def allocate(shape, dtype, use_pyfftw=True):
"""
Allocate a contiguous block of un-initialized typed memory.
If the pyfftw module is importable, allocates 16-byte aligned memory using
for improved SIMD instruction performance. Otherwise, uses the
:func:`numpy.empty` function. When shape is multi-dimensional, the
returned memory is initialized for C (row-major) storage order.
Parameters
----------
shape : int or tuple of ints
Shape of the empty array to allocate.
dtype : numpy data-type
Data type to assign to the empty array.
use_pyfftw : bool
Use the `pyFFTW package
<http://hgomersall.github.io/pyFFTW/index.html>`_ if it is available.
Returns
-------
numpy.ndarray
Array of un-initialized data with the requested shape and data type.
The storage order of multi-dimensional arrays is always C-type
(row-major).
"""
if use_pyfftw:
try:
import pyfftw
return pyfftw.n_byte_align_empty(
shape, pyfftw.simd_alignment, dtype, order='C')
except:
pass
return np.empty(shape, dtype, order='C')
def createplan(s,n,dirF=1):
b = pyfftw.n_byte_align_empty(n,1, 'complex')
if dirF==1:
fftwO = pyfftw.FFTW(s,b,flags=('FFTW_PATIENT',))
else:
fftwO = pyfftw.FFTW(s,b,flags=('FFTW_PATIENT',),direction="FFTW_BACKWARD")
return fftwO
def test_avoid_copy(self):
'''Test the avoid_copy flag
'''
dtype_tuple = io_dtypes[functions[self.func]]
for dtype in dtype_tuple[0]:
for test_shape, s, kwargs in self.test_data:
_kwargs = kwargs.copy()
_kwargs['avoid_copy'] = True
s2 = copy.copy(s)
try:
for each_axis, length in enumerate(s):
s2[each_axis] += 2
except TypeError:
s2 += 2
input_array = dtype_tuple[1](test_shape, dtype)
self.assertRaisesRegex(ValueError,
'Cannot avoid copy.*transform shape.*',
getattr(builders, self.func),
input_array, s2, **_kwargs)
non_contiguous_shape = [
each_dim * 2 for each_dim in test_shape]
non_contiguous_slices = (
[slice(None, None, 2)] * len(test_shape))
misaligned_input_array = dtype_tuple[1](
non_contiguous_shape, dtype)[non_contiguous_slices]
self.assertRaisesRegex(ValueError,
'Cannot avoid copy.*not contiguous.*',
getattr(builders, self.func),
misaligned_input_array, s, **_kwargs)
# Offset by one from 16 byte aligned to guarantee it's not
# 16 byte aligned
_input_array = n_byte_align_empty(
numpy.prod(test_shape)*input_array.itemsize+1,
16, dtype='int8')
misaligned_input_array = _input_array[1:].view(
dtype=input_array.dtype).reshape(*test_shape)
self.assertRaisesRegex(ValueError,
'Cannot avoid copy.*not aligned.*',
getattr(builders, self.func),
misaligned_input_array, s, **_kwargs)
_input_array = n_byte_align(input_array.copy(), 16)
FFTW_object = getattr(builders, self.func)(
_input_array, s, **_kwargs)
# A catch all to make sure the internal array
# is not a copy
self.assertTrue(FFTW_object.get_input_array() is
_input_array)
def filtdem(M,dx,dy,filtype, f):
'''
**M = filtdem(Pmat,Fmat,filtype, f)** \n
filter a fourier transformed DEM into a topo 2D array \n
dependencies: numpy, specfilt2d(), gaussian()
Parameters
----------
**Pmat** : 2D array, Fourier transform resutling from fftdem(). Contains the Fourier coefficients \n
**Fmat** : 2D array, containing the corresponding frequencies \n
**filtype** : string, 3 possibilities: "lowpass", "highpass", and "bandpass" \n
**f** : tuple of frequencies for the filter.
- If filtype = "lowpass", f = (flow, fhigh)
- If filtype = "highpass", f = (flow, fhigh)
- If filtype = "bandpass", f = (flow1, flow2, fhigh1, fhigh2) with flow1 < flow2 < fhigh1 < fhigh2
Returns
-------
**M** : 2D array, inversed transformed of Pmat
'''
nx = np.size(M,1)
ny = np.size(M,0)
pad=0
window=0
_,Pmat, fmat, _, _ = fftdem(M, dx, dy, pad, window)
M = linearDetrend.detrend(M)[0]
M_filt=pyfftw.n_byte_align_empty((nx,ny), 16, 'complex128')
M_filt= pyfftw.interfaces.numpy_fft.fftshift(pyfftw.interfaces.numpy_fft.fft2(M))
F = specfilt2d(fmat, f, filtype)
# take the inverse FFT of the filtered spectrum
M = np.real(pyfftw.interfaces.numpy_fft.ifft2(pyfftw.interfaces.numpy_fft.ifftshift(M_filt*F)))
return M
def initialize(self):
## Aloocate the real and complex arrays
self.__acf_ft = pyfftw.n_byte_align_empty(2 * self.__N - 2,
32,
dtype=np.float64)
self.__cplx_input = pyfftw.n_byte_align_empty(2 * self.__N - 2,
32,
dtype=np.complex128)
self.__cplx_output = pyfftw.n_byte_align_empty(2 * self.__N - 2,
32,
dtype=np.complex128)
## Initialize the FFT object: N - 1 is a power of two!!!
self.__fft_object = pyfftw.FFTW(self.__cplx_input,
self.__cplx_output,
threads=1,
direction='FFTW_FORWARD',
flags=(
'FFTW_DESTROY_INPUT',
'FFTW_ESTIMATE',
)) #
## The autocorrelation structure for the fBM is constant provided the Hurst exponent
## and the size sample are fixed. "Synthese de la covariance du fGn", Synthesise
## the covariance of the fractional Gaussian noise. This autocorrelation function
## models long range (epochal) dependence.
R = np.arange(self.__N, dtype=np.float64)
## The noise autocorrelation structure is directly derivable from the autocorrelation
## of the time-continuous fBM:
## r(s,t) = .5 * ( |s|^{2H}+|t|^{2H}-|s-t|^{2H} )
## If the noise is generated for an equally spaced. sampling of an fBM like process,
## then the autocorrelation function must be multiplied by ∆^{2H}. Since Fourier
## Transform is linear (even the discrete one), this routine can just generate a unit
## variance fractional Gaussian noise.
R = self.__sigma * self.__sigma * .5 * (
np.abs(R - 1)**(2.0 * self.__H) + np.abs(R + 1)**(2.0 * self.__H) -
2 * np.abs(R)**(2.0 * self.__H))
## Generate the first row of the 2Mx2M Toeplitz matrix, where 2M = N + N-2: it should
## be [ r_0, ..., r_{N-1}, r_{N-2}, ..., r_1 ]
self.__cplx_input[:] = np.append(R, R[::-1][1:-1]) + 1j * 0
del R
## The circulant matrix, defined by the autocorrelation structure above is necessarily
## positive definite, which is equivalent to the FFT of any its row being non-negative.
self.__fft_object()
## Due to numerical round-off errors we truncate close to zero negative real Fourier
## coefficients.
self.__acf_ft[:] = np.sqrt(
np.maximum(np.real(self.__cplx_output), 0.0) / (2 * self.__N - 2))
def zeros_like(x):
sz = x.shape
if USINGFFTW:
ret = pyfftw.n_byte_align_empty(x.shape,32,x.dtype)
ret[:] = 0
return ret
else:
return n.zeros(sz,dtype=x.dtype)
def seq_to_four(seq1, fftPlan, inverseVariable, method=1, n=0):
if n == 0:
n = len(seq1)
sF1 = pyfftw.n_byte_align_empty(n, 1, 'complex') #create an empty vector
s1 = seq_convert(seq1, method, n)
sF1[:] = convert_complex_four(s1, fftPlan, inverseVariable)
return sF1
def __init__(self, nbeads, natoms):
"""Initializes nm_trans.
Args:
nbeads: The number of beads.
natoms: The number of atoms.
"""
self.nbeads = nbeads
self.natoms = natoms
try:
import pyfftw
info("Import of PyFFTW successful", verbosity.medium)
self.qdummy = pyfftw.n_byte_align_empty((nbeads, 3 * natoms), 16,
'float32')
self.qnmdummy = pyfftw.n_byte_align_empty(
(nbeads // 2 + 1, 3 * natoms), 16, 'complex64')
self.fft = pyfftw.FFTW(self.qdummy,
self.qnmdummy,
axes=(0, ),
direction='FFTW_FORWARD')
self.ifft = pyfftw.FFTW(self.qnmdummy,
self.qdummy,
axes=(0, ),
direction='FFTW_BACKWARD')
except ImportError: #Uses standard numpy fft library if nothing better
#is available
info("Import of PyFFTW unsuccessful, using NumPy library instead",
verbosity.medium)
self.qdummy = np.zeros((nbeads, 3 * natoms), dtype='float32')
self.qnmdummy = np.zeros((nbeads // 2 + 1, 3 * natoms),
dtype='complex64')
def dummy_fft(self):
self.qnmdummy = np.fft.rfft(self.qdummy, axis=0)
def dummy_ifft(self):
self.qdummy = np.fft.irfft(self.qnmdummy,
n=self.nbeads,
axis=0)
self.fft = lambda: dummy_fft(self)
self.ifft = lambda: dummy_ifft(self)
def __init__(self, shape, dtype, threads=1, inverse=False):
self.threads = threads
if inverse:
self.arr_in = pyfftw.n_byte_align_empty(shape[0],
pyfftw.simd_alignment,
dtype=dtype)
self.fftw = pyfftw.builders.irfft2(self.arr_in,
shape[1],
threads=threads,
avoid_copy=True)
else:
self.arr_in = pyfftw.n_byte_align_empty(shape,
pyfftw.simd_alignment,
dtype=dtype)
self.fftw = pyfftw.builders.rfft2(self.arr_in,
overwrite_input=True,
threads=threads,
avoid_copy=True)
def __init__(self):
super(burst_detector, self).__init__(
name="Burst Detector",
in_sig=[numpy.complex64],
out_sig=[numpy.complex64]
)
self._burst_tag_symbol = gr.pmt.string_to_symbol('burst')
self._burst = False
self.block_size = 256
self.hysteresis_timeout = 3 #int(math.ceil(768 / self.block_size))
self.hysteresis_count = 0
self.fft_window = scipy.signal.hanning(self.block_size)
self.fft_in = pyfftw.n_byte_align_empty((self.block_size,), self.block_size, dtype='complex64')
self.fft_out = pyfftw.n_byte_align_empty((self.block_size,), self.block_size, dtype='complex64')
self.fft = pyfftw.FFTW(self.fft_in, self.fft_out)
def createplan(s, n, dirF=1):
b = pyfftw.n_byte_align_empty(n, 1, 'complex')
if dirF == 1:
fftwO = pyfftw.FFTW(s, b, flags=('FFTW_PATIENT', ))
else:
fftwO = pyfftw.FFTW(s,
b,
flags=('FFTW_PATIENT', ),
direction="FFTW_BACKWARD")
return fftwO
def test_n_byte_align_empty(self):
shape = (10, 10)
# Test a few alignments and dtypes
for each in [(3, 'float64'), (7, 'float64'), (9, 'float32'),
(16, 'int64'), (24, 'bool'), (23, 'complex64'),
(63, 'complex128'), (64, 'int8')]:
n = each[0]
b = n_byte_align_empty(shape, n, dtype=each[1])
self.assertTrue(b.ctypes.data % n == 0)
self.assertTrue(b.dtype == each[1])
def __init__(self, npa, dt, window=None):
n = len(npa)
fftia = pyfftw.n_byte_align_empty(n, 16, 'float32')
fftoa = pyfftw.n_byte_align_empty(n / 2 + 1, 16, 'complex64')
fft = pyfftw.FFTW(fftia,
fftoa,
flags=('FFTW_ESTIMATE', ),
planning_timelimit=60.0)
maxv = npa.max()
if window:
w = window(n)
fftia[:] = w * (npa / maxv)
else:
fftia[:] = npa / maxv
fft()
self.sa = np.abs(fftoa)
## Scale amplitude for window
if window:
scale = 1.0 / np.sum(window(10000) / 10000.0)
print "Scaling postwindow by", scale
self.sa = scale * self.sa
## 2.0 To get magnitude in terms of original V since half of spectrum is returned
## Result is vrms
print "Converting to dBFS"
maxv = self.sa.max()
self.sa = 20.0 * np.log10(self.sa / maxv)
self.mhz2bin = len(self.sa) * 1e6 * 2 * dt
self.bin2mhz = 1.0 / self.mhz2bin
print "Spectrum Array length is", len(self.sa)
def __init__(self, A):
#log=MyLogger.getLogger("ModToolBox.FFTM2.__init__")
pyfftw.interfaces.cache.enable()
self.ThisTypeName = A.dtype.type.__name__
self.ThisType = A.dtype
#print>>log, "[Size, fftwtype]=[%s,%s]"%(str(A.shape),self.ThisType)
#print>>log, "define"
self.a = pyfftw.n_byte_align_empty(A.shape, 16,
self.ThisType) #'complex128')
self.b = pyfftw.n_byte_align_empty(A.shape, 16,
self.ThisType) #'complex128')
#print>>log, "define1"
self.fft_FORWARD = pyfftw.FFTW(self.a,
self.b,
direction="FFTW_FORWARD",
flags=('FFTW_ESTIMATE', )) #,threads=4)
self.fft_BACKWARD = pyfftw.FFTW(
self.a,
self.b,
direction="FFTW_BACKWARD",
flags=('FFTW_ESTIMATE', )) #,threads=4)
def _initialize_fft(self):
# set up fft functions for use later
if self.use_fftw:
A = pyfftw.n_byte_align_empty(self.shape_real, pyfftw.simd_alignment,\
dtype=self.dtype_real)
Ah = pyfftw.n_byte_align_empty(self.shape_cplx, pyfftw.simd_alignment,\
dtype=self.dtype_cplx)
self.fft2 = (lambda x :
pyfftw.interfaces.numpy_fft.rfft2(x, threads=self.ntd,\
planner_effort='FFTW_PATIENT'))
self.ifft2 = (lambda x :
pyfftw.interfaces.numpy_fft.irfft2(x, threads=self.ntd,\
planner_effort='FFTW_PATIENT'))
# Forward transforms
self.A2Ah = pyfftw.builders.rfft2(A,threads=self.ntd,\
planner_effort='FFTW_PATIENT')
self.v2vh = pyfftw.builders.rfft2(A,threads=self.ntd,\
planner_effort='FFTW_PATIENT')
# Backward transforms
self.Ah2A = pyfftw.builders.irfft2(Ah,threads=self.ntd,\
planner_effort='FFTW_PATIENT')
del A, Ah
else:
self.fft2 = (lambda x: np.fft.rfft2(x))
self.ifft2 = (lambda x: np.fft.irfft2(x))
def A2Ah(self, A):
return np.fft.rfft2(A)
def Ah2A(self, Ah):
return np.fft.irfft2(Ah)
