def __ifft(self, F):
"""__ifft(self, F) -> numpy.2darray
apply 2D inverse Fourier transform
Parameters
----------
F : numpy.2darray
Returns
-------
numpy.2darray
"""
if self.__fft_type not in ['numpy', 'fftw', 'cufft']:
raise ValueError('Invalid parameter for the keyword "fft_type."')
if found_pyfftw is True and self.__fft_type == 'fftw':
pyfftw.forget_wisdom()
ifunc = pyfftw.builders.ifft2(F,
overwrite_input=True,
planner_effort='FFTW_ESTIMATE',
threads=CPU_COUNT)
return ifunc()
elif found_cufft is True and self.__fft_type == 'cufft':
self.__x_gpu.set(F.astype(np.complex64))
cu_fft.ifft(self.__x_gpu, self.__xf_gpu, self.__plan, True)
return self.__xf_gpu.get()
else:
return ifft2(F)
def compute_pow2_complex_wisdom(path,
pow2=range(20),
rigor='measure',
threads=16):
"""
???
If you plan with FFTW_PATIENT, it will automatically disable
threads for sizes that don't benefit from parallelization.
"""
flags = [RIGOR_MAP[rigor], 'FFTW_DESTROY_INPUT']
wisdom_fnames = []
for pow2_i in pow2:
N = 2**pow2_i
x_input = pyfftw.empty_aligned(N, dtype='complex128')
x_output = pyfftw.empty_aligned(N, dtype='complex128')
for direction in ['forward', 'backward']:
logger.info('building wisdom for complex 2**{} {}'.format(
pow2_i, direction))
plan = pyfftw.FFTW(x_input,
x_output,
direction=DIRECTION_MAP[direction],
flags=flags,
threads=threads)
wisdom_fnames.append(
wisdom_fname(path, 'complex', pow2_i, threads, direction,
rigor))
logger.info('writing to {}'.format(wisdom_fnames[-1]))
with open(wisdom_fnames[-1], 'w') as fid:
dump(pyfftw.export_wisdom(), fid, -1)
pyfftw.forget_wisdom()
return wisdom_fnames
def test_planning_time_limit(self):
in_shape = self.input_shapes['1d']
out_shape = self.output_shapes['1d']
axes=(0,)
a, b = self.create_test_arrays(in_shape, out_shape)
# run this a few times
runs = 10
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(b, a, axes=axes)
unlimited_time = (time.time() - t1)/runs
time_limit = (unlimited_time)/8
# Now do it again but with an upper limit on the time
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(b, a, axes=axes, planning_timelimit=time_limit)
limited_time = (time.time() - t1)/runs
# Give a 2x margin
self.assertTrue(limited_time < time_limit*2)
def fftw_fast_builder_test(input_data): # See fftw_builder_test for comments
pyfftw.forget_wisdom()
a = np.array(input_data, dtype='float32')
fft_obj = pyfftw.builders.rfft(
a, planner_effort='FFTW_ESTIMATE'
) # FFTW_ESTIMATE is a lower effort planner than the default. This seems to work more quickly for over 8000 points
return fft_obj()
def test_wisdom_only(self):
in_shape = self.input_shapes['small_1d']
out_shape = self.output_shapes['small_1d']
axes = (0, )
a, b = self.create_test_arrays(in_shape, out_shape)
forget_wisdom()
# with no wisdom, an error should be raised with FFTW_WISDOM_ONLY
#
# NB: wisdom is specific to aligned/unaligned distinction, so we must
# ensure that the arrays don't get copied (and potentially
# switched between aligned and unaligned) by run_validate_fft()...
self.assertRaisesRegex(
RuntimeError, 'No FFTW wisdom', self.run_validate_fft,
*(a, b, axes), **{
'flags': ('FFTW_ESTIMATE', 'FFTW_WISDOM_ONLY'),
'create_array_copies': False
})
# now plan the FFT
self.run_validate_fft(a,
b,
axes,
flags=('FFTW_ESTIMATE', ),
create_array_copies=False)
# now FFTW_WISDOM_ONLY should not raise an error because the plan should
# be in the wisdom
self.run_validate_fft(a,
b,
axes,
flags=('FFTW_ESTIMATE', 'FFTW_WISDOM_ONLY'),
create_array_copies=False)
def test_planning_time_limit(self):
in_shape = self.input_shapes['1d']
out_shape = self.output_shapes['1d']
axes=(0,)
a, b = self.create_test_arrays(in_shape, out_shape)
# run this a few times
runs = 10
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(b, a, axes=axes)
unlimited_time = (time.time() - t1)/runs
time_limit = (unlimited_time)/8
# Now do it again but with an upper limit on the time
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(b, a, axes=axes, planning_timelimit=time_limit)
limited_time = (time.time() - t1)/runs
import sys
if sys.platform == 'win32':
# Give a 4x margin on windows. The timers are low
# precision and FFTW seems to take longer anyway
self.assertTrue(limited_time < time_limit*4)
else:
# Otherwise have a 2x margin
self.assertTrue(limited_time < time_limit*2)
def _ifftw(F, *args, **kwargs):
pyfftw.forget_wisdom()
ifunc = pyfftw.builders.ifft2(F,
overwrite_input=True,
planner_effort='FFTW_ESTIMATE',
threads=multiprocessing.cpu_count())
return ifunc()
def test_planning_time_limit(self):
in_shape = self.input_shapes['1d']
out_shape = self.output_shapes['1d']
axes = (0, )
a, b = self.create_test_arrays(in_shape, out_shape)
# run this a few times
runs = 10
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(a, b, axes=axes)
unlimited_time = (time.time() - t1) / runs
time_limit = (unlimited_time) / 8
# Now do it again but with an upper limit on the time
t1 = time.time()
for n in range(runs):
forget_wisdom()
fft = FFTW(a, b, axes=axes, planning_timelimit=time_limit)
limited_time = (time.time() - t1) / runs
import sys
if sys.platform == 'win32':
# Give a 4x margin on windows. The timers are low
# precision and FFTW seems to take longer anyway
self.assertTrue(limited_time < time_limit * 4)
else:
# Otherwise have a 2x margin
self.assertTrue(limited_time < time_limit * 2)
def fftw_builder_test(input_data):
pyfftw.forget_wisdom(
) # This is just here to keep the tests honest, normally pyfftw will remember setup parameters and go more quickly when it is run a second time
a = np.array(
input_data, dtype='float32'
) # This turns the input list into a numpy array. We can make it a 32 bit float because the data is all real, no imaginary component
fft_obj = pyfftw.builders.rfft(
a) # This creates an object which generates the FFT
return fft_obj() # And calling the object returns the FFT
def fftw_test_no_flag(
input_data
): # This is fftw_test with different FFTW options. See fftw_test for comments
pyfftw.forget_wisdom()
outLength = len(input_data) // 2 + 1
a = pyfftw.empty_aligned(len(input_data), dtype='float32')
outData = pyfftw.empty_aligned(outLength, dtype='complex64')
fft_obj = pyfftw.FFTW(a, outData, planning_timelimit=1.0)
a[:] = np.array(input_data, dtype='float32')
return fft_obj()
def test_export(self):
forget_wisdom()
before_wisdom = export_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
self.compare(before_wisdom, after_wisdom)
def test_export(self):
forget_wisdom()
before_wisdom = export_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
self.compare(before_wisdom, after_wisdom)
def test_export(self):
forget_wisdom()
before_wisdom = export_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
for n in range(0,2):
self.assertNotEqual(before_wisdom[n], after_wisdom[n])
def test_export(self):
forget_wisdom()
before_wisdom = export_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
for n in range(0,2):
self.assertNotEqual(before_wisdom[n], after_wisdom[n])
def fft( fftInput ):
"""
Takes fftInput as numpy array
Returns forward FFT calculated by FFTW
"""
pyfftw.forget_wisdom() # Detail of FFTW needed to stop it messing up
start = time.time()
fftOutput = np.empty_like( fftInput )
# Setting up and running FFT (forward) calculation
fft = pyfftw.FFTW( fftInput, fftOutput, direction='FFTW_FORWARD', flags=('FFTW_MEASURE', ), threads=nthreads, planning_timelimit=None )
fft()
timetaken = time.time() - start
print('Time taken for FFT with %d cores is %.3f secs.' % (nthreads, timetaken))
fftOutput *= (1/ (0.5*len(fftOutput)) ) # FFT normalisation
return fftOutput
def fftw_test_complex(
input_data
): # This is fftw_test but running a complex FFT as opposed to a real input FFT. See fftw_test for comments
pyfftw.forget_wisdom()
outLengthMod = len(input_data) // 2 + 1 # Size of expected return data
outLength = len(input_data) # For a complex input FFT, we get more data
a = pyfftw.empty_aligned(
len(input_data), dtype='complex64'
) # The FFTW determines the type of FFT by the type of input
outData = pyfftw.empty_aligned(outLength, dtype='complex64')
fft_obj = pyfftw.FFTW(a,
outData,
flags=('FFTW_ESTIMATE', ),
planning_timelimit=1.0)
a[:] = np.array(input_data, dtype='complex64')
return fft_obj()[0:outLengthMod]
def test_import(self):
forget_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
forget_wisdom()
before_wisdom = export_wisdom()
success = import_wisdom(after_wisdom)
self.compare(before_wisdom, after_wisdom)
self.assertEqual(success, tuple([x in _supported_types for x in ['64', '32', 'ld']]))
def test_import(self):
forget_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
forget_wisdom()
before_wisdom = export_wisdom()
success = import_wisdom(after_wisdom)
for n in range(0,2):
self.assertNotEqual(before_wisdom[n], after_wisdom[n])
self.assertEqual(success, (True, True, True))
def test_import(self):
forget_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
forget_wisdom()
before_wisdom = export_wisdom()
success = import_wisdom(after_wisdom)
for n in range(0,2):
self.assertNotEqual(before_wisdom[n], after_wisdom[n])
self.assertEqual(success, (True, True, True))
def test_import(self):
forget_wisdom()
self.generate_wisdom()
after_wisdom = export_wisdom()
forget_wisdom()
before_wisdom = export_wisdom()
success = import_wisdom(after_wisdom)
self.compare(before_wisdom, after_wisdom)
self.assertEqual(
success,
tuple([x in _supported_types for x in ['64', '32', 'ld']]))
def fftw_test(input_data):
pyfftw.forget_wisdom() # This is just here to keep the tests honest
outLength = len(
input_data
) // 2 + 1 # For a real FFT, the output is symetrical. fftw returns only half the data in this case
a = pyfftw.empty_aligned(
len(input_data), dtype='float32'
) # This is the input array. It will be cleared when the fft object is created
outData = pyfftw.empty_aligned(
outLength, dtype='complex64'
) # This is the output array. Not that the size and type must be appropriate
fft_obj = pyfftw.FFTW(
a, outData, flags=('FFTW_ESTIMATE', ),
planning_timelimit=1.0) # NB: The flags tuple has a comma
a[:] = np.array(
input_data, dtype='float32'
) # We have to fill the array after fft_obj is created. NB: 'a[:] =' puts data into the existing array, 'a =' creates a new array
return fft_obj(
) # Calling the object returns the FFT of the data now in a. The result is also in outData
def calculate_beta(self, beta_percentile=50, beta_count=200):
""" calculate beta for all image sections """
import pyfftw
if self.method == 'numpy':
fft = np.fft.fft2
else:
pyfftw.forget_wisdom()
beta_stack = []
# save image sections
#for i in np.random.choice(self.coord_N, beta_count):
for i in range(self.coord_N):
x, y = self.coords[i]
self.image_section = self._img[x - self._xwidth:x + self._xwidth + 1, y - self._ywidth:y + self._ywidth + 1].copy()
#image_section = signal.convolve(image_section, hanning, mode='same')
self.image_section *= self.hanning
imbar = np.sum(np.sqrt(self.image_section))
if self.method == 'numpy':
fft = np.fft.fft2
else:
#pyfftw.forget_wisdom()
fft = pyfftw.builders.fft2(self.image_section, threads=self._nthread, planner_effort='FFTW_ESTIMATE')
self.fourier_section = fft(self.image_section)
fourier_magnitude = np.abs(np.fft.fftshift(self.fourier_section))
beta_stack.append(fourier_magnitude / imbar)
result = np.percentile(np.stack(beta_stack), beta_percentile, axis=0)
#result = np.median(np.stack(beta_stack), axis=0)
del beta_stack
if self._debug:
print('... beta count: {}, beta percentile: {}'.format(beta_count, beta_percentile))
print('... xwidth: {}, step: {}, NX: {}, NY: {}'.format(self._xwidth, self._xstep, self.NX, self.NY))
plt.imshow(result)
plt.show()
return result
def compute_pow2_real_wisdom(path,
pow2=range(20),
rigor='measure',
threads=16):
"""
???
If you plan with FFTW_PATIENT, it will automatically disable
threads for sizes that don't benefit from parallelization.
"""
flags = [RIGOR_MAP[rigor],
'FFTW_DESTROY_INPUT']
wisdom_fnames = []
for pow2_i in pow2:
N = 2**pow2_i
for direction in ['forward', 'backward']:
logger.info('building wisdom for real 2**{} {}'.format(pow2_i,
direction))
if direction == 'forward':
x_input = pyfftw.empty_aligned(N, dtype='float64')
x_output = pyfftw.empty_aligned(int(N // 2) + 1, dtype='complex128')
else:
x_output = pyfftw.empty_aligned(N, dtype='float64')
x_input = pyfftw.empty_aligned(int(N // 2) + 1, dtype='complex128')
plan = pyfftw.FFTW(x_input,
x_output,
direction=DIRECTION_MAP[direction],
flags=flags,
threads=threads)
wisdom_fnames.append(wisdom_fname(path,
'real',
pow2_i,
threads,
direction,
rigor))
logger.info('writing to {}'.format(wisdom_fnames[-1]))
with open(wisdom_fnames[-1], 'w') as fid:
dump(pyfftw.export_wisdom(), fid, -1)
pyfftw.forget_wisdom()
return wisdom_fnames
def test_wisdom_only(self):
in_shape = self.input_shapes['small_1d']
out_shape = self.output_shapes['small_1d']
axes=(0,)
a, b = self.create_test_arrays(in_shape, out_shape)
forget_wisdom()
# with no wisdom, an error should be raised with FFTW_WISDOM_ONLY
#
# NB: wisdom is specific to aligned/unaligned distinction, so we must
# ensure that the arrays don't get copied (and potentially
# switched between aligned and unaligned) by run_validate_fft()...
self.assertRaisesRegex(RuntimeError, 'No FFTW wisdom',
self.run_validate_fft, *(a, b, axes),
**{'flags':('FFTW_ESTIMATE', 'FFTW_WISDOM_ONLY'),
'create_array_copies': False})
# now plan the FFT
self.run_validate_fft(a, b, axes, flags=('FFTW_ESTIMATE',),
create_array_copies=False)
# now FFTW_WISDOM_ONLY should not raise an error because the plan should
# be in the wisdom
self.run_validate_fft(a, b, axes, flags=('FFTW_ESTIMATE',
'FFTW_WISDOM_ONLY'), create_array_copies=False)
def ifuncfftw(F, *args, **kwargs):
"""ifuncfftw(F, *args, **kwargs) -> numpy.2darray
apply 2D inverse Fourier transform
Parameters
----------
F : numpy.2darray
args : options
kwargs : options
"""
if found_cufft is True and kwargs.get('fft_type') == 'cufft':
x_gpu = gpuarray.to_gpu(F.astype(np.complex64))
xf_gpu = gpuarray.empty(F.shape, np.complex64)
cu_fft.ifft(x_gpu, xf_gpu, args[0], True)
return xf_gpu.get()
elif found_pyfftw is True and kwargs.get('fft_type') == 'fftw':
pyfftw.forget_wisdom()
ifunc = pyfftw.builders.ifft2(F,
overwrite_input=True,
planner_effort='FFTW_ESTIMATE',
threads=CPU_COUNT)
return ifunc()
else:
return ifft2(F)
def __init__(self, img, width=8, step=4, gamma=3.0, beta_percentile=50, beta_count=0, types='float', mode='numpy', debug=False):
# tried to use pyfftw, but it was slower than numpy fft
import pyfftw
import cv2
# prepare variables
extended_img = cv2.copyMakeBorder(img, 2*width, 2*width, 2*width, 2*width, cv2.BORDER_REPLICATE)
self._img = extended_img
self._xwidth, self._ywidth = width, width
self._xstep, self._ystep = step, step
self._types = types
self._debug = debug
#self._nthread = multiprocessing.cpu_count()
self._nthread = 1
self.NX, self.NY = 2 * self._xwidth + 1, 2 * self._ywidth + 1
self.image_section = pyfftw.zeros_aligned((self.NX, self.NY), dtype='float32')
self.fourier_section = pyfftw.zeros_aligned((self.NX, self.NY), dtype='complex64')
self.method = mode
self.gamma = gamma
self.coords = self._build_coordinates()
self.coord_N = len(self.coords)
self.hanning = _build_hanning_window(self.NX, self.NY)
if beta_count == 0:
beta_count = int(self.coord_N * 1.0)
self.beta_count = beta_count
self.beta_percentile = beta_percentile
self.beta = []
pyfftw.forget_wisdom()
def clear_planner():
"""Clears fft planners (pyfftw)"""
if PYFFTW_INSTALLED:
pyfftw.forget_wisdom()
def _set_weights(self, linear_matrix, quadratic_matrix):
"""
Setup the time-dependent ROQ weights.
See https://dcc.ligo.org/LIGO-T2100125 for the detail of how to compute them.
Parameters
==========
linear_matrix, quadratic_matrix: array_like
Arrays of the linear and quadratic basis
"""
time_space = self._get_time_resolution()
number_of_time_samples = int(self.interferometers.duration /
time_space)
try:
import pyfftw
ifft_input = pyfftw.empty_aligned(number_of_time_samples,
dtype=complex)
ifft_output = pyfftw.empty_aligned(number_of_time_samples,
dtype=complex)
ifft = pyfftw.FFTW(ifft_input,
ifft_output,
direction='FFTW_BACKWARD')
except ImportError:
pyfftw = None
logger.warning(
"You do not have pyfftw installed, falling back to numpy.fft.")
ifft_input = np.zeros(number_of_time_samples, dtype=complex)
ifft = np.fft.ifft
earth_light_crossing_time = 2 * radius_of_earth / speed_of_light + 5 * time_space
start_idx = max(
0,
int(
np.floor((self.priors['{}_time'.format(
self.time_reference)].minimum - earth_light_crossing_time -
self.interferometers.start_time) / time_space)))
end_idx = min(
number_of_time_samples - 1,
int(
np.ceil((self.priors['{}_time'.format(
self.time_reference)].maximum + earth_light_crossing_time -
self.interferometers.start_time) / time_space)))
self.weights['time_samples'] = np.arange(start_idx,
end_idx + 1) * time_space
logger.info("Using {} ROQ time samples".format(
len(self.weights['time_samples'])))
for ifo in self.interferometers:
if self.roq_params is not None:
self.perform_roq_params_check(ifo)
# Get scaled ROQ quantities
roq_scaled_minimum_frequency = self.roq_params[
'flow'] * self.roq_scale_factor
roq_scaled_maximum_frequency = self.roq_params[
'fhigh'] * self.roq_scale_factor
roq_scaled_segment_length = self.roq_params[
'seglen'] / self.roq_scale_factor
# Generate frequencies for the ROQ
roq_frequencies = create_frequency_series(
sampling_frequency=roq_scaled_maximum_frequency * 2,
duration=roq_scaled_segment_length)
roq_mask = roq_frequencies >= roq_scaled_minimum_frequency
roq_frequencies = roq_frequencies[roq_mask]
overlap_frequencies, ifo_idxs, roq_idxs = np.intersect1d(
ifo.frequency_array[ifo.frequency_mask],
roq_frequencies,
return_indices=True)
else:
overlap_frequencies = ifo.frequency_array[ifo.frequency_mask]
roq_idxs = np.arange(linear_matrix.shape[0], dtype=int)
ifo_idxs = np.arange(sum(ifo.frequency_mask))
if len(ifo_idxs) != len(roq_idxs):
raise ValueError(
"Mismatch between ROQ basis and frequency array for "
"{}".format(ifo.name))
logger.info(
"Building ROQ weights for {} with {} frequencies between {} "
"and {}.".format(ifo.name, len(overlap_frequencies),
min(overlap_frequencies),
max(overlap_frequencies)))
ifft_input[:] *= 0.
self.weights[ifo.name + '_linear'] = \
np.zeros((len(self.weights['time_samples']), linear_matrix.shape[1]), dtype=complex)
data_over_psd = (
ifo.frequency_domain_strain[ifo.frequency_mask][ifo_idxs] /
ifo.power_spectral_density_array[ifo.frequency_mask][ifo_idxs])
nonzero_idxs = ifo_idxs + int(
ifo.frequency_array[ifo.frequency_mask][0] *
self.interferometers.duration)
for i, basis_element in enumerate(linear_matrix[roq_idxs].T):
ifft_input[nonzero_idxs] = data_over_psd * np.conj(
basis_element)
self.weights[ifo.name + '_linear'][:, i] = ifft(
ifft_input)[start_idx:end_idx + 1]
self.weights[
ifo.name +
'_linear'] *= 4. * number_of_time_samples / self.interferometers.duration
self.weights[ifo.name + '_quadratic'] = build_roq_weights(
1 /
ifo.power_spectral_density_array[ifo.frequency_mask][ifo_idxs],
quadratic_matrix[roq_idxs].real, 1 / ifo.strain_data.duration)
logger.info("Finished building weights for {}".format(ifo.name))
if pyfftw is not None:
pyfftw.forget_wisdom()
def reset(cls):
pyfftw.forget_wisdom()
cls.save()