def test_python_speed(self, dataset, request):
""" test that findpeaks works on each of the arrays and print the
time it took to run. Tests passes if no error is raised"""
print('starting find_peak profiling on: ' + request.node.name)
threshold = dataset[2]
print("Threshold: {0}".format(threshold))
# Run the prototype and check that is gets the same results!
proto_peaks = time_func(find_peaks_compiled,
"find_peaks_compiled",
arr=dataset[0],
thresh=threshold,
trig_int=self.trig_int)
print('Found %i peaks' % len(proto_peaks))
for peak in proto_peaks:
assert abs(peak[0]) > threshold
assert np.allclose(peak[0], dataset[0][peak[1]], atol=0.001)
assert peak[1] in dataset[1]
assert len(proto_peaks) <= len(dataset[1])
peaks = time_func(find_peaks2_short,
"find_peaks2_short",
arr=dataset[0],
thresh=threshold,
trig_int=self.trig_int,
full_peaks=True)
print('Found %i peaks' % len(peaks))
assert len(peaks) <= len(dataset[1])
for peak in peaks:
assert abs(peak[0]) > threshold # Assert peak is above threshold
assert peak[0] == dataset[0][peak[1]]
assert peak[1] in dataset[1] # Assert peak in expected peaks
# Assert that the correct value is given!
assert len(proto_peaks) <= len(dataset[1])
def real_stream_cc_output_dict(real_templates, real_multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
out = {}
fft_len = next_fast_len(real_templates[0][0].stats.npts +
real_multichannel_stream[0].stats.npts + 1)
short_fft_len = 2**8
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
print("Running {0} with {1} cores".format(name, cores))
cc_out = time_func(func,
name,
real_templates,
real_multichannel_stream,
cores=cores,
fft_len=short_fft_len)
out["{0}.{1}".format(name, cores)] = cc_out
if "fftw" in name:
print("Running fixed fft-len: {0}".format(fft_len))
# Make sure that running with a pre-defined fft-len works
cc_out = time_func(func,
name,
real_templates,
real_multichannel_stream,
cores=cores,
fft_len=fft_len)
out["{0}.{1}_fixed_fft".format(name, cores)] = cc_out
return out
def array_ccs_low_amp(array_template, array_stream, pads):
""" Use each function stored in the normxcorr cache to correlate the
templates and arrays, return a dict with keys as func names and values
as the cc calculated by said function.
This specifically tests low amplitude streams as raised in issue #181."""
out = {}
arr_stream = array_stream * 10e-8
for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()):
func = corr.get_array_xcorr(name)
print("Running {0} with low-variance".format(name))
_log_handler.reset()
cc, _ = time_func(func, name, array_template, arr_stream, pads)
out[name] = (cc, copy.deepcopy(log_messages['warning']))
if "fftw" in name:
print("Running fixed len fft")
_log_handler.reset()
fft_len = next_fast_len(
max(len(array_stream) // 4, len(array_template)))
cc, _ = time_func(func,
name,
array_template,
array_stream,
pads,
fft_len=fft_len)
out[name + "_fixed_len"] = (cc,
copy.deepcopy(log_messages['warning']))
return out
def test_python_speed(self, dataset_1d, request):
""" test that findpeaks works on each of the arrays and print the
time it took to run. Tests passes if no error is raised"""
print('starting find_peak profiling on: ' + request.node.name)
threshold = dataset_1d[2]
print("Threshold: {0}".format(threshold))
peaks = time_func(find_peaks2_short,
"find_peaks2_short",
arr=dataset_1d[0],
thresh=threshold,
trig_int=600)
print('Found %i peaks' % len(peaks))
assert len(peaks) <= len(dataset_1d[1])
for peak in peaks:
assert abs(peak[0]) > threshold
assert peak[1] in dataset_1d[1]
# Run the prototype and check that is gets the same results!
proto_peaks = time_func(find_peaks_compiled,
"find_peaks_compiled",
arr=dataset_1d[0],
thresh=threshold,
trig_int=600)
print('Found %i peaks' % len(proto_peaks))
for peak in proto_peaks:
assert abs(peak[0]) > threshold
assert peak[1] in dataset_1d[1]
assert len(proto_peaks) <= len(dataset_1d[1])
def gappy_real_cc_output_dict_unstacked(gappy_real_data_template,
gappy_real_data):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
import warnings
for tr in gappy_real_data:
tr.data = tr.data[0:unstacked_stream_len]
out = {}
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
print("Running {0} with {1} cores".format(name, cores))
with warnings.catch_warnings(record=True) as w:
cc_out = time_func(func,
name,
gappy_real_data_template,
gappy_real_data,
cores=cores,
stack=False)
out["{0}.{1}".format(name, cores)] = (cc_out, w)
if "fftw" in name and cores > 1:
print("Running outer core parallel")
# Make sure that using both parallel methods gives the same
# result
with warnings.catch_warnings(record=True) as w:
cc_out = time_func(func,
name,
gappy_real_data_template,
gappy_real_data,
cores=1,
cores_outer=cores,
stack=False)
out["{0}.{1}_outer".format(name, cores)] = (cc_out, w)
return out
def stream_cc_output_dict_unstacked(multichannel_templates,
multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
for tr in multichannel_stream:
tr.data = tr.data[0:unstacked_stream_len]
multichannel_templates = multichannel_templates[0:5]
out = {}
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
print("Running {0} with {1} cores".format(name, cores))
cc_out = time_func(func,
name,
multichannel_templates,
multichannel_stream,
cores=cores,
stack=False)
out["{0}.{1}".format(name, cores)] = cc_out
if "fftw" in name and cores > 1:
print("Running outer core parallel")
# Make sure that using both parallel methods gives the same
# result
cc_out = time_func(func,
name,
multichannel_templates,
multichannel_stream,
cores=1,
cores_outer=cores,
stack=False)
out["{0}.{1}_outer".format(name, cores)] = cc_out
return out
def test_noisy_timings(self, noisy_multi_array):
arr = noisy_multi_array.astype(np.float32)
threshold = [np.float32(np.median(np.abs(d)))
for d in noisy_multi_array]
print("Running serial loop")
serial_peaks = time_func(
multi_find_peaks, name="serial", arr=arr,
thresh=threshold, trig_int=600, parallel=False)
print("Running parallel loop")
parallel_peaks = time_func(
multi_find_peaks, name="parallel", arr=arr,
thresh=threshold, trig_int=600, parallel=True)
assert len(serial_peaks) == len(parallel_peaks)
for i in range(len(serial_peaks)):
assert serial_peaks[i] == parallel_peaks[i]
def test_noisy_timings(self, noisy_multi_array):
threshold = [np.median(np.abs(d)) for d in noisy_multi_array]
print("Running serial loop")
serial_peaks = time_func(multi_find_peaks,
name="serial",
arr=noisy_multi_array,
thresh=threshold,
trig_int=600,
parallel=False)
print("Running parallel loop")
parallel_peaks = time_func(multi_find_peaks,
name="parallel",
arr=noisy_multi_array,
thresh=threshold,
trig_int=600,
parallel=True)
assert serial_peaks == parallel_peaks
def gappy_real_cc_output_dict(gappy_real_data_template, gappy_real_data):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
out = {}
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
_log_handler.reset()
print("Running {0} with {1} cores".format(name, cores))
cc_out = time_func(func,
name,
gappy_real_data_template,
gappy_real_data,
cores=cores)
out["{0}.{1}".format(
name,
cores)] = (cc_out, copy.deepcopy(log_messages['warning']))
if "fftw" in name:
if cores > 1:
_log_handler.reset()
print("Running outer core parallel")
# Make sure that using both parallel methods gives the same
# result
cc_out = time_func(func,
name,
gappy_real_data_template,
gappy_real_data,
cores=1,
cores_outer=cores)
out["{0}.{1}_outer".format(
name,
cores)] = (cc_out,
copy.deepcopy(log_messages['warning']))
_log_handler.reset()
print("Running shorter, fixed fft-len")
# Make sure that running with a pre-defined fft-len works
cc_out = time_func(func,
name,
gappy_real_data_template,
gappy_real_data,
cores=cores,
fft_len=fft_len)
out["{0}.{1}_fixed_fft".format(
name,
cores)] = (cc_out, copy.deepcopy(log_messages['warning']))
return out
def stream_cc_output_dict(multichannel_templates, multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
out = {}
for name, func in stream_funcs.items():
cc_out = time_func(func, name, multichannel_templates,
multichannel_stream, cores=1)
out[name] = cc_out
return out
def gappy_stream_cc_output_dict(
multichannel_templates, gappy_multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
out = {}
for name, func in stream_funcs.items():
with warnings.catch_warnings(record=True) as w:
cc_out = time_func(func, name, multichannel_templates,
gappy_multichannel_stream, cores=1)
out[name] = (cc_out, w)
return out
def array_ccs(array_template, array_stream, pads):
""" Use each function stored in the normxcorr cache to correlate the
templates and arrays, return a dict with keys as func names and values
as the cc calculated by said function"""
out = {}
for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()):
func = corr.get_array_xcorr(name)
print("Running %s" % name)
cc, _ = time_func(func, name, array_template, array_stream, pads)
out[name] = cc
return out
def test_multi_find_peaks(self, dataset_2d, request):
""" ensure the same results are returned for serial and parallel
in multi_find_peaks """
print('starting find_peak profiling on: ' + request.node.name)
arr = dataset_2d
threshold = [10 * np.median(np.abs(x)) for x in dataset_2d]
print("Running serial loop")
serial_peaks = time_func(multi_find_peaks,
name="serial",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=False)
print("Running parallel loop")
parallel_peaks = time_func(multi_find_peaks,
name="parallel",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=True)
assert serial_peaks == parallel_peaks
def array_ccs_low_amp(array_template, array_stream, pads):
""" Use each function stored in the normxcorr cache to correlate the
templates and arrays, return a dict with keys as func names and values
as the cc calculated by said function.
This specifically tests low amplitude streams as raised in issue #181."""
out = {}
for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()):
func = corr.get_array_xcorr(name)
print("Running %s" % name)
cc, _ = time_func(func, name, array_template, array_stream * 10e-8,
pads)
out[name] = cc
return out
def test_python_speed(self, dataset_1d, request):
""" test that findpeaks works on each of the arrays and print the
time it took to run. Tests passes if no error is raised"""
print('starting find_peak profiling on: ' + request.node.name)
threshold = np.median(dataset_1d) # get threshold
peaks = time_func(find_peaks2_short,
"find_peaks2_short",
arr=dataset_1d,
thresh=threshold,
trig_int=600,
debug=0,
starttime=False,
samp_rate=100)
print('Found %i peaks' % len(peaks))
def array_ccs(array_template, array_stream, pads):
""" Use each function stored in the normxcorr cache to correlate the
templates and arrays, return a dict with keys as func names and values
as the cc calculated by said function"""
out = {}
for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()):
func = corr.get_array_xcorr(name)
print("Running %s" % name)
cc, _ = time_func(func, name, array_template, array_stream, pads)
out[name] = cc
if "fftw" in name:
print("Running fixed len fft")
fft_len = next_fast_len(
max(len(array_stream) // 4, len(array_template)))
cc, _ = time_func(func,
name,
array_template,
array_stream,
pads,
fft_len=fft_len)
out[name + "_fixed_len"] = cc
return out
def gappy_stream_cc_output_dict(
multichannel_templates, gappy_multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
out = {}
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
# Check for same result both single and multi-threaded
print("Running {0} with {1} cores".format(name, cores))
with warnings.catch_warnings(record=True) as w:
cc_out = time_func(func, name, multichannel_templates,
gappy_multichannel_stream, cores=cores)
out["{0}.{1}".format(name, cores)] = (cc_out, w)
if "fftw" in name and cores > 1:
print("Running outer core parallel")
# Make sure that using both parallel methods gives the same
# result
with warnings.catch_warnings(record=True) as w:
cc_out = time_func(
func, name, multichannel_templates,
gappy_multichannel_stream, cores=1, cores_outer=cores)
out["{0}.{1}_outer".format(name, cores)] = (cc_out, w)
return out
def stream_cc_output_dict(multichannel_templates, multichannel_stream):
""" return a dict of outputs from all stream_xcorr functions """
# corr._get_array_dicts(multichannel_templates, multichannel_stream)
out = {}
for name, func in stream_funcs.items():
for cores in [1, cpu_count()]:
print("Running {0} with {1} cores".format(name, cores))
cc_out = time_func(func,
name,
multichannel_templates,
multichannel_stream,
cores=cores)
out["{0}.{1}".format(name, cores)] = cc_out
if "fftw" in name:
if cores > 1:
print("Running outer core parallel")
# Make sure that using both parallel methods gives the same
# result
cc_out = time_func(func,
name,
multichannel_templates,
multichannel_stream,
cores=1,
cores_outer=cores)
out["{0}.{1}_outer".format(name, cores)] = cc_out
print("Running fixed fft-len: {0}".format(fft_len))
# Make sure that running with a pre-defined fft-len works
cc_out = time_func(func,
name,
multichannel_templates,
multichannel_stream,
cores=cores,
fft_len=fft_len)
out["{0}.{1}_fixed_fft".format(name, cores)] = cc_out
return out
def test_multi_find_peaks(self, dataset_2d, request):
""" ensure the same results are returned for serial and parallel
in multi_find_peaks """
print('starting find_peak profiling on: ' + request.node.name)
arr = dataset_2d.astype(np.float32)
threshold = [np.float32(10 * np.median(np.abs(x))) for x in arr]
print("Running serial C loop")
serial_c_peaks = time_func(multi_find_peaks,
name="serial-C",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=False,
internal_func=find_peaks_compiled)
print("Running parallel C loop")
parallel_c_peaks = time_func(multi_find_peaks,
name="parallel-C",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=True,
internal_func=find_peaks_compiled)
print("Running serial Python loop")
serial_py_peaks = time_func(multi_find_peaks,
name="serial-Python",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=False,
internal_func=find_peaks2_short,
full_peaks=True)
print("Running parallel Python loop")
parallel_py_peaks = time_func(multi_find_peaks,
name="parallel-Python",
arr=arr,
thresh=threshold,
trig_int=600,
parallel=True,
internal_func=find_peaks2_short,
full_peaks=True)
assert serial_py_peaks == parallel_py_peaks
if not serial_c_peaks == parallel_c_peaks and self.DEBUG:
for _serial_c_peaks, _parallel_c_peaks in zip(
serial_c_peaks, parallel_c_peaks):
for peak in _serial_c_peaks:
if peak not in _parallel_c_peaks:
print("Peak in serial but not parallel: {0}".format(
peak))
for peak in _parallel_c_peaks:
if peak not in _serial_c_peaks:
print("Peak in parallel but not serial: {0}".format(
peak))
# Test the first step
parallel_peak_vals, parallel_peak_indices = _multi_find_peaks_c(
arrays=arr, thresholds=threshold, threads=2)
parallel_sorted = []
parallel_peaks_sorted = []
parallel_indices_sorted = []
for _peaks, _indices in zip(parallel_peak_vals,
parallel_peak_indices):
if len(_peaks) == 0:
parallel_sorted.append([])
continue
_peaks_sort = sorted(zip(_peaks, _indices),
key=lambda amplitude: abs(amplitude[0]),
reverse=True)
parallel_sorted.append(_peaks_sort)
_arr, _ind = zip(*_peaks_sort)
parallel_peaks_sorted.extend(_arr)
parallel_indices_sorted.extend(_ind)
serial_peak_vals, serial_peak_indices, serial_sorted = ([], [], [])
for sub_arr, thresh in zip(arr, threshold):
_peak_vals, _peak_indices = _find_peaks_c(array=sub_arr,
threshold=thresh)
serial_peak_vals.append(_peak_vals)
serial_peak_indices.append(_peak_indices)
_peaks_sort = sorted(zip(_peak_vals, _peak_indices),
key=lambda amplitude: abs(amplitude[0]),
reverse=True)
serial_sorted.append(_peaks_sort)
for i in range(len(serial_peak_vals)):
if parallel_peak_vals[i].size > 0:
assert np.all(parallel_peak_vals[i] == serial_peak_vals[i])
assert np.all(
parallel_peak_indices[i] == serial_peak_indices[i])
assert parallel_sorted == serial_sorted
else:
assert serial_peak_vals[i].size == 0
np.save("test_2d_array.npy", arr)
assert serial_c_peaks == parallel_c_peaks
for i in range(len(serial_c_peaks)):
diff_count = 0
for j in range(len(serial_c_peaks[i])):
if not serial_c_peaks[i][j] in serial_py_peaks[i]:
diff_count += 1
print("Peak {0} in C but not in py".format(
serial_c_peaks[i][j]))
for j in range(len(serial_py_peaks[i])):
if not serial_py_peaks[i][j] in serial_c_peaks[i]:
diff_count += 1
print("Peak {0} in py but not in C".format(
serial_py_peaks[i][j]))
assert diff_count <= 0.0001 * self.data_len
