def test_python_speed(self, dataset, request): """ test that findpeaks works on each of the arrays and print the time it took to run. Tests passes if no error is raised""" print('starting find_peak profiling on: ' + request.node.name) threshold = dataset[2] print("Threshold: {0}".format(threshold)) # Run the prototype and check that is gets the same results! proto_peaks = time_func(find_peaks_compiled, "find_peaks_compiled", arr=dataset[0], thresh=threshold, trig_int=self.trig_int) print('Found %i peaks' % len(proto_peaks)) for peak in proto_peaks: assert abs(peak[0]) > threshold assert np.allclose(peak[0], dataset[0][peak[1]], atol=0.001) assert peak[1] in dataset[1] assert len(proto_peaks) <= len(dataset[1]) peaks = time_func(find_peaks2_short, "find_peaks2_short", arr=dataset[0], thresh=threshold, trig_int=self.trig_int, full_peaks=True) print('Found %i peaks' % len(peaks)) assert len(peaks) <= len(dataset[1]) for peak in peaks: assert abs(peak[0]) > threshold # Assert peak is above threshold assert peak[0] == dataset[0][peak[1]] assert peak[1] in dataset[1] # Assert peak in expected peaks # Assert that the correct value is given! assert len(proto_peaks) <= len(dataset[1])
def real_stream_cc_output_dict(real_templates, real_multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ out = {} fft_len = next_fast_len(real_templates[0][0].stats.npts + real_multichannel_stream[0].stats.npts + 1) short_fft_len = 2**8 for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: print("Running {0} with {1} cores".format(name, cores)) cc_out = time_func(func, name, real_templates, real_multichannel_stream, cores=cores, fft_len=short_fft_len) out["{0}.{1}".format(name, cores)] = cc_out if "fftw" in name: print("Running fixed fft-len: {0}".format(fft_len)) # Make sure that running with a pre-defined fft-len works cc_out = time_func(func, name, real_templates, real_multichannel_stream, cores=cores, fft_len=fft_len) out["{0}.{1}_fixed_fft".format(name, cores)] = cc_out return out
def array_ccs_low_amp(array_template, array_stream, pads): """ Use each function stored in the normxcorr cache to correlate the templates and arrays, return a dict with keys as func names and values as the cc calculated by said function. This specifically tests low amplitude streams as raised in issue #181.""" out = {} arr_stream = array_stream * 10e-8 for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()): func = corr.get_array_xcorr(name) print("Running {0} with low-variance".format(name)) _log_handler.reset() cc, _ = time_func(func, name, array_template, arr_stream, pads) out[name] = (cc, copy.deepcopy(log_messages['warning'])) if "fftw" in name: print("Running fixed len fft") _log_handler.reset() fft_len = next_fast_len( max(len(array_stream) // 4, len(array_template))) cc, _ = time_func(func, name, array_template, array_stream, pads, fft_len=fft_len) out[name + "_fixed_len"] = (cc, copy.deepcopy(log_messages['warning'])) return out
def test_python_speed(self, dataset_1d, request): """ test that findpeaks works on each of the arrays and print the time it took to run. Tests passes if no error is raised""" print('starting find_peak profiling on: ' + request.node.name) threshold = dataset_1d[2] print("Threshold: {0}".format(threshold)) peaks = time_func(find_peaks2_short, "find_peaks2_short", arr=dataset_1d[0], thresh=threshold, trig_int=600) print('Found %i peaks' % len(peaks)) assert len(peaks) <= len(dataset_1d[1]) for peak in peaks: assert abs(peak[0]) > threshold assert peak[1] in dataset_1d[1] # Run the prototype and check that is gets the same results! proto_peaks = time_func(find_peaks_compiled, "find_peaks_compiled", arr=dataset_1d[0], thresh=threshold, trig_int=600) print('Found %i peaks' % len(proto_peaks)) for peak in proto_peaks: assert abs(peak[0]) > threshold assert peak[1] in dataset_1d[1] assert len(proto_peaks) <= len(dataset_1d[1])
def gappy_real_cc_output_dict_unstacked(gappy_real_data_template, gappy_real_data): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) import warnings for tr in gappy_real_data: tr.data = tr.data[0:unstacked_stream_len] out = {} for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: print("Running {0} with {1} cores".format(name, cores)) with warnings.catch_warnings(record=True) as w: cc_out = time_func(func, name, gappy_real_data_template, gappy_real_data, cores=cores, stack=False) out["{0}.{1}".format(name, cores)] = (cc_out, w) if "fftw" in name and cores > 1: print("Running outer core parallel") # Make sure that using both parallel methods gives the same # result with warnings.catch_warnings(record=True) as w: cc_out = time_func(func, name, gappy_real_data_template, gappy_real_data, cores=1, cores_outer=cores, stack=False) out["{0}.{1}_outer".format(name, cores)] = (cc_out, w) return out
def stream_cc_output_dict_unstacked(multichannel_templates, multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) for tr in multichannel_stream: tr.data = tr.data[0:unstacked_stream_len] multichannel_templates = multichannel_templates[0:5] out = {} for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: print("Running {0} with {1} cores".format(name, cores)) cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=cores, stack=False) out["{0}.{1}".format(name, cores)] = cc_out if "fftw" in name and cores > 1: print("Running outer core parallel") # Make sure that using both parallel methods gives the same # result cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=1, cores_outer=cores, stack=False) out["{0}.{1}_outer".format(name, cores)] = cc_out return out
def test_noisy_timings(self, noisy_multi_array): arr = noisy_multi_array.astype(np.float32) threshold = [np.float32(np.median(np.abs(d))) for d in noisy_multi_array] print("Running serial loop") serial_peaks = time_func( multi_find_peaks, name="serial", arr=arr, thresh=threshold, trig_int=600, parallel=False) print("Running parallel loop") parallel_peaks = time_func( multi_find_peaks, name="parallel", arr=arr, thresh=threshold, trig_int=600, parallel=True) assert len(serial_peaks) == len(parallel_peaks) for i in range(len(serial_peaks)): assert serial_peaks[i] == parallel_peaks[i]
def test_noisy_timings(self, noisy_multi_array): threshold = [np.median(np.abs(d)) for d in noisy_multi_array] print("Running serial loop") serial_peaks = time_func(multi_find_peaks, name="serial", arr=noisy_multi_array, thresh=threshold, trig_int=600, parallel=False) print("Running parallel loop") parallel_peaks = time_func(multi_find_peaks, name="parallel", arr=noisy_multi_array, thresh=threshold, trig_int=600, parallel=True) assert serial_peaks == parallel_peaks
def gappy_real_cc_output_dict(gappy_real_data_template, gappy_real_data): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) out = {} for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: _log_handler.reset() print("Running {0} with {1} cores".format(name, cores)) cc_out = time_func(func, name, gappy_real_data_template, gappy_real_data, cores=cores) out["{0}.{1}".format( name, cores)] = (cc_out, copy.deepcopy(log_messages['warning'])) if "fftw" in name: if cores > 1: _log_handler.reset() print("Running outer core parallel") # Make sure that using both parallel methods gives the same # result cc_out = time_func(func, name, gappy_real_data_template, gappy_real_data, cores=1, cores_outer=cores) out["{0}.{1}_outer".format( name, cores)] = (cc_out, copy.deepcopy(log_messages['warning'])) _log_handler.reset() print("Running shorter, fixed fft-len") # Make sure that running with a pre-defined fft-len works cc_out = time_func(func, name, gappy_real_data_template, gappy_real_data, cores=cores, fft_len=fft_len) out["{0}.{1}_fixed_fft".format( name, cores)] = (cc_out, copy.deepcopy(log_messages['warning'])) return out
def stream_cc_output_dict(multichannel_templates, multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) out = {} for name, func in stream_funcs.items(): cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=1) out[name] = cc_out return out
def gappy_stream_cc_output_dict( multichannel_templates, gappy_multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) out = {} for name, func in stream_funcs.items(): with warnings.catch_warnings(record=True) as w: cc_out = time_func(func, name, multichannel_templates, gappy_multichannel_stream, cores=1) out[name] = (cc_out, w) return out
def array_ccs(array_template, array_stream, pads): """ Use each function stored in the normxcorr cache to correlate the templates and arrays, return a dict with keys as func names and values as the cc calculated by said function""" out = {} for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()): func = corr.get_array_xcorr(name) print("Running %s" % name) cc, _ = time_func(func, name, array_template, array_stream, pads) out[name] = cc return out
def test_multi_find_peaks(self, dataset_2d, request): """ ensure the same results are returned for serial and parallel in multi_find_peaks """ print('starting find_peak profiling on: ' + request.node.name) arr = dataset_2d threshold = [10 * np.median(np.abs(x)) for x in dataset_2d] print("Running serial loop") serial_peaks = time_func(multi_find_peaks, name="serial", arr=arr, thresh=threshold, trig_int=600, parallel=False) print("Running parallel loop") parallel_peaks = time_func(multi_find_peaks, name="parallel", arr=arr, thresh=threshold, trig_int=600, parallel=True) assert serial_peaks == parallel_peaks
def array_ccs_low_amp(array_template, array_stream, pads): """ Use each function stored in the normxcorr cache to correlate the templates and arrays, return a dict with keys as func names and values as the cc calculated by said function. This specifically tests low amplitude streams as raised in issue #181.""" out = {} for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()): func = corr.get_array_xcorr(name) print("Running %s" % name) cc, _ = time_func(func, name, array_template, array_stream * 10e-8, pads) out[name] = cc return out
def test_python_speed(self, dataset_1d, request): """ test that findpeaks works on each of the arrays and print the time it took to run. Tests passes if no error is raised""" print('starting find_peak profiling on: ' + request.node.name) threshold = np.median(dataset_1d) # get threshold peaks = time_func(find_peaks2_short, "find_peaks2_short", arr=dataset_1d, thresh=threshold, trig_int=600, debug=0, starttime=False, samp_rate=100) print('Found %i peaks' % len(peaks))
def array_ccs(array_template, array_stream, pads): """ Use each function stored in the normxcorr cache to correlate the templates and arrays, return a dict with keys as func names and values as the cc calculated by said function""" out = {} for name in list(corr.XCORR_FUNCS_ORIGINAL.keys()): func = corr.get_array_xcorr(name) print("Running %s" % name) cc, _ = time_func(func, name, array_template, array_stream, pads) out[name] = cc if "fftw" in name: print("Running fixed len fft") fft_len = next_fast_len( max(len(array_stream) // 4, len(array_template))) cc, _ = time_func(func, name, array_template, array_stream, pads, fft_len=fft_len) out[name + "_fixed_len"] = cc return out
def gappy_stream_cc_output_dict( multichannel_templates, gappy_multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) out = {} for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: # Check for same result both single and multi-threaded print("Running {0} with {1} cores".format(name, cores)) with warnings.catch_warnings(record=True) as w: cc_out = time_func(func, name, multichannel_templates, gappy_multichannel_stream, cores=cores) out["{0}.{1}".format(name, cores)] = (cc_out, w) if "fftw" in name and cores > 1: print("Running outer core parallel") # Make sure that using both parallel methods gives the same # result with warnings.catch_warnings(record=True) as w: cc_out = time_func( func, name, multichannel_templates, gappy_multichannel_stream, cores=1, cores_outer=cores) out["{0}.{1}_outer".format(name, cores)] = (cc_out, w) return out
def stream_cc_output_dict(multichannel_templates, multichannel_stream): """ return a dict of outputs from all stream_xcorr functions """ # corr._get_array_dicts(multichannel_templates, multichannel_stream) out = {} for name, func in stream_funcs.items(): for cores in [1, cpu_count()]: print("Running {0} with {1} cores".format(name, cores)) cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=cores) out["{0}.{1}".format(name, cores)] = cc_out if "fftw" in name: if cores > 1: print("Running outer core parallel") # Make sure that using both parallel methods gives the same # result cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=1, cores_outer=cores) out["{0}.{1}_outer".format(name, cores)] = cc_out print("Running fixed fft-len: {0}".format(fft_len)) # Make sure that running with a pre-defined fft-len works cc_out = time_func(func, name, multichannel_templates, multichannel_stream, cores=cores, fft_len=fft_len) out["{0}.{1}_fixed_fft".format(name, cores)] = cc_out return out
def test_multi_find_peaks(self, dataset_2d, request): """ ensure the same results are returned for serial and parallel in multi_find_peaks """ print('starting find_peak profiling on: ' + request.node.name) arr = dataset_2d.astype(np.float32) threshold = [np.float32(10 * np.median(np.abs(x))) for x in arr] print("Running serial C loop") serial_c_peaks = time_func(multi_find_peaks, name="serial-C", arr=arr, thresh=threshold, trig_int=600, parallel=False, internal_func=find_peaks_compiled) print("Running parallel C loop") parallel_c_peaks = time_func(multi_find_peaks, name="parallel-C", arr=arr, thresh=threshold, trig_int=600, parallel=True, internal_func=find_peaks_compiled) print("Running serial Python loop") serial_py_peaks = time_func(multi_find_peaks, name="serial-Python", arr=arr, thresh=threshold, trig_int=600, parallel=False, internal_func=find_peaks2_short, full_peaks=True) print("Running parallel Python loop") parallel_py_peaks = time_func(multi_find_peaks, name="parallel-Python", arr=arr, thresh=threshold, trig_int=600, parallel=True, internal_func=find_peaks2_short, full_peaks=True) assert serial_py_peaks == parallel_py_peaks if not serial_c_peaks == parallel_c_peaks and self.DEBUG: for _serial_c_peaks, _parallel_c_peaks in zip( serial_c_peaks, parallel_c_peaks): for peak in _serial_c_peaks: if peak not in _parallel_c_peaks: print("Peak in serial but not parallel: {0}".format( peak)) for peak in _parallel_c_peaks: if peak not in _serial_c_peaks: print("Peak in parallel but not serial: {0}".format( peak)) # Test the first step parallel_peak_vals, parallel_peak_indices = _multi_find_peaks_c( arrays=arr, thresholds=threshold, threads=2) parallel_sorted = [] parallel_peaks_sorted = [] parallel_indices_sorted = [] for _peaks, _indices in zip(parallel_peak_vals, parallel_peak_indices): if len(_peaks) == 0: parallel_sorted.append([]) continue _peaks_sort = sorted(zip(_peaks, _indices), key=lambda amplitude: abs(amplitude[0]), reverse=True) parallel_sorted.append(_peaks_sort) _arr, _ind = zip(*_peaks_sort) parallel_peaks_sorted.extend(_arr) parallel_indices_sorted.extend(_ind) serial_peak_vals, serial_peak_indices, serial_sorted = ([], [], []) for sub_arr, thresh in zip(arr, threshold): _peak_vals, _peak_indices = _find_peaks_c(array=sub_arr, threshold=thresh) serial_peak_vals.append(_peak_vals) serial_peak_indices.append(_peak_indices) _peaks_sort = sorted(zip(_peak_vals, _peak_indices), key=lambda amplitude: abs(amplitude[0]), reverse=True) serial_sorted.append(_peaks_sort) for i in range(len(serial_peak_vals)): if parallel_peak_vals[i].size > 0: assert np.all(parallel_peak_vals[i] == serial_peak_vals[i]) assert np.all( parallel_peak_indices[i] == serial_peak_indices[i]) assert parallel_sorted == serial_sorted else: assert serial_peak_vals[i].size == 0 np.save("test_2d_array.npy", arr) assert serial_c_peaks == parallel_c_peaks for i in range(len(serial_c_peaks)): diff_count = 0 for j in range(len(serial_c_peaks[i])): if not serial_c_peaks[i][j] in serial_py_peaks[i]: diff_count += 1 print("Peak {0} in C but not in py".format( serial_c_peaks[i][j])) for j in range(len(serial_py_peaks[i])): if not serial_py_peaks[i][j] in serial_c_peaks[i]: diff_count += 1 print("Peak {0} in py but not in C".format( serial_py_peaks[i][j])) assert diff_count <= 0.0001 * self.data_len