def main():
nchan = 8
filter_coeff = np.arange(155).astype(np.float32)
t = np.arange(500)
signal = np.sin(t/(5*np.pi)).astype(np.float32)
# signal = np.random.randn(1000).astype(np.float32)
signal = signal + 1j*signal
def filtered():
return np.zeros((int(signal.shape[0] / nchan), nchan),
dtype=np.complex64)
filterer = apply_filter_fft_alt(
signal.copy(), filter_coeff, filtered(), nchan, None)
for i in filterer:
filtered_fft = i
filterer = apply_filter(
signal.copy(), filter_coeff, filtered(), nchan, None)
for i in filterer:
filtered_no_fft = i
plt.ion()
fig, axes = plt.subplots(4, 1)
# print(np.allclose(filtered_no_fft, filtered_fft, atol=1e-2, rtol=1e-2))
for ichan in range(nchan):
fft_ichan = filtered_fft[:, ichan]
no_fft_ichan = filtered_no_fft[:, ichan]
print(np.allclose(fft_ichan, no_fft_ichan, atol=1e-3))
for ax in axes:
ax.grid(True)
mid = fft_ichan.shape[0]
axes[0].plot(np.abs(no_fft_ichan), color="r")
axes[1].plot(np.angle(no_fft_ichan), color="r")
axes[0].plot(np.abs(fft_ichan), color="g")
axes[1].plot(np.angle(fft_ichan), color="g")
xcorr = cross_correlation(fft_ichan, no_fft_ichan)
offset = np.abs(np.argmax(np.abs(xcorr)) - mid)
print(f"ichan={ichan}, offset={offset}")
axes[2].plot(np.abs(xcorr))
diff = fft_ichan - no_fft_ichan
axes[3].plot(np.abs(diff))
axes[3].plot(np.angle(diff))
input(">> ")
def front_end(self, signal):
"""
Apply front-end processes to a signal and return the output.
The front-end consists of the full ARA electronics chain (including
amplification) and signal clipping.
Parameters
----------
signal : Signal
``Signal`` object on which to apply the front-end processes.
Returns
-------
Signal
Signal processed by the antenna front end.
"""
base_signal = signal.copy()
base_signal.filter_frequencies(self.interpolate_filter,
force_real=True)
# Apply sqrt(2) for 3dB splitter for TURF, SURF
base_signal *= self.amplification / np.sqrt(2)
clip_values = lambda times: np.clip(base_signal.with_times(times).
values,
a_min=-self.amplifier_clipping,
a_max=self.amplifier_clipping)
return FunctionSignal(signal.times,
clip_values,
value_type=signal.value_type)
def find_localmax(
self,
signal,
noise_threshold=0.0, # Range: [0.0, 1.0].
jump=None,
frame_length=1024):
""" """
if not librosa_available:
print('ERROR: Error in find_localmax. Librosa not installed.')
index_list = []
return index_list
# Adjust for comparable results for low sampling rates.
if self.sampling_freq < 300000:
frame_length = int(frame_length / 2)
if jump is None:
jump = int(self.sampling_freq / 1000) # Default = 1 ms.
y = signal.copy()
if noise_threshold > 0.0:
y[(np.abs(y) < noise_threshold)] = 0.0
rmse = librosa.feature.rmse(y=y,
hop_length=jump,
frame_length=frame_length,
center=True)
locmax = librosa.util.localmax(rmse.T)
maxindexlist = [index for index, a in enumerate(locmax) if a == True]
# Original index list is related to jump length. Convert.
index_list = librosa.frames_to_samples(maxindexlist, hop_length=jump)
#
return index_list
def convert_to_PMF(signal):
copy = signal.copy()
copy -= np.min(signal)
copy /= np.sum(signal)
return copy
def analyze_spectrum(signal, npoints):
"""Computes FFT for the signal, discards the zero freq and the
above-Nyquist freqs. Auto-pads signals nonmultple of npoints,
auto-averages results from streams longer than npoints.
Thus, npoints results in npoints/2 bands.
Returns a numpy array, each element represents the raw amplitude
of a frequency band.
"""
signal = signal.copy()
if divmod(len(signal), npoints)[1] != 0:
round_up = len(signal) / npoints * npoints + npoints
signal.resize(round_up)
window = scipy.signal.hanning(npoints)
window_blocks = scipy.vstack([window for x in xrange(len(signal) / npoints)])
signal_blocks = signal.reshape((-1, npoints))
windowed_signals = signal_blocks * window_blocks
ffts = numpy.fft.rfft(windowed_signals)[:, 1:]
result = pow(abs(ffts), 2) / npoints
result = result.mean(0)
return result
def analyze_spectrum(signal, npoints):
"""Computes FFT for the signal, discards the zero freq and the
above-Nyquist freqs. Auto-pads signals nonmultple of npoints,
auto-averages results from streams longer than npoints.
Thus, npoints results in npoints/2 bands.
Returns a numpy array, each element represents the raw amplitude
of a frequency band.
"""
signal = signal.copy()
if divmod(len(signal), npoints)[1] != 0:
round_up = len(signal) / npoints * npoints + npoints
signal.resize(round_up)
window = scipy.signal.hanning(npoints)
window_blocks = scipy.vstack(
[window for x in xrange(len(signal) / npoints)])
signal_blocks = signal.reshape((-1, npoints))
windowed_signals = signal_blocks * window_blocks
ffts = numpy.fft.rfft(windowed_signals)[:, 1:]
result = pow(abs(ffts), 2) / npoints
result = result.mean(0)
return result
def deartefact_gsr(signal, neigborhood=2048):
artefacts = bad_gsr_samples(signal)
bad_samples = dilate_trues(artefacts, neigborhood)
signal = signal.copy()
rng = np.arange(len(signal))
if not np.any(bad_samples):
return signal, bad_samples
if np.all(bad_samples):
signal[:] = 0
return signal, bad_samples
valid_interp = interp1d(rng[~bad_samples], signal[~bad_samples], bounds_error=False)
signal[bad_samples] = valid_interp(rng[bad_samples])
def find_first(lst, predicate):
# This really SHOULD be in numpy
for i in xrange(len(lst)):
if predicate(lst[i]):
return i
return None
first_valid = find_first(signal, np.isfinite)
last_valid = -find_first(signal[::-1], np.isfinite)
signal[:first_valid] = signal[first_valid]
if last_valid < 0:
signal[last_valid:] = signal[last_valid-1]
return signal, bad_samples
def deartefact_gsr(signal, neigborhood=2048):
artefacts = bad_gsr_samples(signal)
bad_samples = dilate_trues(artefacts, neigborhood)
signal = signal.copy()
rng = np.arange(len(signal))
if not np.any(bad_samples):
return signal, bad_samples
if np.all(bad_samples):
signal[:] = 0
return signal, bad_samples
valid_interp = interp1d(rng[~bad_samples], signal[~bad_samples], bounds_error=False)
signal[bad_samples] = valid_interp(rng[bad_samples])
def find_first(lst, predicate):
# This really SHOULD be in numpy
for i in xrange(len(lst)):
if predicate(lst[i]):
return i
return None
first_valid = find_first(signal, np.isfinite)
last_valid = -find_first(signal[::-1], np.isfinite)
signal[:first_valid] = signal[first_valid]
if last_valid < 0:
signal[last_valid:] = signal[last_valid-1]
return signal, bad_samples
def zero_noise(signal, percent):
copy = signal.copy()
r = np.random.rand(len(copy))
# zero random elements
copy[r < percent] = 0.0
return copy
def Arrange_mean(signal, channels, diff, channel_range, reverse=False):
signal_out = signal.copy()
if reverse:
diff = -diff
for i in range(channel_range):
signal_out[channels == i] -= diff[i]
return signal_out
def clip(signal, high, low):
'''Clip a signal from above at high and from below at low.'''
s = signal.copy()
s[np.where(s > high)] = high
s[np.where(s < low)] = low
return s
def add_gausian_noise(signal, std_level):
copy = signal.copy()
std_of_signal = copy.std()
# gaussian noise
noise = np.random.normal(scale = std_of_signal * std_level, size=len(signal))
copy += noise
return copy
def sprawdzacz(signalData, w):
low = 120
high = 6000
signalData = np.array(signalData) # to numpy array
if (len(signalData.shape) > 1): # if array 2D then make it 1D
signalData = [s[0] for s in signalData]
signal = []
if w * 3 < len(signalData):
for i in range(w, w * 3):
signal.append(signalData[i])
else:
signal = signalData
signalfft = fft(signal)
signalfft = abs(signalfft)
signal = []
freqs = range(int(len(signalfft) / 2))
for i in freqs:
signal.append(signalfft[i])
if i < low or i > high:
signal[i] = 0
output = []
result = signal.copy()
output.append(signal)
for i in range(1, 8):
output.append(scipy.signal.decimate(
signal, i)) # downsampling the signal by applying filter
for j in range(len(output[i])):
result[j] = result[j] * output[i][
j] # apply filtered signal to destination array
for i in range(len(result)):
if result[i] < 1:
result[i] = 0
# plt.subplot(211)
# p1 = plt.plot(freqs, signal, '-')
# plt.yscale('log')
# plt.subplot(212)
# p2 = plt.plot(freqs, result, '-')
# plt.yscale('log')
# plt.show()
print(freqs[argmax(result, 0)])
if freqs[argmax(result, 0)] > 350:
return ("K")
else:
return ("M")
def _remove_jumps(self, signal, flag, peaks, tol):
"""
Removes the jumps described by peaks from x.
Adds a buffer of flags with radius of tol.
"""
corrected_signal = signal.copy()
flag_out = flag.copy()
for peak, _, amplitude in peaks:
corrected_signal[peak:] -= amplitude
flag_out[peak - np.int(tol):peak + np.int(tol)] = True
return corrected_signal, flag_out
def normalize_signal(signal):
original_signal = signal.copy()
modified_signal = signal.copy()
only_the_positives = signal.copy()
for index,item in enumerate(modified_signal):
if item < 0:
modified_signal[index] = -1*item
only_the_positives[index] = 0.0
min_value = min(modified_signal)
max_value = max(modified_signal)
new_signal = [0]*len(modified_signal)
for index,item in enumerate(modified_signal):
normalized_item = (float(item) - min_value)/float(max_value - min_value)
new_signal[index] = normalized_item
for index,item in enumerate(original_signal):
if item < 0:
new_signal[index] = new_signal[index]*-1
return new_signal
def transform_beat(sig, train=False):
# 前置不可或缺的步骤
# sig = resample(sig, config.target_point_num)
# # 数据增强
if train:
if np.random.randn() > 0.5: sig = scaling(sig)
if np.random.randn() > 0.5: sig = verflip(sig)
if np.random.randn() > 0.5: sig = shift(sig)
# if np.random.randn() > 0.4: sig = wavelet_db6(sig)
# if np.random.randn() > 0.5: sig = wavelet_db4(sig) # time consuming
# if np.random.randn() > 0.3: sig = wavelet_sym(sig)
# 后置不可或缺的步骤
sig = sig.transpose()
sig = torch.tensor(sig.copy(), dtype=torch.float)
return sig
def normalize(signal, bits=None):
'''
normalize to be in a given range. The default is to normalize the maximum
amplitude to be one. An optional argument allows to normalize the signal
to be within the range of a given signed integer representation of bits.
'''
s = signal.copy()
s = s / np.abs(s).max()
# if one wants to scale for bits allocated
if bits is not None:
s *= 2**(bits - 1) - 1
s = clip(s, 2**(bits - 1) - 1, -2**(bits - 1))
return s
def highpass(signal, Fs, fc=None, plot=False):
""" Filter out the really low frequencies, default is below 50Hz """
if fc is None:
fc = constants.get("fc_hp")
# have some predefined parameters
rp = 5 # minimum ripple in dB in pass-band
rs = 60 # minimum attenuation in dB in stop-band
n = 4 # order of the filter
type = "butter"
# normalized cut-off frequency
wc = 2.0 * fc / Fs
# design the filter
from scipy.signal import iirfilter, lfilter, freqz
b, a = iirfilter(n, Wn=wc, rp=rp, rs=rs, btype="highpass", ftype=type)
# plot frequency response of filter if requested
if plot:
try:
import matplotlib.pyplot as plt
except ImportError:
import warnings
warnings.warn("Matplotlib is required for plotting")
return
w, h = freqz(b, a)
plt.figure()
plt.title("Digital filter frequency response")
plt.plot(w, 20 * np.log10(np.abs(h)))
plt.title("Digital filter frequency response")
plt.ylabel("Amplitude Response [dB]")
plt.xlabel("Frequency (rad/sample)")
plt.grid()
# apply the filter
signal = lfilter(b, a, signal.copy())
return signal
def transform(sig, train=True):
# 前置不可或缺的步骤
#sig = resample(sig, config.target_point_num)
# sig_ext = np.zeros([config.target_point_num,12])
# sig = resample(sig, int(sig.shape[0]/500 * config.target_fs))
#print(sig.shape)
# if sig.shape[0] < config.target_point_num:
# sig_ext[:sig.shape[0],:] = sig
# if sig.shape[0] > config.target_point_num:
# sig_ext = sig[:config.target_point_num,:]
# sig = sig_ext
# # 数据增强
if train:
if np.random.randn() > 0.5: sig = scaling(sig)
if np.random.randn() > 0.3: sig = verflip(sig)
if np.random.randn() > 0.5: sig = shift(sig)
if np.random.randn() > 0.3:
sig = butter_bandpass_filter(sig,0.05,46,256)
# if np.random.randn() > -1:
# fi = np.random.randint(11)
# if fi % 2 == 0 and fi != 2 and fi != 0 :
# sig = wavelet_db6(sig,'db{}'.format(fi) ,8)
# else:#if fi % 2 != 0:
# if np.random.randn() > -0.5:
# sig = butter_bandpass_filter(sig,0.05,40,256)
# else:
# sig = butter_bandpass_forward_backward_filter(sig,0.05,40,256)
else:
#sig = butter_bandpass_filter(sig,0.05,46,256)
pass
# 后置不可或缺的步骤
sig = sig.transpose()
sig = torch.tensor(sig.copy(), dtype=torch.float)
return sig
def highpass(signal, Fs, fc=None, plot=False):
''' Filter out the really low frequencies, default is below 50Hz '''
if fc is None:
fc = constants.get('fc_hp')
# have some predefined parameters
rp = 5 # minimum ripple in dB in pass-band
rs = 60 # minimum attenuation in dB in stop-band
n = 4 # order of the filter
type = 'butter'
# normalized cut-off frequency
wc = 2. * fc / Fs
# design the filter
from scipy.signal import iirfilter, lfilter, freqz
b, a = iirfilter(n, Wn=wc, rp=rp, rs=rs, btype='highpass', ftype=type)
# plot frequency response of filter if requested
if (plot):
import matplotlib.pyplot as plt
w, h = freqz(b, a)
plt.figure()
plt.title('Digital filter frequency response')
plt.plot(w, 20 * np.log10(np.abs(h)))
plt.title('Digital filter frequency response')
plt.ylabel('Amplitude Response [dB]')
plt.xlabel('Frequency (rad/sample)')
plt.grid()
# apply the filter
signal = lfilter(b, a, signal.copy())
return signal
def apply_response(self,
signal,
direction=None,
polarization=None,
force_real=False):
"""
Process the complete antenna response for an incoming signal.
Processes the incoming signal according to the frequency response of
the antenna, the efficiency, and the antenna factor. May also apply the
directionality and the polarization gain depending on the provided
parameters. Subclasses may wish to overwrite this function if the
full antenna response cannot be divided nicely into the described
pieces.
Parameters
----------
signal : Signal
Incoming ``Signal`` object to process.
direction : array_like, optional
Vector denoting the direction of travel of the signal as it reaches
the antenna (in the global coordinate frame). If ``None`` no
directional response will be applied.
polarization : array_like, optional
Vector denoting the signal's polarization direction (in the global
coordinate frame). If ``None`` no polarization gain will be applied.
force_real : boolean, optional
Whether or not the frequency response should be redefined in the
negative-frequency domain to keep the values of the filtered signal
real.
Returns
-------
Signal
Processed ``Signal`` object after the complete antenna response has
been applied. Should have a ``value_type`` of ``voltage``.
Raises
------
ValueError
If the given `signal` does not have a ``value_type`` of ``voltage``
or ``field``.
See Also
--------
pyrex.Signal : Base class for time-domain signals.
"""
new_signal = signal.copy()
new_signal.value_type = Signal.Type.voltage
new_signal.filter_frequencies(self.frequency_response,
force_real=force_real)
if direction is None:
d_gain = 1
else:
# Calculate theta and phi relative to the antenna's orientation
origin = self.position - normalize(direction)
r, theta, phi = self._convert_to_antenna_coordinates(origin)
d_gain = self.directional_gain(theta=theta, phi=phi)
if polarization is None:
p_gain = 1
else:
p_gain = self.polarization_gain(normalize(polarization))
signal_factor = d_gain * p_gain * self.efficiency
if signal.value_type == Signal.Type.voltage:
pass
elif signal.value_type == Signal.Type.field:
signal_factor /= self.antenna_factor
else:
raise ValueError("Signal's value type must be either " +
"voltage or field. Given " +
str(signal.value_type))
new_signal *= signal_factor
return new_signal
def remove_noise(signal, threshold = epsilon):
denoised = signal.copy()
denoised[denoised.abs() < threshold] = 0.0
return denoised
def butterworth(signals, sampling_rate, low_pass=None, high_pass=None,
order=5, copy=False, save_memory=False):
""" Apply a low-pass, high-pass or band-pass Butterworth filter
Apply a filter to remove signal below the `low` frequency and above the
`high` frequency.
Parameters
----------
signals: numpy.ndarray (1D sequence or n_samples x n_sources)
Signals to be filtered. A signal is assumed to be a column
of `signals`.
sampling_rate: float
Number of samples per time unit (sample frequency)
low_pass: float, optional
If specified, signals above this frequency will be filtered out
(low pass). This is -3dB cutoff frequency.
high_pass: float, optional
If specified, signals below this frequency will be filtered out
(high pass). This is -3dB cutoff frequency.
order: integer, optional
Order of the Butterworth filter. When filtering signals, the
filter has a decay to avoid ringing. Increasing the order
sharpens this decay. Be aware that very high orders could lead
to numerical instability.
copy: bool, optional
If False, `signals` is modified inplace, and memory consumption is
lower than for copy=True, though computation time is higher.
Returns
-------
filtered_signals: numpy.ndarray
Signals filtered according to the parameters
"""
if low_pass is None and high_pass is None:
if copy:
return signal.copy()
else:
return signal
if low_pass is not None and high_pass is not None \
and high_pass >= low_pass:
raise ValueError(
"High pass cutoff frequency (%f) is greater or equal"
"to low pass filter frequency (%f). This case is not handled "
"by this function."
% (high_pass, low_pass))
nyq = sampling_rate * 0.5
wn = None
if low_pass is not None:
lf = low_pass / nyq
btype = 'low'
wn = lf
if high_pass is not None:
hf = high_pass / nyq
btype = 'high'
wn = hf
if low_pass is not None and high_pass is not None:
btype = 'band'
wn = [hf, lf]
b, a = signal.butter(order, wn, btype=btype)
if signals.ndim == 1:
# 1D case
output = signal.lfilter(b, a, signals)
if copy: # lfilter does a copy in all cases.
signals = output
else:
signals[...] = output
else:
if copy:
# No way to save memory when a copy has been requested,
# because lfilter does out-of-place processing
signals = signal.lfilter(b, a, signals, axis=0)
else:
# Lesser memory consumption, slower.
for timeseries in signals.T:
timeseries[:] = signal.lfilter(b, a, timeseries)
return signals
def filter_artefacts(signal):
signal = signal.copy()
signal[signal > 200] = np.nan
return signal
def butterworth(signals, sampling_rate, low_pass=None, high_pass=None,
order=5, copy=False, save_memory=False):
""" Apply a low-pass, high-pass or band-pass Butterworth filter
Apply a filter to remove signal below the `low` frequency and above the
`high` frequency.
Parameters
----------
signals: numpy.ndarray (1D sequence or n_samples x n_sources)
Signals to be filtered. A signal is assumed to be a column
of `signals`.
sampling_rate: float
Number of samples per time unit (sample frequency)
low_pass: float, optional
If specified, signals above this frequency will be filtered out
(low pass). This is -3dB cutoff frequency.
high_pass: float, optional
If specified, signals below this frequency will be filtered out
(high pass). This is -3dB cutoff frequency.
order: integer, optional
Order of the Butterworth filter. When filtering signals, the
filter has a decay to avoid ringing. Increasing the order
sharpens this decay. Be aware that very high orders could lead
to numerical instability.
copy: bool, optional
If False, `signals` is modified inplace, and memory consumption is
lower than for copy=True, though computation time is higher.
Returns
-------
filtered_signals: numpy.ndarray
Signals filtered according to the parameters
"""
if low_pass is None and high_pass is None:
if copy:
return signal.copy()
else:
return signal
if low_pass is not None and high_pass is not None \
and high_pass >= low_pass:
raise ValueError(
"High pass cutoff frequency (%f) is greater or equal"
"to low pass filter frequency (%f). This case is not handled "
"by this function."
% (high_pass, low_pass))
nyq = sampling_rate * 0.5
wn = None
if low_pass is not None:
lf = low_pass / nyq
btype = 'low'
wn = lf
if high_pass is not None:
hf = high_pass / nyq
btype = 'high'
wn = hf
if low_pass is not None and high_pass is not None:
btype = 'band'
wn = [hf, lf]
b, a = signal.butter(order, wn, btype=btype)
if signals.ndim == 1:
# 1D case
output = signal.lfilter(b, a, signals)
if copy: # lfilter does a copy in all cases.
signals = output
else:
signals[...] = output
else:
if copy:
# No way to save memory when a copy has been requested,
# because lfilter does out-of-place processing
signals = signal.lfilter(b, a, signals, axis=0)
else:
# Lesser memory consumption, slower.
for timeseries in signals.T:
timeseries[:] = signal.lfilter(b, a, timeseries)
return signals
def correct(self,
signal,
flag,
phase,
dark=False,
signal_estimate=None,
signal_estimate_is_binned=True):
"""
Perform jump correction on given signal. Inputs:
signal -- demodulated and gap-filled signal to be corrected
(masked array)
phase -- spin phase in RADIANS
dark -- enable dark bolometer mode (disable signal subtraction)
signal_estimate(None) -- estimate of the total sky emission in
the same units as signal
"""
corrected_signal = signal.copy()
flag_out = flag.copy()
step_filter = self._get_stepfilter(self.filterlen)
amplitudes = []
positions = self._find_gaps(flag)
njump = 0
while True:
cleaned_signal = corrected_signal.copy()
good = flag_out == 0
if not dark:
signal_estimate_is_binned = self._subtract_signal(
signal_estimate, signal, good, phase, cleaned_signal)
else:
signal_estimate_is_binned = False
polyfilter(
self.order,
flag_out.astype(np.uint8),
[cleaned_signal],
np.array([0]),
np.array([signal.size]),
)
self._fill_gaps(good, cleaned_signal)
filtered_signal = self._apply_filter(cleaned_signal, step_filter)
self._suppress_gaps(filtered_signal, positions)
# find the peaks in the filtered TOI and correct for the
# jumps accordingly
peaks = self._find_peaks(filtered_signal,
flag,
flag_out,
lim=self.threshold,
tol=self.filterlen // 2)
if len(peaks) == 0:
break
njump += len(peaks)
if njump > 10:
# Prevent indefinite iterations. 10 jumps is too much anyways.
break
if not dark and signal_estimate_is_binned:
self._correct_for_signal_subtraction(good, peaks, phase,
len(signal))
corrected_signal, flag_out = self._remove_jumps(
corrected_signal, flag_out, peaks, self.tol)
for peak, _, amplitude in peaks:
positions.append(peak)
amplitudes.append(amplitude)
return corrected_signal, flag_out, njump
def filter_artefacts(signal): signal = signal.copy() signal[signal > 200] = np.nan return signal
def listen_signal(signal: np.ndarray,
Fs: int = 44100,
normalize: bool = True) -> None:
if normalize:
signal = signal.copy() / np.amax(np.absolute(signal))
display(Audio(signal, rate=Fs))
