def butter(self, time, data, **kwargs):
"""
Source:
https://azitech.wordpress.com/2011/03/15/
designing-a-butterworth-low-pass-filter-with-scipy/
"""
sample_rate = kwargs.get("sample_rate", None)
if not sample_rate:
sample_rate = calc_sample_rate(time)
# The cutoff frequency of the filter.
cutoff_hz = kwargs.get("cutoff_hz", 1.0)
# design filter
norm_pass = cutoff_hz / (sample_rate / 2.0)
norm_stop = 1.5 * norm_pass
(N, Wn) = sp.signal.buttord(wp=norm_pass, ws=norm_stop, gpass=2, gstop=30, analog=0)
(b, a) = sp.signal.butter(N, Wn, btype="low", analog=0, output="ba")
# filtered output
# zi = signal.lfiltic(b, a, x[0:5], x[0:5])
# (y, zi) = signal.lfilter(b, a, x, zi=zi)
data_filt = sp.signal.lfilter(b, a, data)
return data_filt
def __init__(self, time, freq_ds=10):
"""
Parameters
----------
time : ndarray(n)
freq_ds : float, default=0.1
Frequency in Hz of the downsampled signal. Downsampling is applied
after the low pass filter.
outputs : ndarray(m,n)
Signals corresponding to the output of the system
inputs : ndarray(m,n), default=None
Signals that corresponds to the inputs of the system. When None,
inputs are ignored and now additional overlaying is done in order
to achieve both steady curves on the in- and outputs.
"""
self.sps = calc_sample_rate(time)
self.t_ds = np.arange(time[0], time[-1], 1.0/freq_ds)
self.time = time
self.freq_ds = freq_ds
def setup_filter(self, time, data, **kwargs):
"""
Load the callibration runs and convert voltage signal to yaw angles
Parameters
----------
time : ndarray(k)
data : ndarray(k)
Returns
-------
time_stair : ndarray(n)
Average time stamp over the stair step
data_stair : ndarray(n)
Average value of the selected stair step
"""
# time and data should both be 1D and have the same shape!
assert time.shape == data.shape
runid = kwargs.get('runid', self.runid)
# smoothen method: spline or moving average
smoothen = kwargs.get('smoothen', 'spline')
# what is the window of the moving average in seconds
smooth_window = kwargs.get('smooth_window', 2)
# specify the window of the staircase
#start, end = 30100, -30001
start = kwargs.get('start', 0)
end = kwargs.get('end', len(time))
dt = kwargs.get('dt', 1)
cutoff_hz = kwargs.get('cutoff_hz', None)
self.points_per_stair = kwargs.get('points_per_stair', 20)
# at what is the minimum required value on dt or dt2 for a new stair
self.stair_step_tresh = kwargs.get('stair_step_tresh', 1)
# plot_data = kwargs.get('plot_data', False)
# respath = kwargs.get('respath', None)
# run = kwargs.get('run', None)
# sample rate of the signal
sample_rate = calc_sample_rate(time)
# prepare the data
time = time[start:end]
# the actual raw signal
data = data[start:end]
# -------------------------------------------------
# Progress plotting
# ----------------------------------------------
if self.plt_progress:
plt.figure()
Pxx, freqs = plt.psd(data, Fs=sample_rate, label='data')
plt.show()
plt.figure()
plt.plot(time, data, label='raw data')
# -------------------------------------------------
# setup plot
# -------------------------------------------------
# labels = np.ndarray(3, dtype='<U100')
# labels[0] = label
# labels[1] = 'yawchan derivative'
# labels[2] = 'psd'
# remove any underscores for latex printing
grandtitle = self.figfile.replace('_', '\_')
plot = plotting.A4Tuned(scale=1.5)
plot.setup(self.figpath+self.figfile+'_filter', nr_plots=3,
grandtitle=grandtitle, wsleft_cm=1.5, wsright_cm=1.8,
hspace_cm=1.2, size_x_perfig=10, size_y_perfig=5,
wsbottom_cm=1.0, wstop_cm=1.5)
# -------------------------------------------------
# plotting original and smoothend signal
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 1)
ax1.plot(time, data, 'b', label='raw data', alpha=0.6)
data_raw = data.copy()
# -------------------------------------------------
# signal frequency filtering, if applicable
# -------------------------------------------------
# filter the local derivatives if applicable
if cutoff_hz:
filt = Filters()
data_filt, N, delay = filt.fir(time, data, ripple_db=20,
freq_trans_width=0.5, cutoff_hz=cutoff_hz,
figpath=self.figpath,
figfile=self.figfile + 'filter_design',
sample_rate=sample_rate, plot=False,)
if self.plt_progress:
# add the results of the filtering technique
plt.plot(time[N-1:], data_filt[N-1:], 'r', label='freq filt')
data = data_filt
time = time[N-1:]#-delay
else:
N = 1
# -------------------------------------------------------
# smoothen the signal with some splines or moving average
# -------------------------------------------------------
# NOTE: the smoothing will make the transitions also smoother. This
# is not good. The edges of the stair need to be steep!
# for the binary data this is actually a good thing, since the dt's
# are almost always the same between time steps. We would otherwise
# need a dt based on several time steps
if smoothen == 'splines':
print 'start applying spline ...',
uni_spline = UnivariateSpline(time, data)
data = uni_spline(time)
print 'done!'
NN = 0 # no time shift due to filtering?
if self.plt_progress:
plt.plot(time, data, label='spline data')
elif smoothen == 'moving':
print 'start calculating movering average ...',
filt = Filters()
# take av2s window, calculate the number of samples per window
ws = int(smooth_window*sample_rate)
data = filt.smooth(data, window_len=ws, window='hanning')
NN = len(data) - len(time)
data = data[NN:]
print 'done!'
if self.plt_progress:
plt.plot(time, data, label='moving average')
else:
raise ValueError, 'smoothen method should be moving or splines'
# -------------------------------------------------
# additional smoothening: downsampling
# -------------------------------------------------
# and up again in order not to brake the plotting further down
time_down = np.arange(time[0], time[-1], 0.1)
data_down = sp.interpolate.griddata(time, data, time_down)
# and upsampling again
data = sp.interpolate.griddata(time_down, data_down, time)
# -------------------------------------------------
# plotting original and smoothend signal
# -------------------------------------------------
ax1.plot(time, data, 'r', label='data smooth')
ax1.grid(True)
leg1 = ax1.legend(loc='best')
leg1.get_frame().set_alpha(0.5)
ax1.set_title('smoothing method: ' + smoothen)
# -------------------------------------------------
# local derivatives of the signal and filtering
# -------------------------------------------------
data_dt = np.ndarray(data.shape)
data_dt[1:] = data[1:] - data[0:-1]
data_dt[0] = np.nan
data_dt = np.abs(data_dt)
# frequency filter was applied here originally
data_filt_dt = data_dt
# if no threshold is given, just take the 20% of the max value
dt_max = np.nanmax(np.abs(data_filt_dt))*0.2
dt_treshold = kwargs.get('dt_treshold', dt_max)
# -------------------------------------------------
# filter dt or dt2 above certain treshold?
# -----------------------------------------------
# only keep values which are steady, meaning dt signal is low!
if dt == 2:
tmp = np.ndarray(data_filt_dt.shape)
tmp[1:] = data_filt_dt[1:] - data_filt_dt[0:-1]
tmp[0] = np.nan
data_filt_dt = tmp
# based upon the filtering, only select data points for which the
# filtered derivative is between a certain treshold
staircase_i = np.abs(data_filt_dt).__ge__(dt_treshold)
# reduce to 1D
staircase_arg=np.argwhere(np.abs(data_filt_dt)<=dt_treshold).flatten()
# -------------------------------------------------
# replace values for too high dt with Nan
# ------------------------------------------------
# ---------------------------------
# METHOD version2, slower because of staircase_arg computation above
data_masked = data.copy()
data_masked[staircase_i] = np.nan
data_masked_dt = data_filt_dt.copy()
data_masked_dt[staircase_i] = np.nan
data_trim = data[staircase_arg]
time_trim = time[staircase_arg]
print 'max in data_masked_dt:', np.nanmax(data_masked_dt)
# ---------------------------------
# METHOD version2, faster if staircase_arg is not required!
## make a copy of the original signal and fill in Nans on the selected
## values
#data_masked = data.copy()
#data_masked[staircase_i] = np.nan
#
#data_masked_dt = data_filt_dt.copy()
#data_masked_dt[staircase_i] = np.nan
#
## remove all the nan values
#data_trim = data_masked[np.isnan(data_masked).__invert__()]
#time_trim = time[np.isnan(data_masked).__invert__()]
#
#dt_noise_treshold = np.nanmax(data_masked_dt)
#print 'max in data_masked_dt', dt_noise_treshold
# ---------------------------------
# # figure out which dt's are above the treshold
# data_trim2 = data_trim.copy()
# data_trim2.sort()
# data_trim2.
# # where the dt of the masked format is above the noise treshold,
# # we have a stair
# data_trim_dt = np.abs(data_trim[1:] - data_trim[:-1])
# argstairs = data_trim_dt.__gt__(dt_noise_treshold)
# data_trim2 = data_trim_dt.copy()
# data_trim_dt.sort()
# data_trim_dt.__gt__(dt_noise_treshold)
# -------------------------------------------------
# intermediate checking of the signal
# -------------------------------------------------
if self.plt_progress:
# add the results of the filtering technique
plt.plot(time[N-1:], data_masked[N-1:], 'rs', label='data red')
plt.legend(loc='best')
plt.grid(True)
plt.twinx()
# plt.plot(time, data_filt_dt, label='data_filt_dt')
plt.plot(time, data_masked_dt, 'm', label='data\_masked\_dt',
alpha=0.4)
plt.legend(loc='best')
plt.show()
print 'saving plt_progress:',
print self.figpath+'filter_design_progress.png'
plt.savefig(self.figpath+'filter_design_progress.png')
# -------------------------------------------------
# check if we have had sane filtering
# -------------------------------------------------
print 'data :', data.shape
print 'data_trim :', data_trim.shape
print 'trim ratio:', len(data)/len(data_trim)
# there should be at least one True value
assert staircase_i.any()
# they can't all be True, than filtering is too heavy
if len(data_trim) < len(data)*0.01:
msg = 'dt_treshold is too low, not enough data left'
raise ValueError, msg
# if no data is filtered at all, filtering is too conservative
elif len(data_trim) > len(data)*0.95:
msg = 'dt_treshold is too high, too much data left'
raise ValueError, msg
# if the data array is too big, abort on memory concerns
if len(data_trim) > 200000:
msg = 'too much data points for stair case analysis (cfr memory)'
raise ValueError, msg
# -------------------------------------------------
# read the average value over each stair (time and data)
# ------------------------------------------------
#try:
##np.save('time_trim', time_trim)
##np.save('data_trim', data_trim)
##np.save('staircase_arg', staircase_arg)
##tmp = np.array([self.points_per_stair, self.stair_step_tresh])
##np.save('tmp', tmp)
#data_ordered, time_stair, data_stair, arg_stair \
#= cython_func.order_staircase(time_trim, data_trim,
#staircase_arg, self.points_per_stair, self.stair_step_tresh)
#except ImportError:
data_ordered, time_stair, data_stair, arg_stair \
= self.order_staircase(time_trim, data_trim, staircase_arg)
# convert the arg_stair to a flat set and replace start/stop pairs
# with all indices in between. Now we can select all stair values
# in the raw dataset
arg_st_fl = np.empty(data_raw.shape, dtype=np.int)
i = 0
for k in range(arg_stair.shape[1]):
#print '%6i %6i' % (arg_stair[0,k],arg_stair[1,k])
tmp = np.arange(arg_stair[0,k], arg_stair[1,k]+1, 1, dtype=np.int)
#print tmp, '->', i, ':', i+len(tmp)
arg_st_fl[i:i+len(tmp)] = tmp
i += len(tmp)
# remove the unused elements from the array
arg_st_fl = arg_st_fl[:i]
# -------------------------------------------------
# plotting of smoothen signal and stairs
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 2)
ax1.plot(time, data, label='data smooth', alpha=0.6)
# add the results of the filtering technique
ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked')
# ax1.plot(time[N-1:], data_filt[N-1:], 'g', label='data_filt')
# also include the selected chair data
figlabel = '%i stairs' % data_stair.shape[0]
ax1.plot(time_stair, data_stair, 'ko', label=figlabel, alpha=0.4)
ax1.grid(True)
# the legend, on or off?
#leg1 = ax1.legend(loc='upper left')
#leg1.get_frame().set_alpha(0.5)
# -------------------------------------------------
# plotting derivatives on right axis
# -------------------------------------------------
ax1b = ax1.twinx()
# ax1b.plot(time[N:]-delay,data_s_dt[N:],alpha=0.2,label='data_s_dt')
ax1b.plot(time[N:], data_filt_dt[N:], 'r', alpha=0.35,
label='data\_filt\_dt')
majorFormatter = FormatStrFormatter('%8.1e')
ax1b.yaxis.set_major_formatter(majorFormatter)
# ax1b.plot(time[N:], data_masked_dt[N:], 'b', alpha=0.2,
# label='data_masked_dt')
# ax1b.plot(time[N-1:]-delay, filtered_x_dt[N-1:], alpha=0.2)
# leg1b = ax1b.legend(loc='best')
# leg1b.get_frame().set_alpha(0.5)
# ax1b.grid(True)
# -------------------------------------------------
# 3th plot to check if the raw chair signal is ok
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3)
ax1.plot(time[arg_st_fl], data_raw[arg_st_fl], 'k+', label='rawstair',
alpha=0.1)
ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked')
ax1.set_xlabel('time [s]')
# -------------------------------------------------
# the power spectral density
# -------------------------------------------------
# ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3)
# Pxx, freqs = ax3.psd(data, Fs=sample_rate, label='data smooth')
## Pxx, freqs = ax3.psd(data_dt, Fs=sample_rate, label='data_dt')
## Pxx, freqs = ax3.psd(data_filt_dt[N-1:], Fs=sample_rate,
## label='data_filt_dt')
# ax3.legend()
## print Pxx.shape, freqs.shape
plot.save_fig()
# -------------------------------------------------
# get amplitudes of the stair edges
# -------------------------------------------------
# # max step
# data_trim_dt_sort = data_trim_dt.sort()[0]
# # estimate at what kind of a delta we are looking for when changing
# # stairs
# data_dt_std = data_trim_dt.std()
# data_dt_mean = (np.abs(data_trim_dt)).mean()
#
# time_data_dt = np.transpose(np.array([time, data_filt_dt]))
# data_filt_dt_amps = HawcPy.dynprop().amplitudes(time_data_dt, h=1e-3)
#
# print '=== nr amplitudes'
# print len(data_filt_dt_amps)
# print data_filt_dt_amps
# -------------------------------------------------
# save the data
# -------------------------------------------------
filename = runid + '-time_stair'
np.savetxt(self.pprpath + filename, time_stair)
filename = runid + '-data_stair'
np.savetxt(self.pprpath + filename, data_stair)
# in order to maintain backwards compatibility, save the arguments
# of the stair to self
self.arg_st_fl = arg_st_fl # flat, contains all indices on the stairs
# start/stop indeces for stair k = arg_stair[0,k], arg_stair[1,k]
self.arg_stair = arg_stair
return time_stair, data_stair
def scipy_example(self, time, data, sample_rate=None):
"""
Example from the SciPy Cookboock, see
http://www.scipy.org/Cookbook/FIRFilter
"""
chk.array_1d(time)
chk.array_1d(data)
if not sample_rate:
sample_rate = calc_sample_rate(time)
# ------------------------------------------------
# Create a FIR filter and apply it to data[:,channel]
# ------------------------------------------------
# The Nyquist rate of the signal.
nyq_rate = sample_rate / 2.0
# The desired width of the transition from pass to stop,
# relative to the Nyquist rate. We'll design the filter
# with a 5 Hz transition width.
width = 5.0 / nyq_rate
# The desired attenuation in the stop band, in dB.
ripple_db = 60.0
# Compute the order and Kaiser parameter for the FIR filter.
N, beta = sp.signal.kaiserord(ripple_db, width)
# The cutoff frequency of the filter.
cutoff_hz = 10.0
# Use firwin with a Kaiser window to create a lowpass FIR filter.
taps = sp.signal.firwin(N, cutoff_hz / nyq_rate, window=("kaiser", beta))
# Use lfilter to filter x with the FIR filter.
filtered_x = sp.signal.lfilter(taps, 1.0, data)
# ------------------------------------------------
# Setup the figure parameters
# ------------------------------------------------
figpath = "processing/"
figfile = "filterdesign"
plot = plotting.A4Tuned()
plot.setup(figpath + figfile, nr_plots=3, grandtitle=figfile, figsize_y=20, wsleft_cm=2.0)
# ------------------------------------------------
# Plot the FIR filter coefficients.
# ------------------------------------------------
plot_nr = 1
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr)
ax1.plot(taps, "bo-", linewidth=2)
ax1.set_title("Filter Coefficients (%d taps)" % N)
ax1.grid(True)
# ------------------------------------------------
# Plot the magnitude response of the filter.
# ------------------------------------------------
plot_nr += 1
ax2 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr)
w, h = sp.signal.freqz(taps, worN=8000)
ax2.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2)
ax2.set_xlabel("Frequency (Hz)")
ax2.set_ylabel("Gain")
ax2.set_title("Frequency Response")
ax2.set_ylim(-0.05, 1.05)
# ax2.grid(True)
# in order to place the nex axes inside following figure, first
# determine the ax2 bounding box
# points: a 2x2 numpy array of the form [[x0, y0], [x1, y1]]
ax2box = ax2.get_window_extent().get_points()
# seems to be expressed in pixels so convert to relative coordinates
# print ax2box
# figure size in pixels
figsize_x_pix = plot.figsize_x * plot.dpi
figsize_y_pix = plot.figsize_y * plot.dpi
# ax2 box in relative coordinates
ax2box[:, 0] = ax2box[:, 0] / figsize_x_pix
ax2box[:, 1] = ax2box[:, 1] / figsize_y_pix
# print ax2box[0,0], ax2box[1,0], ax2box[0,1], ax2box[1,1]
# left position new box at 10% of x1
left = ax2box[0, 0] + ((ax2box[1, 0] - ax2box[0, 0]) * 0.15)
bottom = ax2box[0, 1] + ((ax2box[1, 1] - ax2box[0, 1]) * 0.30) # x2
width = (ax2box[1, 0] - ax2box[0, 0]) * 0.35
height = (ax2box[1, 1] - ax2box[0, 1]) * 0.6
# print [left, bottom, width, height]
# left inset plot.
# [left, bottom, width, height]
# ax2a = plot.fig.add_axes([0.42, 0.6, .45, .25])
ax2a = plot.fig.add_axes([left, bottom, width, height])
ax2a.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2)
ax2a.set_xlim(0, 8.0)
ax2a.set_ylim(0.9985, 1.001)
ax2a.grid(True)
# right inset plot
left = ax2box[0, 0] + ((ax2box[1, 0] - ax2box[0, 0]) * 0.62)
bottom = ax2box[0, 1] + ((ax2box[1, 1] - ax2box[0, 1]) * 0.30) # x2
width = (ax2box[1, 0] - ax2box[0, 0]) * 0.35
height = (ax2box[1, 1] - ax2box[0, 1]) * 0.6
# Lower inset plot
# ax2b = plot.fig.add_axes([0.42, 0.25, .45, .25])
ax2b = plot.fig.add_axes([left, bottom, width, height])
ax2b.plot((w / np.pi) * nyq_rate, np.absolute(h), linewidth=2)
ax2b.set_xlim(12.0, 20.0)
ax2b.set_ylim(0.0, 0.0025)
ax2b.grid(True)
# ------------------------------------------------
# Plot the original and filtered signals.
# ------------------------------------------------
# The phase delay of the filtered signal.
delay = 0.5 * (N - 1) / sample_rate
plot_nr += 1
ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, plot_nr)
# Plot the original signal.
ax3.plot(time, data, label="original signal")
# Plot the filtered signal, shifted to compensate for the phase delay.
ax3.plot(time - delay, filtered_x, "r-", label="filtered signal")
# Plot just the "good" part of the filtered signal. The first N-1
# samples are "corrupted" by the initial conditions.
ax3.plot(time[N - 1 :] - delay, filtered_x[N - 1 :], "g", linewidth=4)
ax3.set_xlabel("t")
ax3.grid(True)
plot.save_fig()
def fir(self, time, data, **kwargs):
"""
Based on the xxample from the SciPy cook boock, see
http://www.scipy.org/Cookbook/FIRFilter
Parameters
----------
time : ndarray(n)
data : ndarray(n)
plot : boolean, default=False
figpath : str, default=False
figfile : str, default=False
sample_rate : int, default=None
If None, sample rate will be calculated from the given signal
freq_trans_width : float, default=1
The desired width of the transition from pass to stop,
relative to the Nyquist rate.
ripple_db : float, default=10
The desired attenuation in the stop band, in dB.
cutoff_hz : float, default=10
Frequencies above cutoff_hz are filtered out
Returns
-------
filtered_x : ndarray(n - (N-1))
filtered signal
N : float
order of the firwin filter
delay : float
phase delay due to the filtering process
"""
plot = kwargs.get("plot", False)
figpath = kwargs.get("figpath", False)
figfile = kwargs.get("figfile", False)
sample_rate = kwargs.get("sample_rate", None)
# The desired width of the transition from pass to stop,
# relative to the Nyquist rate. We'll design the filter
# with a 5 Hz transition width.
freq_trans_width = kwargs.get("freq_trans_width", 1)
# The desired attenuation in the stop band, in dB.
ripple_db = kwargs.get("ripple_db", 10)
# The cutoff frequency of the filter.
cutoff_hz = kwargs.get("cutoff_hz", 10)
chk.array_1d(time)
chk.array_1d(data)
if not sample_rate:
sample_rate = calc_sample_rate(time)
# ------------------------------------------------
# Create a FIR filter and apply it to data[:,channel]
# ------------------------------------------------
# The Nyquist rate of the signal.
nyq_rate = sample_rate / 2.0
# The desired width of the transition from pass to stop,
# relative to the Nyquist rate. We'll design the filter
# with a 5 Hz transition width.
width = freq_trans_width / nyq_rate
# Compute the order and Kaiser parameter for the FIR filter.
N, beta = sp.signal.kaiserord(ripple_db, width)
# Use firwin with a Kaiser window to create a lowpass FIR filter.
taps = sp.signal.firwin(N, cutoff_hz / nyq_rate, window=("kaiser", beta))
# Use lfilter to filter x with the FIR filter.
filtered_x = sp.signal.lfilter(taps, 1.0, data)
# The phase delay of the filtered signal.
delay = 0.5 * (N - 1) / sample_rate
# # the filtered signal, shifted to compensate for the phase delay.
# time_shifted = time-delay
# # the "good" part of the filtered signal. The first N-1
# # samples are "corrupted" by the initial conditions.
# time_good = time[N-1:] - delay
if plot:
self.plot_fir(figpath, figfile, time, data, filtered_x, N, delay, sample_rate, taps, nyq_rate)
return filtered_x, N, delay