def setup_filter(self, respath, run, **kwargs):
"""
Load the callibration runs and convert voltage signal to yaw angles
"""
# specify the window of the staircase
#start, end = 30100, -30001
start = kwargs.get('start', None)
end = kwargs.get('end', None)
figpath = kwargs.get('figpath', None)
# figfile = kwargs.get('figfile', None)
dt_treshold = kwargs.get('dt_treshold', None)
# plot_data = kwargs.get('plot_data', False)
# respath = kwargs.get('respath', None)
# run = kwargs.get('run', None)
# load the dspace mat file
dspace = ojfresult.DspaceMatFile(respath + run)
# the yaw channel
ch = 6
# or a more robust way of determining the channel number
ch = dspace.labels_ch['Yaw Laser']
# sample rate of the signal
sample_rate = calc_sample_rate(dspace.time)
# file name based on the run file
figfile = dspace.matfile.split('/')[-1] + '_ch' + str(ch)
# prepare the data
time = dspace.time[start:end]
# the actual yaw signal
data = dspace.data[start:end,ch]
# -------------------------------------------------
# smoothen the signal with some splines
# -------------------------------------------------
# NOTE: the smoothing will make the transitions also smoother. This
# is not good. The edges of the stair need to be steep!
# smoothen = UnivariateSpline(dspace.time, dspace.data[:,ch], s=2)
# data_s_full = smoothen(dspace.time)
# # first the derivatices
# data_s_dt = data_s_full[start+1:end+1]-data_s_full[start:end]
# # than cut it off
# data_s = data_s_full[start:end]
# -------------------------------------------------
# local derivatives of the yaw signal and filtering
# -------------------------------------------------
data_dt = dspace.data[start+1:end+1,ch]-dspace.data[start:end,ch]
# filter the local derivatives
filt = Filters()
data_filt, N, delay = filt.fir(time, data, ripple_db=20,
freq_trans_width=0.5, cutoff_hz=0.3, plot=False,
figpath=figpath, figfile=figfile + 'filter_design',
sample_rate=sample_rate)
data_filt_dt = np.ndarray(data_filt.shape)
data_filt_dt[1:] = data_filt[1:] - data_filt[0:-1]
data_filt_dt[0] = np.nan
# -------------------------------------------------
# smoothen the signal with some splines
# -------------------------------------------------
# smoothen = UnivariateSpline(time, data_filt, s=2)
# data_s = smoothen(time)
# # first the derivatices
# data_s_dt = np.ndarray(data_s.shape)
# data_s_dt[1:] = data_s[1:]-data_s[:-1]
# data_s_dt[0] = np.nan
# -------------------------------------------------
# filter values above certain treshold
# ------------------------------------------------
# only keep values which are steady, meaning dt signal is low!
# based upon the filtering, only select data points for which the
# filtered derivative is between a certain treshold
staircase_i = np.abs(data_filt_dt).__ge__(dt_treshold)
# make a copy of the original signal and fill in Nans on the selected
# values
data_reduced = data.copy()
data_reduced[staircase_i] = np.nan
data_reduced_dt = np.ndarray(data_reduced.shape)
data_reduced_dt[1:] = np.abs(data_reduced[1:] - data_reduced[:-1])
data_reduced_dt[0] = np.nan
nonnan_i = np.isnan(data_reduced_dt).__invert__()
dt_noise_treshold = data_reduced_dt[nonnan_i].max()
print ' dt_noise_treshold ', dt_noise_treshold
# remove all the nan values
data_trim = data_reduced[np.isnan(data_reduced).__invert__()]
time_trim = time[np.isnan(data_reduced).__invert__()]
# # figure out which dt's are above the treshold
# data_trim2 = data_trim.copy()
# data_trim2.sort()
# data_trim2.
# # where the dt of the reduced format is above the noise treshold,
# # we have a stair
# data_trim_dt = np.abs(data_trim[1:] - data_trim[:-1])
# argstairs = data_trim_dt.__gt__(dt_noise_treshold)
# data_trim2 = data_trim_dt.copy()
# data_trim_dt.sort()
# data_trim_dt.__gt__(dt_noise_treshold)
# -------------------------------------------------
# read the average value over each stair (time and data)
# ------------------------------------------------
data_ordered, time_stair, data_stair = self.order_staircase(time_trim,
data_trim, dt_noise_treshold*4.)
# -------------------------------------------------
# setup plot
# -------------------------------------------------
labels = np.ndarray(3, dtype='<U100')
labels[0] = dspace.labels[ch]
labels[1] = 'yawchan derivative'
labels[2] = 'psd'
plot = plotting.A4Tuned()
title = figfile.replace('_', ' ')
plot.setup(figpath+figfile+'_filter', nr_plots=2, grandtitle=title,
figsize_y=20, wsleft_cm=2., wsright_cm=2.5)
# -------------------------------------------------
# plotting of signal
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 1)
ax1.plot(time, data, label='data')
# add the results of the filtering technique
time_stair, data_stair
ax1.plot(time[N-1:], data_reduced[N-1:], 'r', label='data red')
# ax1.plot(time[N-1:], data_filt[N-1:], 'g', label='data_filt')
# also include the selected chair data
label = '%i stairs' % data_stair.shape[0]
ax1.plot(time_stair, data_stair, 'ko', label=label, alpha=0.2)
ax1.grid(True)
ax1.legend(loc='lower left')
# -------------------------------------------------
# plotting derivatives on right axis
# -------------------------------------------------
ax1b = ax1.twinx()
# ax1b.plot(time[N:]-delay,data_s_dt[N:],alpha=0.2,label='data_s_dt')
ax1b.plot(time[N:], data_filt_dt[N:], 'r', alpha=0.2,
label='data filt dt')
# ax1b.plot(time[N:], data_reduced_dt[N:], 'b', alpha=0.2,
# label='data_reduced_dt')
# ax1b.plot(time[N-1:]-delay, filtered_x_dt[N-1:], alpha=0.2)
ax1b.legend()
ax1b.grid(True)
# -------------------------------------------------
# the power spectral density
# -------------------------------------------------
ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 2)
Pxx, freqs = ax3.psd(data, Fs=sample_rate, label='data')
Pxx, freqs = ax3.psd(data_dt, Fs=sample_rate, label='data dt')
# Pxx, freqs = ax3.psd(data_s_dt, Fs=sample_rate, label='data_s_dt')
Pxx, freqs = ax3.psd(data_filt_dt[N-1:], Fs=sample_rate,
label='data filt dt')
ax3.legend()
# print Pxx.shape, freqs.shape
plot.save_fig()
# -------------------------------------------------
# get amplitudes of the stair edges
# -------------------------------------------------
# # max step
# data_trim_dt_sort = data_trim_dt.sort()[0]
# # estimate at what kind of a delta we are looking for when changing
# # stairs
# data_dt_std = data_trim_dt.std()
# data_dt_mean = (np.abs(data_trim_dt)).mean()
#
# time_data_dt = np.transpose(np.array([time, data_filt_dt]))
# data_filt_dt_amps = HawcPy.dynprop().amplitudes(time_data_dt, h=1e-3)
#
# print '=== nr amplitudes'
# print len(data_filt_dt_amps)
# print data_filt_dt_amps
return time_stair, data_stair
def setup_filter(self, time, data, **kwargs):
"""
Load the callibration runs and convert voltage signal to yaw angles
Parameters
----------
time : ndarray(k)
data : ndarray(k)
Returns
-------
time_stair : ndarray(n)
Average time stamp over the stair step
data_stair : ndarray(n)
Average value of the selected stair step
"""
# time and data should both be 1D and have the same shape!
assert time.shape == data.shape
runid = kwargs.get('runid', self.runid)
# smoothen method: spline or moving average
smoothen = kwargs.get('smoothen', 'spline')
# what is the window of the moving average in seconds
smooth_window = kwargs.get('smooth_window', 2)
# specify the window of the staircase
#start, end = 30100, -30001
start = kwargs.get('start', 0)
end = kwargs.get('end', len(time))
dt = kwargs.get('dt', 1)
cutoff_hz = kwargs.get('cutoff_hz', None)
self.points_per_stair = kwargs.get('points_per_stair', 20)
# at what is the minimum required value on dt or dt2 for a new stair
self.stair_step_tresh = kwargs.get('stair_step_tresh', 1)
# plot_data = kwargs.get('plot_data', False)
# respath = kwargs.get('respath', None)
# run = kwargs.get('run', None)
# sample rate of the signal
sample_rate = calc_sample_rate(time)
# prepare the data
time = time[start:end]
# the actual raw signal
data = data[start:end]
# -------------------------------------------------
# Progress plotting
# ----------------------------------------------
if self.plt_progress:
plt.figure()
Pxx, freqs = plt.psd(data, Fs=sample_rate, label='data')
plt.show()
plt.figure()
plt.plot(time, data, label='raw data')
# -------------------------------------------------
# setup plot
# -------------------------------------------------
# labels = np.ndarray(3, dtype='<U100')
# labels[0] = label
# labels[1] = 'yawchan derivative'
# labels[2] = 'psd'
# remove any underscores for latex printing
grandtitle = self.figfile.replace('_', '\_')
plot = plotting.A4Tuned(scale=1.5)
plot.setup(self.figpath+self.figfile+'_filter', nr_plots=3,
grandtitle=grandtitle, wsleft_cm=1.5, wsright_cm=1.8,
hspace_cm=1.2, size_x_perfig=10, size_y_perfig=5,
wsbottom_cm=1.0, wstop_cm=1.5)
# -------------------------------------------------
# plotting original and smoothend signal
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 1)
ax1.plot(time, data, 'b', label='raw data', alpha=0.6)
data_raw = data.copy()
# -------------------------------------------------
# signal frequency filtering, if applicable
# -------------------------------------------------
# filter the local derivatives if applicable
if cutoff_hz:
filt = Filters()
data_filt, N, delay = filt.fir(time, data, ripple_db=20,
freq_trans_width=0.5, cutoff_hz=cutoff_hz,
figpath=self.figpath,
figfile=self.figfile + 'filter_design',
sample_rate=sample_rate, plot=False,)
if self.plt_progress:
# add the results of the filtering technique
plt.plot(time[N-1:], data_filt[N-1:], 'r', label='freq filt')
data = data_filt
time = time[N-1:]#-delay
else:
N = 1
# -------------------------------------------------------
# smoothen the signal with some splines or moving average
# -------------------------------------------------------
# NOTE: the smoothing will make the transitions also smoother. This
# is not good. The edges of the stair need to be steep!
# for the binary data this is actually a good thing, since the dt's
# are almost always the same between time steps. We would otherwise
# need a dt based on several time steps
if smoothen == 'splines':
print 'start applying spline ...',
uni_spline = UnivariateSpline(time, data)
data = uni_spline(time)
print 'done!'
NN = 0 # no time shift due to filtering?
if self.plt_progress:
plt.plot(time, data, label='spline data')
elif smoothen == 'moving':
print 'start calculating movering average ...',
filt = Filters()
# take av2s window, calculate the number of samples per window
ws = int(smooth_window*sample_rate)
data = filt.smooth(data, window_len=ws, window='hanning')
NN = len(data) - len(time)
data = data[NN:]
print 'done!'
if self.plt_progress:
plt.plot(time, data, label='moving average')
else:
raise ValueError, 'smoothen method should be moving or splines'
# -------------------------------------------------
# additional smoothening: downsampling
# -------------------------------------------------
# and up again in order not to brake the plotting further down
time_down = np.arange(time[0], time[-1], 0.1)
data_down = sp.interpolate.griddata(time, data, time_down)
# and upsampling again
data = sp.interpolate.griddata(time_down, data_down, time)
# -------------------------------------------------
# plotting original and smoothend signal
# -------------------------------------------------
ax1.plot(time, data, 'r', label='data smooth')
ax1.grid(True)
leg1 = ax1.legend(loc='best')
leg1.get_frame().set_alpha(0.5)
ax1.set_title('smoothing method: ' + smoothen)
# -------------------------------------------------
# local derivatives of the signal and filtering
# -------------------------------------------------
data_dt = np.ndarray(data.shape)
data_dt[1:] = data[1:] - data[0:-1]
data_dt[0] = np.nan
data_dt = np.abs(data_dt)
# frequency filter was applied here originally
data_filt_dt = data_dt
# if no threshold is given, just take the 20% of the max value
dt_max = np.nanmax(np.abs(data_filt_dt))*0.2
dt_treshold = kwargs.get('dt_treshold', dt_max)
# -------------------------------------------------
# filter dt or dt2 above certain treshold?
# -----------------------------------------------
# only keep values which are steady, meaning dt signal is low!
if dt == 2:
tmp = np.ndarray(data_filt_dt.shape)
tmp[1:] = data_filt_dt[1:] - data_filt_dt[0:-1]
tmp[0] = np.nan
data_filt_dt = tmp
# based upon the filtering, only select data points for which the
# filtered derivative is between a certain treshold
staircase_i = np.abs(data_filt_dt).__ge__(dt_treshold)
# reduce to 1D
staircase_arg=np.argwhere(np.abs(data_filt_dt)<=dt_treshold).flatten()
# -------------------------------------------------
# replace values for too high dt with Nan
# ------------------------------------------------
# ---------------------------------
# METHOD version2, slower because of staircase_arg computation above
data_masked = data.copy()
data_masked[staircase_i] = np.nan
data_masked_dt = data_filt_dt.copy()
data_masked_dt[staircase_i] = np.nan
data_trim = data[staircase_arg]
time_trim = time[staircase_arg]
print 'max in data_masked_dt:', np.nanmax(data_masked_dt)
# ---------------------------------
# METHOD version2, faster if staircase_arg is not required!
## make a copy of the original signal and fill in Nans on the selected
## values
#data_masked = data.copy()
#data_masked[staircase_i] = np.nan
#
#data_masked_dt = data_filt_dt.copy()
#data_masked_dt[staircase_i] = np.nan
#
## remove all the nan values
#data_trim = data_masked[np.isnan(data_masked).__invert__()]
#time_trim = time[np.isnan(data_masked).__invert__()]
#
#dt_noise_treshold = np.nanmax(data_masked_dt)
#print 'max in data_masked_dt', dt_noise_treshold
# ---------------------------------
# # figure out which dt's are above the treshold
# data_trim2 = data_trim.copy()
# data_trim2.sort()
# data_trim2.
# # where the dt of the masked format is above the noise treshold,
# # we have a stair
# data_trim_dt = np.abs(data_trim[1:] - data_trim[:-1])
# argstairs = data_trim_dt.__gt__(dt_noise_treshold)
# data_trim2 = data_trim_dt.copy()
# data_trim_dt.sort()
# data_trim_dt.__gt__(dt_noise_treshold)
# -------------------------------------------------
# intermediate checking of the signal
# -------------------------------------------------
if self.plt_progress:
# add the results of the filtering technique
plt.plot(time[N-1:], data_masked[N-1:], 'rs', label='data red')
plt.legend(loc='best')
plt.grid(True)
plt.twinx()
# plt.plot(time, data_filt_dt, label='data_filt_dt')
plt.plot(time, data_masked_dt, 'm', label='data\_masked\_dt',
alpha=0.4)
plt.legend(loc='best')
plt.show()
print 'saving plt_progress:',
print self.figpath+'filter_design_progress.png'
plt.savefig(self.figpath+'filter_design_progress.png')
# -------------------------------------------------
# check if we have had sane filtering
# -------------------------------------------------
print 'data :', data.shape
print 'data_trim :', data_trim.shape
print 'trim ratio:', len(data)/len(data_trim)
# there should be at least one True value
assert staircase_i.any()
# they can't all be True, than filtering is too heavy
if len(data_trim) < len(data)*0.01:
msg = 'dt_treshold is too low, not enough data left'
raise ValueError, msg
# if no data is filtered at all, filtering is too conservative
elif len(data_trim) > len(data)*0.95:
msg = 'dt_treshold is too high, too much data left'
raise ValueError, msg
# if the data array is too big, abort on memory concerns
if len(data_trim) > 200000:
msg = 'too much data points for stair case analysis (cfr memory)'
raise ValueError, msg
# -------------------------------------------------
# read the average value over each stair (time and data)
# ------------------------------------------------
#try:
##np.save('time_trim', time_trim)
##np.save('data_trim', data_trim)
##np.save('staircase_arg', staircase_arg)
##tmp = np.array([self.points_per_stair, self.stair_step_tresh])
##np.save('tmp', tmp)
#data_ordered, time_stair, data_stair, arg_stair \
#= cython_func.order_staircase(time_trim, data_trim,
#staircase_arg, self.points_per_stair, self.stair_step_tresh)
#except ImportError:
data_ordered, time_stair, data_stair, arg_stair \
= self.order_staircase(time_trim, data_trim, staircase_arg)
# convert the arg_stair to a flat set and replace start/stop pairs
# with all indices in between. Now we can select all stair values
# in the raw dataset
arg_st_fl = np.empty(data_raw.shape, dtype=np.int)
i = 0
for k in range(arg_stair.shape[1]):
#print '%6i %6i' % (arg_stair[0,k],arg_stair[1,k])
tmp = np.arange(arg_stair[0,k], arg_stair[1,k]+1, 1, dtype=np.int)
#print tmp, '->', i, ':', i+len(tmp)
arg_st_fl[i:i+len(tmp)] = tmp
i += len(tmp)
# remove the unused elements from the array
arg_st_fl = arg_st_fl[:i]
# -------------------------------------------------
# plotting of smoothen signal and stairs
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 2)
ax1.plot(time, data, label='data smooth', alpha=0.6)
# add the results of the filtering technique
ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked')
# ax1.plot(time[N-1:], data_filt[N-1:], 'g', label='data_filt')
# also include the selected chair data
figlabel = '%i stairs' % data_stair.shape[0]
ax1.plot(time_stair, data_stair, 'ko', label=figlabel, alpha=0.4)
ax1.grid(True)
# the legend, on or off?
#leg1 = ax1.legend(loc='upper left')
#leg1.get_frame().set_alpha(0.5)
# -------------------------------------------------
# plotting derivatives on right axis
# -------------------------------------------------
ax1b = ax1.twinx()
# ax1b.plot(time[N:]-delay,data_s_dt[N:],alpha=0.2,label='data_s_dt')
ax1b.plot(time[N:], data_filt_dt[N:], 'r', alpha=0.35,
label='data\_filt\_dt')
majorFormatter = FormatStrFormatter('%8.1e')
ax1b.yaxis.set_major_formatter(majorFormatter)
# ax1b.plot(time[N:], data_masked_dt[N:], 'b', alpha=0.2,
# label='data_masked_dt')
# ax1b.plot(time[N-1:]-delay, filtered_x_dt[N-1:], alpha=0.2)
# leg1b = ax1b.legend(loc='best')
# leg1b.get_frame().set_alpha(0.5)
# ax1b.grid(True)
# -------------------------------------------------
# 3th plot to check if the raw chair signal is ok
# -------------------------------------------------
ax1 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3)
ax1.plot(time[arg_st_fl], data_raw[arg_st_fl], 'k+', label='rawstair',
alpha=0.1)
ax1.plot(time[N-1:], data_masked[N-1:], 'r', label='data masked')
ax1.set_xlabel('time [s]')
# -------------------------------------------------
# the power spectral density
# -------------------------------------------------
# ax3 = plot.fig.add_subplot(plot.nr_rows, plot.nr_cols, 3)
# Pxx, freqs = ax3.psd(data, Fs=sample_rate, label='data smooth')
## Pxx, freqs = ax3.psd(data_dt, Fs=sample_rate, label='data_dt')
## Pxx, freqs = ax3.psd(data_filt_dt[N-1:], Fs=sample_rate,
## label='data_filt_dt')
# ax3.legend()
## print Pxx.shape, freqs.shape
plot.save_fig()
# -------------------------------------------------
# get amplitudes of the stair edges
# -------------------------------------------------
# # max step
# data_trim_dt_sort = data_trim_dt.sort()[0]
# # estimate at what kind of a delta we are looking for when changing
# # stairs
# data_dt_std = data_trim_dt.std()
# data_dt_mean = (np.abs(data_trim_dt)).mean()
#
# time_data_dt = np.transpose(np.array([time, data_filt_dt]))
# data_filt_dt_amps = HawcPy.dynprop().amplitudes(time_data_dt, h=1e-3)
#
# print '=== nr amplitudes'
# print len(data_filt_dt_amps)
# print data_filt_dt_amps
# -------------------------------------------------
# save the data
# -------------------------------------------------
filename = runid + '-time_stair'
np.savetxt(self.pprpath + filename, time_stair)
filename = runid + '-data_stair'
np.savetxt(self.pprpath + filename, data_stair)
# in order to maintain backwards compatibility, save the arguments
# of the stair to self
self.arg_st_fl = arg_st_fl # flat, contains all indices on the stairs
# start/stop indeces for stair k = arg_stair[0,k], arg_stair[1,k]
self.arg_stair = arg_stair
return time_stair, data_stair