def plb_raw(self, sensor_dist=None):
# initialize
n_channel_data = None
if sensor_dist is 'near':
n_channel_data = read_all_tdms_from_folder(self.path_plb_2to12)
elif sensor_dist is 'far':
n_channel_data = read_all_tdms_from_folder(self.path_plb_10to22)
# swap axis, so shape[0] is sensor (for easy using)
n_channel_data = np.swapaxes(n_channel_data, 1,
2) # swap axis[0] and [1]
return n_channel_data
def plb_4_sensor(self, leak_pos=0):
# ---------------------[Select the file and read]------------------------
'''
:param leak_pos: the leak position on the pipe
:return
n_channel_data -> 3d matrix where shape[0]-> no of set(sample size),
shape[1]-> no. of AE data points,
shape[2]-> no. of sensors
phase_map_all -> 4d matrix where shape[0]-> no of set(sample size),
shape[1]-> no. of sensors
shape[2]-> no. of freq band,
shape[3]-> no. of time steps
'''
if leak_pos is 0:
n_channel_data = read_all_tdms_from_folder(self.path_0m_plb)
elif leak_pos is 2:
n_channel_data = read_all_tdms_from_folder(self.path_2m_plb)
elif leak_pos is 4:
n_channel_data = read_all_tdms_from_folder(self.path_4m_plb)
elif leak_pos is 6:
n_channel_data = read_all_tdms_from_folder(self.path_6m_plb)
# ---------------------[STFT into feature(mag/phase) maps]------------------------
# for all sets (samples)
phase_map_all = []
for set_no in range(n_channel_data.shape[0]):
phase_map_bank = []
# for all sensors
for sensor_no in range(n_channel_data.shape[2]):
t, f, Sxx, _ = spectrogram_scipy(n_channel_data[set_no,
500000:1500000,
sensor_no],
fs=1e6,
nperseg=2000,
noverlap=0,
mode='magnitude',
return_plot=False,
verbose=False,
vis_max_freq_range=1e6 / 2)
phase_map_bank.append(Sxx)
phase_map_bank = np.array(phase_map_bank)
phase_map_all.append(phase_map_bank)
# convert to array
phase_map_all = np.array(phase_map_all)
print('Phase Map Dim (set_no, sensor_no, freq_band, time steps): ',
phase_map_all.shape, '\n')
return n_channel_data, phase_map_all, f, t
def leak_noleak_4_sensor(self, leak=True):
# leak pos at 0m, 1bar, 5mm hole
# ---------------------[Select the file and read]------------------------
'''
:param leak: True is leak, False is no leak
:return
n_channel_data -> 3d matrix where shape[0]-> no of set(sample size),
shape[1]-> no. of AE data points,
shape[2]-> no. of sensors
'''
if leak is True:
n_channel_data = read_all_tdms_from_folder(
self.path_leak_1bar_10mm)
else:
n_channel_data = read_all_tdms_from_folder(self.path_noleak_1bar)
return n_channel_data
scale = np.linspace(2, 30, 100)
fs = 1e6
# segmentation
no_of_segment = 2 # 10 is showing a consistent pattern
# DATA POINT ----------------------------------------------------------------------------------------------------------
# read leak data
on_pc = True
if on_pc:
data = AcousticEmissionDataSet_13_7_2018(drive='F')
n_channel_leak = data.test_data(sensor_dist='near', pressure=1, leak=True)
else:
data_dir = direct_to_dir(where='yh_laptop_test_data') + '1bar_leak/'
n_channel_leak = read_all_tdms_from_folder(data_dir)
n_channel_leak = np.swapaxes(n_channel_leak, 1, 2)
n_channel_leak = n_channel_leak[0]
# processing
print(n_channel_leak.shape)
# break into a list of segmented points
n_channel_leak = np.split(n_channel_leak,
axis=1,
indices_or_sections=no_of_segment)
print('Total Segment: ', len(n_channel_leak))
print('Each Segment Dim: ', n_channel_leak[0].shape)
# CWT + XCOR + VISUALIZE SCRIPT ---------------------------------------------------------------------------------------
# xcor pairing commands - [near] = 0m, 1m,..., 10m
def sleak_1bar_7pos(self, train_split=0.7, f_range=(0, 3000)):
full_path_0123 = self.drive + self.path_sleak_pos0123_1bar
full_path_0456 = self.drive + self.path_sleak_pos0456_1bar
# get all 4 channels sensor data in np matrix of 4 columns
data_0123 = read_all_tdms_from_folder(full_path_0123)
data_0456 = read_all_tdms_from_folder(full_path_0456)
# ensure both has same number of sets before concatenate
assert data_0123.shape[0] == data_0456.shape[
0], 'Different no of sets detected'
data_0123456 = []
for i in range(data_0123.shape[0]):
# take only column 1-3 for data_0456
data_0123456.append(
np.concatenate((data_0123[i], data_0456[i, :, 1:]), axis=1))
data_0123456 = np.array(data_0123456)
print('Concatenated Data Dim: ', data_0123456.shape, '\n')
# FREQ-TIME TRANSFORMATION
ft_bank = []
ft_bank_label = []
label = 0
# for all 7 sensors (0-6)
pb = ProgressBarForLoop('Spectrogram_scipy Transformation',
end=data_0123456.shape[2])
for i in range(data_0123456.shape[2]):
pb.update(now=i)
# for all 3 sets
for j in range(data_0123456.shape[0]):
# getting a f-t of dim (5001, 1000) for a 5 seconds signal
_, _, ft_mat = spectrogram_scipy(
sampled_data=data_0123456[j, :, i],
fs=1e6,
nperseg=10000,
noverlap=5007,
return_plot=False,
verbose=False,
save=False,
plot_title='Class[{}] Set[{}]'.format(i, j))
# split 5 sec into 25x0.2sec for bigger sample size
# take only 0 - 300kHz (3000 points) out of 0 - 500kHz (5001 points)
index_start = 0
index_end = ft_mat.shape[1] # time step
interval = 25 # how many parts u wan to split into
# plus 1 is necessary because N points only has N-1 intervals
segmented_index = np.linspace(index_start, index_end,
interval + 1)
# convert all in segmented_index to int
segmented_index = [int(i) for i in segmented_index]
# for all interval in segmented_index
for k in range(len(segmented_index) - 1):
ft_bank.append(
ft_mat[f_range[0]:f_range[1],
segmented_index[k]:segmented_index[k + 1]])
ft_bank_label.append(label)
# label for nex position
label += 1
# kill progress bar
pb.destroy()
ft_bank = np.array(ft_bank)
ft_bank_label = np.array(ft_bank_label)
print('f-t Data Dim: ', ft_bank.shape)
print('Label Dim:', ft_bank_label.shape)
# slicing them into train and test set
train_x, test_x, train_y, test_y = break_balanced_class_into_train_test(
input=ft_bank,
label=ft_bank_label,
num_classes=7,
train_split=train_split,
verbose=True)
return train_x, test_x, train_y, test_y
def leak_2class(self):
'''
This uses the sensor[-1m] & [23m] as a pair and sensor[-2m] & [22m] as a pair. Both of them has distance of 24m.
This can act as 2 different leak position, differ by 1m. Since we have only 3 sets of time series, each is 5M
points, we break them into segments of 100k points (equiv. to 0.1s), so each class will have 150 samples.
:return:
class_1 -> 3d array where shape[0]-> no. of samples(150)
shape[1]-> freq axis (20kHz to 100kHz)
shape[2]-> xcor steps (2000 units)
'''
n_channel_data = read_all_tdms_from_folder(self.path_leak_1bar_10mm)
# concat all 3 sets into (15M, 4)
n_channel_data_all_set = np.concatenate(
(n_channel_data[0], n_channel_data[1], n_channel_data[2]), axis=0)
# split 15M points into 150 segments
sample_size = 150
all_segment_of_4_sensor = np.split(n_channel_data_all_set,
indices_or_sections=sample_size)
all_segment_of_4_sensor = np.array(all_segment_of_4_sensor)
# STFT for every sample, for every sensors ----------------------------------
sensor_0_stft, sensor_1_stft, sensor_2_stft, sensor_3_stft = [], [], [], []
# initiate progressbar
pb = ProgressBarForLoop(title='Bandpass + STFT',
end=all_segment_of_4_sensor.shape[0])
progress = 0
# for all samples
for sample in all_segment_of_4_sensor:
temp_stft_bank_4_sensor = []
# for all sensors
for i in range(4):
# bandpass from 20kHz t0 100kHz
filtered_signal = butter_bandpass_filtfilt(
sampled_data=sample[:, i],
fs=1e6,
f_hicut=1e5,
f_locut=20e3)
# STFT
_, _, sxx, _ = spectrogram_scipy(sampled_data=filtered_signal,
fs=1e6,
mode='magnitude',
nperseg=100,
nfft=500,
noverlap=0,
return_plot=False,
verbose=False)
# Take only
temp_stft_bank_4_sensor.append(
sxx[10:51, :]) # index_10 -> f=20kHz; index_50 -> f=100kHz
sensor_0_stft.append(temp_stft_bank_4_sensor[0])
sensor_1_stft.append(temp_stft_bank_4_sensor[1])
sensor_2_stft.append(temp_stft_bank_4_sensor[2])
sensor_3_stft.append(temp_stft_bank_4_sensor[3])
# update progressbar
pb.update(now=progress)
progress += 1
pb.destroy()
# convert to np array
sensor_0_stft = np.array(sensor_0_stft)
sensor_1_stft = np.array(sensor_1_stft)
sensor_2_stft = np.array(sensor_2_stft)
sensor_3_stft = np.array(sensor_3_stft)
# Xcor in freq band
class_1, class_2 = [], []
# initiate progressbar
pb = ProgressBarForLoop(title='Cross Correlation',
end=all_segment_of_4_sensor.shape[0])
progress = 0
for i in range(sample_size):
# for class 1, sensor[-2m] & [22m]
stft_map = np.array([sensor_0_stft[i], sensor_2_stft[i]])
sensor_pair = [(0, 1)]
xcor_map, _ = one_dim_xcor_2d_input(input_mat=stft_map,
pair_list=sensor_pair,
verbose=False)
class_1.append(xcor_map[0, :, 700:1300])
# for class 1, sensor[-1m] & [23m]
stft_map = np.array([sensor_1_stft[i], sensor_3_stft[i]])
sensor_pair = [(0, 1)]
xcor_map, _ = one_dim_xcor_2d_input(input_mat=stft_map,
pair_list=sensor_pair,
verbose=False)
class_2.append(xcor_map[0, :, 700:1300])
pb.update(now=progress)
pb.destroy()
# packaging dataset and label
class_1 = np.array(class_1)
class_2 = np.array(class_2)
dataset = np.concatenate((class_1, class_2), axis=0)
label = np.array([0] * class_1.shape[0] + [1] * class_2.shape[0])
print('Dataset Dim: ', dataset.shape)
print('Label Dim: ', label.shape)
return dataset, label
def test(self):
n_channel_data = read_all_tdms_from_folder(self.path_leak_1bar_10mm)
return n_channel_data[0, 0:100000, 0]
def plb_unseen(self):
# use near only
n_channel_data = read_all_tdms_from_folder(self.path_plb_2to12)
# swap axis, so shape[0] is sensor (for easy using)
n_channel_data = np.swapaxes(n_channel_data, 1,
2) # swap axis[0] and [1]
# pairing order for xcor
sensor_pair = [(0, 5)]
class_1 = []
# initiate progressbar
pb = ProgressBarForLoop(title='Bandpass + STFT + XCOR',
end=n_channel_data.shape[0])
progress = 0
# for all plb samples
for sample in n_channel_data:
all_channel_stft = []
# for all sensors
for each_sensor_data in sample:
# band pass filter
filtered_signal = butter_bandpass_filtfilt(
sampled_data=each_sensor_data[90000:130000],
fs=1e6,
f_hicut=1e5,
f_locut=20e3)
# stft
_, _, sxx, _ = spectrogram_scipy(sampled_data=filtered_signal,
fs=1e6,
mode='magnitude',
nperseg=100,
nfft=500,
noverlap=0,
return_plot=False,
verbose=True)
all_channel_stft.append(
sxx[10:51, :]) # index_10 -> f=20kHz; index_50 -> f=100kHz
all_channel_stft = np.array(all_channel_stft)
# xcor for sensor pair
xcor_map, _ = one_dim_xcor_2d_input(input_mat=all_channel_stft,
pair_list=sensor_pair,
verbose=False)
# visualize and saving the training data
savepath = 'C:/Users/YH/PycharmProjects/AE-signal-model/result/'
visualize = False
if visualize:
fig_1 = heatmap_visualizer(
x_axis=np.arange(0, xcor_map.shape[2], 1),
y_axis=np.arange(0, xcor_map.shape[1], 1),
zxx=xcor_map[0],
output='2d',
label=['xcor step', 'freq'],
title='(-2, 2)')
fig_2 = heatmap_visualizer(
x_axis=np.arange(0, xcor_map.shape[2], 1),
y_axis=np.arange(0, xcor_map.shape[1], 1),
zxx=xcor_map[1],
output='2d',
label=['xcor step', 'freq'],
title='(-2, 4)')
fig_3 = heatmap_visualizer(
x_axis=np.arange(0, xcor_map.shape[2], 1),
y_axis=np.arange(0, xcor_map.shape[1], 1),
zxx=xcor_map[2],
output='2d',
label=['xcor step', 'freq'],
title='(-2, 6)')
fig_1_title = '{}sample{}_xcormap(-2, 2)'.format(
savepath, progress)
fig_2_title = '{}sample{}_xcormap(-2, 4)'.format(
savepath, progress)
fig_3_title = '{}sample{}_xcormap(-2, 6)'.format(
savepath, progress)
fig_1.savefig(fig_1_title)
fig_2.savefig(fig_2_title)
fig_3.savefig(fig_3_title)
plt.close('all')
class_1.append(xcor_map[0, 10:20, 300:500])
# update progress
pb.update(now=progress)
progress += 1
pb.destroy()
class_1 = np.array(class_1)
print('Dataset Dim: ', class_1.shape)
return class_1
def plb(self):
'''
This function returns a dataset of xcor map, each belongs to a class of PLB captured by 2 sensors at different
distance. 51 samples for each class.
:return:
Xcor map dataset where axis[0] -> total sample size of all classes
axis[1] -> frequency bin
axis[2] -> time step (xcor steps)
'''
# initialize
n_channel_data_near = read_all_tdms_from_folder(self.path_plb_2to12)
n_channel_data_far = read_all_tdms_from_folder(self.path_plb_10to22)
# take only first 51 samples, so it has same class sizes with data_near
n_channel_data_far = n_channel_data_far[:51]
# swap axis, so shape[0] is sensor (for easy using)
n_channel_data_near = np.swapaxes(n_channel_data_near, 1,
2) # swap axis[1] and [2]
n_channel_data_far = np.swapaxes(n_channel_data_far, 1, 2)
# -----------[BANDPASS + STFT + XCOR]-------------
# xcor pairing commands - [near] = 0m, 1m,..., 10m
sensor_pair_near = self.sensor_pair_near
# invert the sensor pair to generate the opposite lag
sensor_pair_near_inv = [(pair[1], pair[0])
for pair in sensor_pair_near]
# xcor pairing commands - [far] = 11m, 12m,..., 20m
sensor_pair_far = self.sensor_pair_far
# invert the sensor pair to generate the opposite lag
sensor_pair_far_inv = [(pair[1], pair[0]) for pair in sensor_pair_far]
# horizontal segmentation of the xcormap (along xcor step axis)
xcormap_extent = (250, 550)
# initialize a dict of lists for every classes
all_class = {}
for i in range(-20, 21, 1):
all_class['class_[{}]'.format(i)] = []
# class_1, class_2, class_3, class_4, class_5, class_6, class_7, class_8, class_9, class_10 = \
# [], [], [], [], [], [], [], [], [], []
# Sensor [Near] -------------------------------------------------
# initiate progressbar
pb = ProgressBarForLoop(title='Bandpass + STFT + XCOR --> [Near]',
end=n_channel_data_near.shape[0])
progress = 0
# for all plb samples
for sample in n_channel_data_near:
all_channel_stft = []
# for all sensors
for each_sensor_data in sample:
# band pass filter
filtered_signal = butter_bandpass_filtfilt(
sampled_data=each_sensor_data[90000:130000],
fs=1e6,
f_hicut=1e5,
f_locut=20e3)
# denoise
# dwt denoising setting
dwt_wavelet = 'db2'
dwt_smooth_level = 3
filtered_signal = dwt_smoothing(x=filtered_signal,
wavelet=dwt_wavelet,
level=dwt_smooth_level)
# stft
_, _, sxx, _ = spectrogram_scipy(sampled_data=filtered_signal,
fs=1e6,
mode='magnitude',
nperseg=100,
nfft=500,
noverlap=0,
return_plot=False,
verbose=False)
all_channel_stft.append(
sxx[10:51, :]) # index_10 -> f=20kHz; index_50 -> f=100kHz
all_channel_stft = np.array(all_channel_stft)
# xcor for sensor pair (0m) - (10m)
xcor_map, _ = one_dim_xcor_2d_input(input_mat=all_channel_stft,
pair_list=sensor_pair_near,
verbose=False)
for i in range(0, 11, 1):
all_class['class_[{}]'.format(i)].append(
xcor_map[i, 10:20, xcormap_extent[0]:xcormap_extent[1]])
# xcor for sensor pair (0m) - (-10m)
xcor_map, _ = one_dim_xcor_2d_input(input_mat=all_channel_stft,
pair_list=sensor_pair_near_inv,
verbose=False)
for i in range(0, 11, 1):
all_class['class_[{}]'.format(-i)].append(
xcor_map[i, 10:20, xcormap_extent[0]:xcormap_extent[1]])
# update progress bar
pb.update(now=progress)
progress += 1
pb.destroy()
# shuffle the class_0 and truncate only 51 samples
shuffle(all_class['class_[0]'])
all_class['class_[0]'] = all_class['class_[0]'][:51]
# Sensor [Far] -------------------------------------------------
# initiate progressbar
pb = ProgressBarForLoop(title='Bandpass + STFT + XCOR --> [Far]',
end=n_channel_data_near.shape[0])
progress = 0
for sample in n_channel_data_far:
all_channel_stft = []
# for all sensors
for each_sensor_data in sample:
# band pass filter
filtered_signal = butter_bandpass_filtfilt(
sampled_data=each_sensor_data[90000:130000],
fs=1e6,
f_hicut=1e5,
f_locut=20e3)
# stft
_, _, sxx, _ = spectrogram_scipy(sampled_data=filtered_signal,
fs=1e6,
mode='magnitude',
nperseg=100,
nfft=500,
noverlap=0,
return_plot=False,
verbose=False)
all_channel_stft.append(
sxx[10:51, :]) # index_10 -> f=20kHz; index_50 -> f=100kHz
all_channel_stft = np.array(all_channel_stft)
# horizontal segmentation of the xcormap (along xcor step axis)
xcormap_extent = (250, 550)
# xcor for sensor pair (11m) - (20m)
xcor_map, _ = one_dim_xcor_2d_input(input_mat=all_channel_stft,
pair_list=sensor_pair_far,
verbose=False)
for i in range(0, 10, 1):
all_class['class_[{}]'.format(i + 11)].append(
xcor_map[i, 10:20, xcormap_extent[0]:xcormap_extent[1]])
# xcor for sensor pair (11m) - (-20m)
xcor_map, _ = one_dim_xcor_2d_input(input_mat=all_channel_stft,
pair_list=sensor_pair_far_inv,
verbose=False)
for i in range(0, 10, 1):
all_class['class_[{}]'.format(-11 - i)].append(
xcor_map[i, 10:20, xcormap_extent[0]:xcormap_extent[1]])
# visualize and saving the training data
# savepath = 'C:/Users/YH/PycharmProjects/AE-signal-model/result/'
# visualize = False
# if visualize:
# for i in range(xcor_map.shape[0]):
# fig = three_dim_visualizer(x_axis=np.arange(xcormap_extent[0], xcormap_extent[1], 1),
# y_axis=np.arange(10, 20, 1),
# zxx=xcor_map[i, 10:20, xcormap_extent[0]:xcormap_extent[1]],
# output='2d',
# label=['xcor step', 'freq'],
# title='({}, {}) = -{}m'.format(sensor_pair_far_inv[i][0],
# sensor_pair_far_inv[i][1],
# i+11))
# fig_title = '{}sample{}_xcormap(dist=-{}m)'.format(savepath, progress, i+11)
# fig.savefig(fig_title)
# plt.close('all')
# update progress
pb.update(now=progress)
progress += 1
pb.destroy()
dataset = []
for i in range(-20, 21, 1):
temp = np.array(all_class['class_[{}]'.format(i)])
dataset.append(temp)
dataset = np.concatenate(dataset, axis=0)
label = [[i] * 51 for i in np.arange(-20, 21, 1)]
label = np.array([item for l in label for item in l])
print('Dataset Dim: ', dataset.shape)
print('Label Dim: ', label.shape)
return dataset, label