def get_avg_spectrums(signals, frequency, avg_sp_params):
smooth_avg_specs = None
for index in range(signals.shape[1]):
s_avg_spec = average_spectrum(
signal=signals[:, index],
frequency=frequency,
window=avg_sp_params['window'],
offset=avg_sp_params['offset'],
med_filter=avg_sp_params['med_filter'],
marmett_filter=avg_sp_params['marmett_filter'])
if smooth_avg_specs is None:
smooth_avg_specs = s_avg_spec
else:
smooth_avg_specs = np.column_stack(
(smooth_avg_specs, s_avg_spec[:, 1]))
return smooth_avg_specs
def get_avg_spectrums(files_list, frequency, avg_sp_params):
signals = None
for path in files_list:
signal = np.loadtxt(path, delimiter=' ')[:, 1]
if signals is None:
signals = signal
else:
signals = np.column_stack((signals, signal))
smooth_avg_specs = None
for index in range(signals.shape[1]):
s_avg_spec = average_spectrum(
signal=signals[:, index],
frequency=frequency,
window=avg_sp_params['window'],
offset=avg_sp_params['offset'],
med_filter=avg_sp_params['med_filter'],
marmett_filter=avg_sp_params['marmett_filter'])
if smooth_avg_specs is None:
smooth_avg_specs = s_avg_spec
else:
smooth_avg_specs = np.column_stack(
(smooth_avg_specs, s_avg_spec[:, 1]))
return smooth_avg_specs
def proc_file(src_file_name, detrend_type, proc_type, files_list, frequency,
avg_sp_params, cepster_params, nak_sp_params, output_path):
# print(f'Processing: {src_file_name}')
signals = None
for path in files_list:
signal = np.loadtxt(path, delimiter=' ')[:, 1]
if str(np.max(signal)) == 'nan':
print('Bad signal')
return
if signals is None:
signals = signal
else:
signals = np.column_stack((signals, signal))
no_smooth_avg_specs = None
smooth_avg_specs = None
for index in range(signals.shape[1]):
ns_avg_spec = average_spectrum(signal=signals[:, index],
frequency=frequency,
window=avg_sp_params['window'],
offset=avg_sp_params['offset'],
med_filter=None,
marmett_filter=None)
s_avg_spec = average_spectrum(
signal=signals[:, index],
frequency=frequency,
window=avg_sp_params['window'],
offset=avg_sp_params['offset'],
med_filter=avg_sp_params['med_filter'],
marmett_filter=avg_sp_params['marmett_filter'])
if no_smooth_avg_specs is None:
no_smooth_avg_specs = ns_avg_spec
smooth_avg_specs = s_avg_spec
else:
no_smooth_avg_specs = np.column_stack(
(no_smooth_avg_specs, ns_avg_spec[:, 1]))
smooth_avg_specs = np.column_stack(
(smooth_avg_specs, s_avg_spec[:, 1]))
smooth_cep_specs = None
no_smooth_cep_specs = None
for i in range(signals.shape[1]):
spectrum = np.zeros(shape=(no_smooth_avg_specs.shape[0], 2))
spectrum[:, 0] = no_smooth_avg_specs[:, 0]
spectrum[:, 1] = no_smooth_avg_specs[:, i + 1]
ns_cep_spec = cepstral_spectrum_from_spectrum(spectrum_data=spectrum)
spectrum[:, 1] = smooth_avg_specs[:, i + 1]
s_cep_spec = cepstral_spectrum_from_spectrum(spectrum_data=spectrum)
if smooth_cep_specs is None:
smooth_cep_specs = s_cep_spec
no_smooth_cep_specs = ns_cep_spec
else:
smooth_cep_specs = np.column_stack(
(smooth_cep_specs, s_cep_spec[:, 1]))
no_smooth_cep_specs = np.column_stack(
(no_smooth_cep_specs, ns_cep_spec[:, 1]))
no_smooth_nak_spec = nakamura_spectrum(
components_spectrum_data=no_smooth_avg_specs,
components_order='XYZ',
spectrum_type=nak_sp_params['type'])
smooth_nak_spec = nakamura_spectrum(
components_spectrum_data=smooth_avg_specs,
components_order='XYZ',
spectrum_type=nak_sp_params['type'])
# export data
out_folder = os.path.join(output_path, 'AverageSpectrums', proc_type,
detrend_type)
if not os.path.exists(out_folder):
os.makedirs(out_folder)
file_name = f'{src_file_name}.sc'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
no_smooth_avg_specs,
delimiter='\t',
header='Frequency\tX\tY\tZ',
comments='')
file_name = f'{src_file_name}.ssc'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
smooth_avg_specs,
delimiter='\t',
header='Frequency\tX\tY\tZ',
comments='')
for index, component in enumerate('XYZ'):
plot_average_spectrum(
frequency=smooth_avg_specs[:, 0],
spectrum_begin_amplitudes=no_smooth_avg_specs[:, index + 1],
spectrum_smooth_amplitudes=smooth_avg_specs[:, index + 1],
f_min=avg_sp_params['f_min'],
f_max=avg_sp_params['f_max'],
output_folder=out_folder,
output_name=f'{src_file_name}_Component_{component}')
out_folder = os.path.join(output_path, 'CepsterSpectrums', proc_type,
detrend_type)
if not os.path.exists(out_folder):
os.makedirs(out_folder)
file_name = f'{src_file_name}.cep'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
no_smooth_cep_specs,
delimiter='\t',
header='Time\tX\tY\tZ',
comments='')
file_name = f'{src_file_name}.scep'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
smooth_cep_specs,
delimiter='\t',
header='Time\tX\tY\tZ',
comments='')
for index, component in enumerate('XYZ'):
plot_average_spectrum(
frequency=smooth_cep_specs[:, 0],
spectrum_begin_amplitudes=no_smooth_cep_specs[:, index + 1],
spectrum_smooth_amplitudes=smooth_cep_specs[:, index + 1],
f_min=cepster_params['t_min'],
f_max=cepster_params['t_max'],
output_folder=out_folder,
output_name=f'Cepster_{src_file_name}_Component_{component}')
out_folder = os.path.join(output_path, 'NakamuraSpectrum', proc_type,
detrend_type)
if not os.path.exists(out_folder):
os.makedirs(out_folder)
file_name = f'{nak_sp_params["type"]}_{src_file_name}.nak'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
no_smooth_nak_spec,
delimiter='\t',
header='Freq\tAmplitude',
comments='')
file_name = f'{nak_sp_params["type"]}_{src_file_name}.snak'
out_path = os.path.join(out_folder, file_name)
np.savetxt(out_path,
smooth_nak_spec,
delimiter='\t',
header='Freq\tAmplitude',
comments='')
plot_average_spectrum(
frequency=smooth_nak_spec[:, 0],
spectrum_begin_amplitudes=no_smooth_nak_spec[:, 1],
spectrum_smooth_amplitudes=smooth_nak_spec[:, 1],
f_min=nak_sp_params['f_min'],
f_max=nak_sp_params['f_max'],
output_folder=out_folder,
output_name=f'{nak_sp_params["type"]}_{src_file_name}')
break
right_time_lim = bin_data.datetime_start + timedelta(
minutes=(i + 1) * time_step_minutes)
if right_time_lim > bin_data.datetime_stop:
right_time_lim = bin_data.datetime_stop
bin_data.read_date_time_start = left_time_lim
bin_data.read_date_time_stop = right_time_lim
signal_data = bin_data.ordered_signal_by_components
for component in components:
signal = signal_data[:, 'XYZ'.index(component)]
smooth_av_spec = average_spectrum(
signal=signal,
frequency=bin_data.signal_frequency,
window=window_size,
overlap=overlap_size,
med_filter=median_param,
marmett_filter=marmett_param)
no_smooth_av_spectrum = average_spectrum(
signal=signal,
frequency=bin_data.signal_frequency,
window=window_size,
overlap=overlap_size,
med_filter=None,
marmett_filter=None)
exp_folder = os.path.join(export_folder, name,
f'{component}-Component')
date_fmt = '%Y-%m-%d-%H-%M-%S'
export_name = f'{name}-{left_time_lim.strftime(date_fmt)}'
def run(self):
params = self.parameters
dt = (params['dt_stop'] - params['dt_start']).total_seconds()
discrete_amount = int(round(dt * params['resample_frequency']))
components_amount = len(params['components'])
files_amount = len(self.files_info.keys())
frequencies_list = rfftfreq(params['window_size'],
1 / params['resample_frequency'])
frequency_count = frequencies_list.shape[0]
joined_signal_array = np.empty(
shape=(components_amount, discrete_amount, files_amount),
dtype=np.int)
average_spectrum_data = np.empty(
shape=(components_amount, 2, frequency_count, files_amount),
dtype=np.float)
bin_files_list = list(self.files_info.keys())
for index in range(len(bin_files_list)):
bin_files_list[index] = bin_files_list[index].split('.')[0]
finished_parts_count = 0
for index_a, key in enumerate(self.files_info):
cur_file_data = self.files_info[key]
file_path = cur_file_data['path']
bin_data = BinaryFile()
bin_data.path = file_path
bin_data.use_avg_values = True
bin_data.resample_frequency = params['resample_frequency']
bin_data.read_date_time_start = params['dt_start']
bin_data.read_date_time_stop = params['dt_stop']
signals = bin_data.signals
if signals.shape[0]>discrete_amount:
signals=signals[:discrete_amount]
if params['detrend_frequency']!=0:
for i in range(signals.shape[1]):
signals[:,i]=detrend(signal=signals[:,i],
frequency=bin_data.resample_frequency,
edge_frequency=params['detrend_frequency'])
record_type=bin_data.record_type
for index_b, component in enumerate(params['components']):
compoment_signal = signals[:, record_type.index(component)]
joined_signal_array[index_b, :, index_a] = compoment_signal
# calculating average unsmoothed spectrum
av_spec_simple_component = average_spectrum(
signal=compoment_signal,
frequency=params['resample_frequency'],
window=params['window_size'],
offset=params['overlap_size'],
med_filter=None,
marmett_filter=None)
average_spectrum_data[index_b, 0, :, index_a] = \
av_spec_simple_component[:, 1]
# calculating average smoothed spectrum
av_spec_smooth_component = average_spectrum(
signal=compoment_signal,
frequency=params['resample_frequency'],
window=params['window_size'],
offset=params['overlap_size'],
med_filter=params['median_filter'],
marmett_filter=params['marmett_filter'])
average_spectrum_data[index_b, 1, :, index_a] = \
av_spec_smooth_component[:, 1]
finished_parts_count += 1
percent_value = finished_parts_count / self.parts_count * 100
self.finished_percent.emit(percent_value)
result_correlate_matrix = np.empty(
shape=(components_amount, files_amount, files_amount),
dtype=np.float)
for component in range(components_amount):
result_correlate_matrix[component, :, :] = cross_correlation(
frequency=frequencies_list,
f_min_analysis=params['correlation_min_frequency'],
f_max_analysis=params['correlation_max_frequency'],
amplitudes=average_spectrum_data[component, 1, :, :])
# result data exporting
read_dt_start_label = datetime.strftime(params['dt_start'],
'%Y-%m-%d_%H-%M-%S')
for index_a, bin_file_name in enumerate(self.files_info):
bin_file_name = bin_file_name.split('.')[0]
file_processing_result_folder = \
os.path.join(params['output_folder'], bin_file_name)
if not os.path.exists(file_processing_result_folder):
os.mkdir(file_processing_result_folder)
for index_b, component_label in enumerate(params['components']):
if params['is_selection_signal_file']:
dat_file_name = f'{bin_file_name}_ClearSignal_' \
f'{component_label}_Component.dat'
export_path = os.path.join(file_processing_result_folder,
dat_file_name)
np.savetxt(fname=export_path,
X=joined_signal_array[index_b, :, index_a],
fmt='%i')
if params['is_selection_signal_graph']:
png_file_name = \
f'{bin_file_name}_ClearSignal_{component_label}_{read_dt_start_label}'
plot_signal(time_start_sec=0,
frequency=params['resample_frequency'],
signal=joined_signal_array[index_b, :, index_a],
label=png_file_name,
output_folder=file_processing_result_folder,
output_name=png_file_name)
if params['is_spectrum_graph']:
png_file_name = '{}_AverageSpectrum_{}_Component_' \
'Graph'.format(bin_file_name,
component_label)
plot_average_spectrum(
frequency=frequencies_list,
origin_amplitudes=average_spectrum_data[
index_b, 0, :, index_a],
smooth_amplitudes=average_spectrum_data[
index_b, 1, :, index_a],
f_min=params['visual_min_frequency'],
f_max=params['visual_max_frequency'],
output_folder=file_processing_result_folder,
output_name=png_file_name)
if params['is_unsmoothed_spectrum_file']:
file_name = \
f'{bin_file_name}_{component_label}_Component.sc'
export_path = os.path.join(
file_processing_result_folder, file_name)
temp_array = np.empty(
shape=(frequencies_list.shape[0], 2),
dtype=np.float)
temp_array[:, 0] = frequencies_list
temp_array[:, 1] = average_spectrum_data[index_b, 0, :,
index_a]
np.savetxt(fname=export_path, X=temp_array, fmt='%f',
delimiter='\t')
if params['is_smoothed_spectrum_file']:
file_name = \
f'{bin_file_name}_{component_label}_Component.ssc'
export_path = os.path.join(
file_processing_result_folder, file_name)
temp_array = np.empty(
shape=(frequencies_list.shape[0], 2),
dtype=np.float)
temp_array[:, 0] = frequencies_list
temp_array[:, 1] = \
average_spectrum_data[index_b, 1, :, index_a]
np.savetxt(fname=export_path, X=temp_array, fmt='%f',
delimiter='\t')
if params['is_correlation_graph']:
png_file_name = \
f'{bin_file_name}_Separate_Correlation_{component_label}_Component_Graph'
plot_single_correlation(
devices=bin_files_list,
correlation_data=result_correlate_matrix[
index_b, index_a, :],
output_folder=file_processing_result_folder,
output_name=png_file_name)
finished_parts_count += 1
percent_value = finished_parts_count / self.parts_count * 100
self.finished_percent.emit(percent_value)
colors = list()
for i in range(len(self.files_info.keys())):
colors.append(generate_random_color())
for index_a, component_label in enumerate(params['components']):
if params['is_general_smooth_spectrum_graph']:
file_name = f'SmoothSpectrums_{component_label}_Component'
plot_all_smooth_spectrums(
spectrums_name_list=bin_files_list,
frequency=frequencies_list,
spectrum_data=average_spectrum_data[index_a, 1, :, :],
f_min_visualize=params['visual_min_frequency'],
f_max_visualize=params['visual_max_frequency'],
colors=colors, output_folder=params['output_folder'],
output_name=file_name)
if params['is_correlation_matrix_file']:
file_name = f'CorrelationMatrix_{component_label}_Component.dat'
header = 'NULL\t' + '\t'.join(bin_files_list) + '\n'
write_lines = list()
write_lines.append(header)
corr_matrix = result_correlate_matrix[index_a, :, :]
for i in range(corr_matrix.shape[0]):
t = list()
t.append(bin_files_list[i])
for j in range(corr_matrix.shape[1]):
t.append(str(corr_matrix[i, j]))
s = '\t'.join(t) + '\n'
write_lines.append(s)
export_path = os.path.join(params['output_folder'], file_name)
f = open(export_path, 'w')
for line in write_lines:
f.write(line)
f.close()
if params['is_general_correlation_graph']:
file_name = f'Correlations_{component_label}_Component'
plot_correlation(devices=bin_files_list,
colors=colors,
correlation_matrix=result_correlate_matrix[
index_a, :, :],
output_folder=params['output_folder'],
output_name=file_name)
def calculation(self):
folder_path = self._ui.leInputFolder.text()
export_folder_path = self._ui.leOutputFolder.text()
file_list = self._ui.teFilesList.toPlainText()
dt_start = self._ui.dtMinDt.dateTime()
dt_stop = self._ui.dtMaxDt.dateTime()
if folder_path is None or file_list is None or export_folder_path is None:
return None
files = file_list.split('\n')
extract_data = None
for index, file in enumerate(files):
full_path = os.path.join(folder_path, file)
bin_data = BinaryFile()
bin_data.path = full_path
bin_data.record_type = 'XYZ'
bin_data.use_avg_values = False
bin_data.read_date_time_start = dt_start.toPyDateTime()
bin_data.read_date_time_stop = dt_stop.toPyDateTime()
bin_data.resample_frequency = self._ui.sbResampleFrequency.value()
if bin_data.signals is None:
continue
if extract_data is None:
discrete_count = bin_data.signals.shape[0]
extract_data = np.empty(shape=(discrete_count, 3 * len(files)),
dtype=np.int)
extract_data[:, 3 * index:3 * (index + 1)] = bin_data.signals
if extract_data is None:
return None
sum_trace = np.empty(shape=(extract_data.shape[0], 3),
dtype=np.int)
channel_plots = [self.channel_a_plot, self.channel_b_plot, self.channel_c_plot]
spectrum_plots = [self.channel_a_spectrum, self.channel_b_spectrum,
self.channel_c_spectrum]
window_size = self._ui.sbWindowSize.value()
overlap_size = self._ui.sbOverlapSize.value()
median_filter = self._ui.sbMedianFilter.value()
marmett_filter = self._ui.sbMarmettFilter.value()
resample_frequency = self._ui.sbResampleFrequency.value()
if marmett_filter == 0:
marmett_filter = None
if median_filter == 0:
median_filter = None
spectrums = None
for i in range(3):
channel_signals = extract_data[:, i:3 * len(files):3]
sum_trace[:, i] = np.sum(channel_signals, axis=1)
channel_plot = channel_plots[i]
spectrum_plot = spectrum_plots[i]
channel_plot.clear()
channel_plot.plot(sum_trace[:, i], pen=(196, 255, 18))
spectrum_plot.clear()
spectrum_data = average_spectrum(signal=sum_trace[:, i],
frequency=resample_frequency,
window=window_size,
overlap=overlap_size,
med_filter=median_filter,
marmett_filter=marmett_filter)
if spectrums is None:
spectrums = np.empty(shape=(spectrum_data.shape[0], 4),
dtype=np.float)
spectrums[:, :2] = spectrum_data
else:
spectrums[:, 1 + i] = spectrum_data[:, 1]
spectrum_plot.plot(spectrum_data, pen=(255, 0, 0))
tmp_file_a = os.path.join(export_folder_path, 'sum_trace.npy')
tmp_file_b = os.path.join(export_folder_path, 'spectrums.npy')
np.save(tmp_file_a, sum_trace)
np.save(tmp_file_b, spectrums)
output_file_sum_trace = os.path.join(export_folder_path, 'SumTraces.dat')
output_file_spectrums = os.path.join(export_folder_path, 'Spectrums.dat')
np.savetxt(output_file_sum_trace, sum_trace, fmt='%i', delimiter='\t',
header='Channel_1\tChannel_2\tChannel_3', comments='')
np.savetxt(output_file_spectrums, spectrums, fmt='%f', delimiter='\t',
header='Frequency\tChannel_1\tChannel_2\tChannel_3',
comments='')
number = int(t[0])
x, y = [float(q) for q in t[1:]]
coords_data[number] = (x, y)
f_intervals = [[4, 5], [28, 30]]
intervals_count = len(f_intervals)
point_count = len(sensor_numbers)
result = None
freq_array = None
for index in range(point_count):
signal = signal_data[:, 1 + 3 * index:1 + 3 * (index + 1)]
sp_x = average_spectrum(signal=signal[:, 0],
frequency=frequency,
window=8192,
offset=4096,
med_filter=7,
marmett_filter=7)
sp_y = average_spectrum(signal=signal[:, 1],
frequency=frequency,
window=8192,
offset=4096,
med_filter=7,
marmett_filter=7)
sp_z = average_spectrum(signal=signal[:, 2],
frequency=frequency,
window=8192,
offset=4096,
med_filter=7,
marmett_filter=7)