def run(self):
params = self.parameters
for index_i, item_i in enumerate(params):
export_name, file_stat = item_i
export_folder = os.path.dirname(file_stat[0][0])
if export_name[-3:] != '.00':
export_name += '.00'
output_file = open(os.path.join(export_folder, export_name), 'wb')
for index_j, item_j in enumerate(file_stat):
file_path, dt_start, dt_stop = item_j
bin_data = BinaryFile()
bin_data.path = file_path
bin_data.use_avg_values = False
if index_j == 0:
output_file.write(bin_data.main_header.get_binary_format())
for k in range(3):
output_file.write(
bin_data.get_channel_header(k).get_binary_format())
# channel_headers_hole = np.array([0] * 54, dtype=np.int32)
# channel_headers_hole.tofile(output_file)
signal = bin_data.signals
signal.astype(np.int32).tofile(output_file)
self.finished_part_stitching.emit(
(index_j + 1) / len(file_stat) * 100)
self.finished_file.emit((index_i + 1) / len(params) * 100)
def load_origin_signal(self):
if self.__parameters is None:
self.__origin_signal = None
self.__origin_spectrogram = None
if self.__parameters is not None and self.__origin_signal is None:
params = self.__parameters
bin_data = BinaryFile()
bin_data.path = params['file_path']
bin_data.use_avg_values = False
bin_data.resample_frequency = params['resample_frequency']
bin_data.read_date_time_start = params['dt_start']
bin_data.read_date_time_stop = params['dt_stop']
if bin_data.signals is None:
self.__origin_signal = None
self.__origin_spectrogram = None
else:
index_channel = 'XYZ'.index(params['component'])
signal = bin_data.ordered_signal_by_components[:,
index_channel]
self.__origin_signal = signal
self.__origin_spectrogram = specgram(
signal_data=signal,
frequency_of_signal=params['resample_frequency'])
marmett_param = 7
time_step_minutes = 60
components = ['X', 'Y', 'Z']
f_min = 1
f_max = 10
# ----------------------- Don't touch ---------------------------------------
for root, folders, files in os.walk(root_folder):
for file in files:
name, extension = file.split('.')
if extension not in ['00', 'xx', 'bin']:
continue
bin_data = BinaryFile()
bin_data.path = os.path.join(root, file)
bin_data.use_avg_values = False
i = 0
while True:
left_time_lim = bin_data.datetime_start + timedelta(
minutes=i * time_step_minutes)
if left_time_lim > bin_data.datetime_stop:
break
right_time_lim = bin_data.datetime_start + timedelta(
minutes=(i + 1) * time_step_minutes)
if right_time_lim > bin_data.datetime_stop:
right_time_lim = bin_data.datetime_stop
bin_data.read_date_time_start = left_time_lim
bin_data.read_date_time_stop = right_time_lim
signal_data = bin_data.ordered_signal_by_components
def run(self):
finished_parts_count = 0
params = self.parameters
for index, key in enumerate(self.files_info):
cur_file_data = self.files_info[key]
dt_start = cur_file_data['datetime_start']
dt_stop = cur_file_data['datetime_stop']
file_path = cur_file_data['path']
file_name = os.path.basename(file_path).split('.')[0]
cross_dt_start = max(params['dt_start'], dt_start)
cross_dt_stop = min(params['dt_stop'], dt_stop)
bin_data = BinaryFile()
bin_data.path = file_path
bin_data.use_avg_values = True
bin_data.resample_frequency = params['resample_frequency']
part_count = int(round(
(cross_dt_stop - cross_dt_start).total_seconds() // (
params['interval_size'] * 60)))
for i in range(part_count):
read_dt_start = cross_dt_start + timedelta(
minutes=params['interval_size'] * i)
read_dt_stop = read_dt_start + timedelta(
minutes=params['interval_size'])
read_dt_start_label = datetime.strftime(read_dt_start,
'%Y-%m-%d_%H-%M-%S')
bin_data.read_date_time_start = read_dt_start
bin_data.read_date_time_stop = read_dt_stop
signals = bin_data.signals
record_type=bin_data.record_type
for component in params['components']:
component_signal=signals[:, record_type.index(component)]
output_folder = os.path.join(params['output_folder'],
file_name,
f'{component}Component')
if not os.path.exists(output_folder):
os.makedirs(output_folder)
output_name = f'SP_{component}Component_{file_name}_{read_dt_start_label}'
create_spectrogram(signal_data=component_signal,
frequency=params['resample_frequency'],
output_folder=output_folder,
output_name=output_name,
min_frequency=params['visual_min_frequency'],
max_frequency=params['visual_max_frequency'],
time_start=0)
output_name = f'SIG_{component}Component_{file_name}_{read_dt_start_label}'
plot_signal(time_start_sec=0,
frequency=params['resample_frequency'],
signal=component_signal, label=output_name,
output_folder=output_folder,
output_name=output_name)
finished_parts_count += 1
percent_value = finished_parts_count / self.parts_count * 100
self.finished_percent.emit(percent_value)
def run(self):
params = self.parameters
dt = (params['dt_stop'] - params['dt_start']).total_seconds()
discrete_amount = int(round(dt * params['resample_frequency']))
components_amount = len(params['components'])
files_amount = len(self.files_info.keys())
frequencies_list = rfftfreq(params['window_size'],
1 / params['resample_frequency'])
frequency_count = frequencies_list.shape[0]
joined_signal_array = np.empty(
shape=(components_amount, discrete_amount, files_amount),
dtype=np.int)
average_spectrum_data = np.empty(
shape=(components_amount, 2, frequency_count, files_amount),
dtype=np.float)
bin_files_list = list(self.files_info.keys())
for index in range(len(bin_files_list)):
bin_files_list[index] = bin_files_list[index].split('.')[0]
finished_parts_count = 0
for index_a, key in enumerate(self.files_info):
cur_file_data = self.files_info[key]
file_path = cur_file_data['path']
bin_data = BinaryFile()
bin_data.path = file_path
bin_data.use_avg_values = True
bin_data.resample_frequency = params['resample_frequency']
bin_data.read_date_time_start = params['dt_start']
bin_data.read_date_time_stop = params['dt_stop']
signals = bin_data.signals
if signals.shape[0]>discrete_amount:
signals=signals[:discrete_amount]
if params['detrend_frequency']!=0:
for i in range(signals.shape[1]):
signals[:,i]=detrend(signal=signals[:,i],
frequency=bin_data.resample_frequency,
edge_frequency=params['detrend_frequency'])
record_type=bin_data.record_type
for index_b, component in enumerate(params['components']):
compoment_signal = signals[:, record_type.index(component)]
joined_signal_array[index_b, :, index_a] = compoment_signal
# calculating average unsmoothed spectrum
av_spec_simple_component = average_spectrum(
signal=compoment_signal,
frequency=params['resample_frequency'],
window=params['window_size'],
offset=params['overlap_size'],
med_filter=None,
marmett_filter=None)
average_spectrum_data[index_b, 0, :, index_a] = \
av_spec_simple_component[:, 1]
# calculating average smoothed spectrum
av_spec_smooth_component = average_spectrum(
signal=compoment_signal,
frequency=params['resample_frequency'],
window=params['window_size'],
offset=params['overlap_size'],
med_filter=params['median_filter'],
marmett_filter=params['marmett_filter'])
average_spectrum_data[index_b, 1, :, index_a] = \
av_spec_smooth_component[:, 1]
finished_parts_count += 1
percent_value = finished_parts_count / self.parts_count * 100
self.finished_percent.emit(percent_value)
result_correlate_matrix = np.empty(
shape=(components_amount, files_amount, files_amount),
dtype=np.float)
for component in range(components_amount):
result_correlate_matrix[component, :, :] = cross_correlation(
frequency=frequencies_list,
f_min_analysis=params['correlation_min_frequency'],
f_max_analysis=params['correlation_max_frequency'],
amplitudes=average_spectrum_data[component, 1, :, :])
# result data exporting
read_dt_start_label = datetime.strftime(params['dt_start'],
'%Y-%m-%d_%H-%M-%S')
for index_a, bin_file_name in enumerate(self.files_info):
bin_file_name = bin_file_name.split('.')[0]
file_processing_result_folder = \
os.path.join(params['output_folder'], bin_file_name)
if not os.path.exists(file_processing_result_folder):
os.mkdir(file_processing_result_folder)
for index_b, component_label in enumerate(params['components']):
if params['is_selection_signal_file']:
dat_file_name = f'{bin_file_name}_ClearSignal_' \
f'{component_label}_Component.dat'
export_path = os.path.join(file_processing_result_folder,
dat_file_name)
np.savetxt(fname=export_path,
X=joined_signal_array[index_b, :, index_a],
fmt='%i')
if params['is_selection_signal_graph']:
png_file_name = \
f'{bin_file_name}_ClearSignal_{component_label}_{read_dt_start_label}'
plot_signal(time_start_sec=0,
frequency=params['resample_frequency'],
signal=joined_signal_array[index_b, :, index_a],
label=png_file_name,
output_folder=file_processing_result_folder,
output_name=png_file_name)
if params['is_spectrum_graph']:
png_file_name = '{}_AverageSpectrum_{}_Component_' \
'Graph'.format(bin_file_name,
component_label)
plot_average_spectrum(
frequency=frequencies_list,
origin_amplitudes=average_spectrum_data[
index_b, 0, :, index_a],
smooth_amplitudes=average_spectrum_data[
index_b, 1, :, index_a],
f_min=params['visual_min_frequency'],
f_max=params['visual_max_frequency'],
output_folder=file_processing_result_folder,
output_name=png_file_name)
if params['is_unsmoothed_spectrum_file']:
file_name = \
f'{bin_file_name}_{component_label}_Component.sc'
export_path = os.path.join(
file_processing_result_folder, file_name)
temp_array = np.empty(
shape=(frequencies_list.shape[0], 2),
dtype=np.float)
temp_array[:, 0] = frequencies_list
temp_array[:, 1] = average_spectrum_data[index_b, 0, :,
index_a]
np.savetxt(fname=export_path, X=temp_array, fmt='%f',
delimiter='\t')
if params['is_smoothed_spectrum_file']:
file_name = \
f'{bin_file_name}_{component_label}_Component.ssc'
export_path = os.path.join(
file_processing_result_folder, file_name)
temp_array = np.empty(
shape=(frequencies_list.shape[0], 2),
dtype=np.float)
temp_array[:, 0] = frequencies_list
temp_array[:, 1] = \
average_spectrum_data[index_b, 1, :, index_a]
np.savetxt(fname=export_path, X=temp_array, fmt='%f',
delimiter='\t')
if params['is_correlation_graph']:
png_file_name = \
f'{bin_file_name}_Separate_Correlation_{component_label}_Component_Graph'
plot_single_correlation(
devices=bin_files_list,
correlation_data=result_correlate_matrix[
index_b, index_a, :],
output_folder=file_processing_result_folder,
output_name=png_file_name)
finished_parts_count += 1
percent_value = finished_parts_count / self.parts_count * 100
self.finished_percent.emit(percent_value)
colors = list()
for i in range(len(self.files_info.keys())):
colors.append(generate_random_color())
for index_a, component_label in enumerate(params['components']):
if params['is_general_smooth_spectrum_graph']:
file_name = f'SmoothSpectrums_{component_label}_Component'
plot_all_smooth_spectrums(
spectrums_name_list=bin_files_list,
frequency=frequencies_list,
spectrum_data=average_spectrum_data[index_a, 1, :, :],
f_min_visualize=params['visual_min_frequency'],
f_max_visualize=params['visual_max_frequency'],
colors=colors, output_folder=params['output_folder'],
output_name=file_name)
if params['is_correlation_matrix_file']:
file_name = f'CorrelationMatrix_{component_label}_Component.dat'
header = 'NULL\t' + '\t'.join(bin_files_list) + '\n'
write_lines = list()
write_lines.append(header)
corr_matrix = result_correlate_matrix[index_a, :, :]
for i in range(corr_matrix.shape[0]):
t = list()
t.append(bin_files_list[i])
for j in range(corr_matrix.shape[1]):
t.append(str(corr_matrix[i, j]))
s = '\t'.join(t) + '\n'
write_lines.append(s)
export_path = os.path.join(params['output_folder'], file_name)
f = open(export_path, 'w')
for line in write_lines:
f.write(line)
f.close()
if params['is_general_correlation_graph']:
file_name = f'Correlations_{component_label}_Component'
plot_correlation(devices=bin_files_list,
colors=colors,
correlation_matrix=result_correlate_matrix[
index_a, :, :],
output_folder=params['output_folder'],
output_name=file_name)
def calculation(self):
folder_path = self._ui.leInputFolder.text()
export_folder_path = self._ui.leOutputFolder.text()
file_list = self._ui.teFilesList.toPlainText()
dt_start = self._ui.dtMinDt.dateTime()
dt_stop = self._ui.dtMaxDt.dateTime()
if folder_path is None or file_list is None or export_folder_path is None:
return None
files = file_list.split('\n')
extract_data = None
for index, file in enumerate(files):
full_path = os.path.join(folder_path, file)
bin_data = BinaryFile()
bin_data.path = full_path
bin_data.record_type = 'XYZ'
bin_data.use_avg_values = False
bin_data.read_date_time_start = dt_start.toPyDateTime()
bin_data.read_date_time_stop = dt_stop.toPyDateTime()
bin_data.resample_frequency = self._ui.sbResampleFrequency.value()
if bin_data.signals is None:
continue
if extract_data is None:
discrete_count = bin_data.signals.shape[0]
extract_data = np.empty(shape=(discrete_count, 3 * len(files)),
dtype=np.int)
extract_data[:, 3 * index:3 * (index + 1)] = bin_data.signals
if extract_data is None:
return None
sum_trace = np.empty(shape=(extract_data.shape[0], 3),
dtype=np.int)
channel_plots = [self.channel_a_plot, self.channel_b_plot, self.channel_c_plot]
spectrum_plots = [self.channel_a_spectrum, self.channel_b_spectrum,
self.channel_c_spectrum]
window_size = self._ui.sbWindowSize.value()
overlap_size = self._ui.sbOverlapSize.value()
median_filter = self._ui.sbMedianFilter.value()
marmett_filter = self._ui.sbMarmettFilter.value()
resample_frequency = self._ui.sbResampleFrequency.value()
if marmett_filter == 0:
marmett_filter = None
if median_filter == 0:
median_filter = None
spectrums = None
for i in range(3):
channel_signals = extract_data[:, i:3 * len(files):3]
sum_trace[:, i] = np.sum(channel_signals, axis=1)
channel_plot = channel_plots[i]
spectrum_plot = spectrum_plots[i]
channel_plot.clear()
channel_plot.plot(sum_trace[:, i], pen=(196, 255, 18))
spectrum_plot.clear()
spectrum_data = average_spectrum(signal=sum_trace[:, i],
frequency=resample_frequency,
window=window_size,
overlap=overlap_size,
med_filter=median_filter,
marmett_filter=marmett_filter)
if spectrums is None:
spectrums = np.empty(shape=(spectrum_data.shape[0], 4),
dtype=np.float)
spectrums[:, :2] = spectrum_data
else:
spectrums[:, 1 + i] = spectrum_data[:, 1]
spectrum_plot.plot(spectrum_data, pen=(255, 0, 0))
tmp_file_a = os.path.join(export_folder_path, 'sum_trace.npy')
tmp_file_b = os.path.join(export_folder_path, 'spectrums.npy')
np.save(tmp_file_a, sum_trace)
np.save(tmp_file_b, spectrums)
output_file_sum_trace = os.path.join(export_folder_path, 'SumTraces.dat')
output_file_spectrums = os.path.join(export_folder_path, 'Spectrums.dat')
np.savetxt(output_file_sum_trace, sum_trace, fmt='%i', delimiter='\t',
header='Channel_1\tChannel_2\tChannel_3', comments='')
np.savetxt(output_file_spectrums, spectrums, fmt='%f', delimiter='\t',
header='Frequency\tChannel_1\tChannel_2\tChannel_3',
comments='')
from SeisCore.BinaryFile.BinaryFile import BinaryFile
import numpy as np
origin_file = r'/home/michael/Загрузки/K42/YM_027_K42_2019-11-30_09-49.00'
test_file = r'/home/michael/Загрузки/K42/parts/file.00'
origin_bin = BinaryFile()
origin_bin.path = origin_file
origin_bin.use_avg_values = False
print(origin_bin.discrete_amount, origin_bin.signals.shape[0])
test_bin = BinaryFile()
test_bin.path = test_file
test_bin.use_avg_values = False
for i in range(3):
diff = np.abs(origin_bin.signals[:, i] - test_bin.signals[:, i])
print(np.max(diff))