def data_integrity_check(root_dir, number_of_samples, number_of_files, contains=[], not_contains=[], extension=[]):
integrity_check = 1
samples_dirs = fio.get_root_dirs(root_dir, contains=contains, not_contains=not_contains, print_path = False)
if len(samples_dirs) != number_of_samples:
print('Wrong number of samples, {} of {} samples found'.format(len(samples_dirs), number_of_samples))
integrity_check = 0
else:
print('All files found')
for sample_dir in samples_dirs:
files = fio.get_files(sample_dir, extension=extension, contains=contains, not_contains=not_contains, search_subdirs = False, print_path = False)
if len(files) != number_of_files:
print('Wrong number of measured points at sample {}. {} of {} points found'.format(fio.get_part_of_path(sample_dir,-1), len(files), number_of_files))
integrity_check = 0
else:
pass
for file in files:
file = file.replace('\\', '_').split('.')[0].split('_')
if file[-1] == '110':
print('Voltage out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '101':
print('Temperature out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '011':
print('Pressure out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '001':
print('Pressure and temperature out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '010':
print('Pressure and voltage out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '100':
print('Temperature and voltage out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
elif file[-1] == '000':
print('Pressure, temperature and voltage out of limits for sample {} at point {} mbar {} °C {} mV'.format(file[-7],file[-4], file[-3], file[-2] ))
integrity_check = 0
else: pass
return integrity_check
# =============================================================================
# calculation settings variables declaration
# =============================================================================
column_indexes = [ 20, 21, 77 , 78 , 79 , 80 , 81 , 82 , 85 ]
column_names = ['t_MSP_ok', 't_BMP_ok', 'p_diff', 'p_BMP', 'p_MSP', 'valid_meas', 'pressure', 'voltage', 'temperature']
sorted_columns = ['pressure', 'voltage', 'temperature', 'valid_meas', 't_MSP_ok', 't_BMP_ok', 'p_diff', 'p_BMP', 'p_MSP' ]
plt.ioff()
# =============================================================================
# full calculation or only plots picker
# =============================================================================
calc = '1'
res_name = os.path.normpath('{}/{}/{}'.format(cwd, results_dir, results_name))
apaths = fio.get_files(cwd, extension = ['xlsx'], contains = [res_name], not_contains=['~'], print_path=False)
for name in apaths:
calc = input("results file has been found, calculate again ? ")
"""
# =============================================================================
# CALCULATION STARTING POINT
# =============================================================================
"""
if calc == '1' or calc == 'y':
# =============================================================================
# getting all data file names
# =============================================================================
abs_paths = fio.get_files(cwd, extension = ['xlsm'], contains = included_dirs_keywords, not_contains = excluded_dirs_keywords, print_path=False)
root_dirs = fio.get_parts_of_paths_list(abs_paths, -3)
root_dir_graphs = 'root_dir_graphs'
combined_graph = 'combined_graph'
combined_graph_name = 'data'
"""
# =============================================================================
# calculation
# =============================================================================
#"""
# creating dir structure for results
os.makedirs('{}/{}'.format(cwd, results_dir), exist_ok=True)
os.makedirs('{}/{}/{}'.format(cwd, results_dir, combined_graph), exist_ok=True)
os.makedirs('{}/{}/{}'.format(cwd, results_dir, single_file_graphs),
exist_ok=True)
abs_paths = fio.get_files(cwd,
extension=['csv'],
contains=used_dirs_keys,
not_contains=['~'],
print_path=False)
root_dirs = fio.get_parts_of_paths_list(abs_paths, -3)
sample_dirs = fio.get_parts_of_paths_list(abs_paths, -2)
filenames = fio.get_parts_of_paths_list(abs_paths, -1)
frame = pd.DataFrame({
'abs_paths': abs_paths,
'root_dir': root_dirs,
'sample_dir': sample_dirs,
'filename': filenames
})
unique_root_dirs = list(sorted(set(root_dirs)))
# =============================================================================
# MAIN LOOP
# =============================================================================
c_map = fmts.ColorMap()
c_map.assign_new_c_map(plt.get_cmap("tab10").colors)
a_paths = []
for cur_data in data:
if cur_data.line_calculation_settings_calculate == False: continue
else: pass
cur_data.root_dir = fio.get_part_of_path(
cur_data.root_dir_settings_root_dir, -1)
cur_data.all_files_paths = fio.get_files(
cur_data.root_dir_settings_root_dir, ["csv"], [], [])
a_paths = fio.get_files(cur_data.root_dir_settings_root_dir)
cur_data.file_names = fio.get_parts_of_paths_list(cur_data.all_files_paths,
-1)
cur_data.last_dirs_names = fio.get_parts_of_paths_list(
cur_data.all_files_paths, -2)
# root_dir = fio.get_part_of_given_os_path(cur_data.root_dir_settings_root_dir, -1)
# all_files_paths = fio.get_files_from_dirs(cur_data.root_dir_settings_root_dir,[],[], ["csv"])
# a_paths = fio.get_files_from_dirs(cur_data.root_dir_settings_root_dir)
# file_names = fio.get_parts_of_given_os_path_list(cur_data.all_files_paths, -1)
# last_dirs_names = fio.get_parts_of_given_os_path_list(cur_data.all_files_paths, -2)
if cur_data.line_label_plot_settings_text == "file_name":
cur_data.labels_line = [
"{}{}{}".format(cur_data.line_label_plot_settings_text_pref, i,
timedelta = current - start
times.append(timedelta.total_seconds())
return times
fplot.set_rc_params(font_size_offset=0,
figsize=(1400, 600),
linewidth=2,
markersize=6)
path = r'X:\Dnox\Erprobung\13_Data_transfer\Carda\BFP_volumetric_bypass_test\Logged data-7.5V-10.01A'
# =============================================================================
#
# =============================================================================
filesa = fio.get_files(path, contains=['A0'], extension=['csv'])
filesaa = fio.get_files(path, contains=['A1'], extension=['csv'])
filesa = filesa + filesaa
for file in filesa:
print(file)
dataframe = pd.read_csv(file, sep=';', decimal=',')
weight = dataframe[r'Weight [g]'].tolist()
time = dataframe.Time
time = time.tolist()
time = time_stamp_to_time_diff(time)
plt.plot(time, weight, 'b')
fplot.add_label('A', 'b')
filesb = fio.get_files(path, contains=['B0'], extension=['csv'])
filesbb = fio.get_files(path, contains=['B1'], extension=['csv'])
# Data integrity check
# =============================================================================
print("\nChecking data integrity\n------------------------")
data_integrity_check = fpmm.data_integrity_check(cwd, number_of_samples, number_of_files, contains=included_dirs_keywords, not_contains=excluded_dirs_keywords, extension=extension)
fio.terminate_question(data_integrity_check, "\nProceed or abort ? y/n ")
# =============================================================================
# Data reading
# =============================================================================
data_frame_list = []
sample_dirs = fio.get_root_dirs(cwd, contains=included_dirs_keywords, not_contains=excluded_dirs_keywords, print_path = False)
print()
for sample_dir in sample_dirs:
print("Processing sample.....{}".format(fio.get_part_of_path(sample_dir,-1)))
files = fio.get_files(sample_dir, extension=extension, contains=included_dirs_keywords, not_contains=excluded_dirs_keywords, search_subdirs = False, print_path = False)
current_file = 1
for file in files:
print('Processing sample {}....... ({:02d}/{:02d})'.format(fio.get_part_of_path(sample_dir,-1),current_file, len(files)), end="\r", flush=True)
current_file += 1
data_frame = fpmm.read_mdf_data(file)
data_frame_list.append(data_frame)
# sleep(0.5) # Time in seconds.
result_frame = pd.concat(data_frame_list, ignore_index=True)
# =============================================================================
# Data saving
# =============================================================================
os.makedirs('{}/{}'.format(cwd, results_dir), exist_ok=True)
writer = pd.ExcelWriter('{}/{}/{}'.format( cwd, results_dir, results_name), engine='xlsxwriter')
result_frame.to_excel(writer,'Sheet1', index=False)
script_path = os.path.normpath(os.path.abspath(__file__))
cwd = os.path.split(script_path)[0]
os.makedirs('{}/{}'.format(cwd, 'results'), exist_ok=True)
def find_nearest(array,value):
idx = np.searchsorted(array, value, side="left")
if idx > 0 and (idx == len(array) or math.fabs(value - array[idx-1]) < math.fabs(value - array[idx])):
return idx-1
else:
return idx
columns = ['sample','assembly force 1', 'assembly force 2', 'assembly force 3', 'assembly force 4', 'assembly force 5', 'assembly force 6', 'assembly force 7', 'assembly force 8', 'assembly force 9', 'assembly force 10']
llist = []
files = fio.get_files(cwd, extension=['csv'])
dirs = fio.get_parts_of_paths_list(files, -2)
for file, dirr in zip(files, dirs):
data = fcsv.read_data_from_csv_file(file, [0,2], 7, None, 0)
data = fdp.nested_list_to_numpy_arr(data)
fig = plt.figure(figsize=(13,7))
plt.plot(data[1], data[0])
plt.title(dirr)
points = np.arange(0,11)*34
for index, i in enumerate(points):
if index == 0:
points[index] = 0