def read_one_line(in_file):
line = in_file.readline()
gl.N_READ += 1
u.step_log(gl.N_READ, gl.SL_STEP_READ, what='lines processed')
return line
def process_line(line):
gl.n_r += 1
line_list = u.csv_to_list(line)
if filter_line(line_list):
line_list = extract_col(line_list)
gl.out_list.append(line_list)
gl.n_o += 1
u.step_log(gl.n_r, gl.SL_STEP, what=gl.s, nb=gl.n_o)
def compare_inf(line_1_list, line_2_list, in_file_1, out_file):
while compare_elt(line_1_list, line_2_list) == "<":
gl.c_diff += 1
if gl.DIFF:
line_1_list.append(gl.LABEL_1)
u.write_csv_line(line_1_list, out_file)
gl.c_out += 1
line_1_list = read_list(in_file_1)
gl.c_1 += 1
u.step_log(gl.c_1, gl.SL_STEP, gl.msg, gl.c_out)
return (line_1_list, line_2_list)
def write_min_elt(min_elt, out_file):
cur_key = min_elt[gl.PIVOT_IDX]
prev_key = gl.prev_elt[gl.PIVOT_IDX]
if cur_key != prev_key:
gl.DUP_KEY = False
u.write_csv_line(min_elt, out_file)
gl.c_tot_out += 1
u.step_log(gl.c_tot_out, gl.SL_STEP)
gl.prev_elt = min_elt
elif check_dup(min_elt):
# Pure duplicates are not written in output file
# But key duplicates are (lines differ but key equal)
u.write_csv_line(min_elt, out_file)
gl.c_tot_out += 1
u.step_log(gl.c_tot_out, gl.SL_STEP)
def save_img_dict(out_path):
u.log('Saving parse dictionary as csv...')
header = []
for elt in gl.parse_dict:
header.append(elt)
with open(out_path, 'w', encoding='utf-8') as out_file:
u.write_csv_line(header, out_file)
u.init_sl_time()
gl.N_WRITE = 0
while gl.N_WRITE < gl.N_ROW:
cur_row = []
for elt in gl.parse_dict:
cur_row.append(gl.parse_dict[elt][gl.N_WRITE])
u.write_csv_line(cur_row, out_file)
gl.N_WRITE += 1
u.step_log(gl.N_WRITE, gl.SL_STEP_WRITE, what='lines written')
u.log(f"csv file saved in {out_path}")
def gen_sorted_temp_files(in_path, out_path):
# Generation of sorted temporary files
has_header = u.has_header(in_path)
u.log("Generating first list to be sorted...")
u.init_sl_time()
with open(in_path, 'r', encoding='utf-8') as in_file:
first_line = in_file.readline()
if not has_header:
gl.cur_list.append(u.csv_to_list(first_line))
gl.c_sf_read = 1
for line in in_file:
gl.c_sf_read += 1
gl.cur_list.append(u.csv_to_list(line))
s = "lines read"
u.step_log(gl.c_sf_read, gl.SL_STEP, s)
check_max_row(gl.c_sf_read)
gen_last_file(out_path)
del gl.cur_list
def compare_equal(line_1_list, line_2_list, in_file_1, in_file_2, out_file):
while compare_elt(line_1_list, line_2_list) == "=":
if line_1_list != line_2_list:
gl.c_diff += 1
if gl.DIFF:
line_diff = compare_line(line_1_list, line_2_list)
u.write_csv_line(line_diff, out_file)
gl.c_out += 1
elif gl.EQUAL:
line_1_list.append(gl.EQUAL_LABEL)
u.write_csv_line(line_1_list, out_file)
gl.c_out += 1
line_1_list = read_list(in_file_1)
gl.c_1 += 1
line_2_list = read_list(in_file_2)
gl.c_2 += 1
u.step_log(gl.c_1, gl.SL_STEP, gl.msg, gl.c_out)
return (line_1_list, line_2_list)
# This script shows you simple examples of use for the log and step_log functions
import time
import partools.utils as u
u.log("This won't be logged in a file")
u.init_log('test')
u.log("This will be logged in a file")
out_list = []
u.init_sl_time()
for i in range(1, 21):
time.sleep(0.05) # simulates io / calculation
out_list.append(i)
u.step_log(i, 5, "elements appended")
u.log_print(f'out_list: {out_list}')