# --------TRAINING LOOP--------# for ep in range(N_ep): tr_loss_tot = 0 tr_error_tot = 0 tr_time_tot = 0 print( '------------------------------ Epoch %s / %s ------------------------------' % (format(ep, N_ep_str_format), format(N_ep - 1, N_ep_str_format))) for tr_data in tr_data_lst: # Compute the total number of chunks for each training epoch N_ck_tr = compute_n_chunks(out_folder, tr_data, ep, N_ep_str_format, 'train') N_ck_str_format = '0' + str(max(math.ceil(np.log10(N_ck_tr)), 1)) + 'd' # ***Epoch training*** for ck in range(N_ck_tr): # paths of the output files (info,model,chunk_specific cfg file) info_file = out_folder + '/exp_files/train_' + tr_data + '_ep' + format( ep, N_ep_str_format) + '_ck' + format( ck, N_ck_str_format) + '.info' if ep + ck == 0: model_files_past = {} else: model_files_past = model_files
for ep in range(N_ep): tr_loss_tot = 0 tr_error_tot = 0 tr_time_tot = 0 val_time_tot = 0 print( "------------------------------ Epoch %s / %s ------------------------------" % (format(ep, N_ep_str_format), format(N_ep - 1, N_ep_str_format)) ) for tr_data in tr_data_lst: # Compute the total number of chunks for each training epoch N_ck_tr = compute_n_chunks(out_folder, tr_data, ep, N_ep_str_format, "train") N_ck_str_format = "0" + str(max(math.ceil(np.log10(N_ck_tr)), 1)) + "d" # ***Epoch training*** for ck in range(N_ck_tr): # paths of the output files (info,model,chunk_specific cfg file) info_file = ( out_folder + "/exp_files/train_" + tr_data + "_ep" + format(ep, N_ep_str_format) + "_ck" + format(ck, N_ck_str_format) + ".info"
# A variable that tells if the current chunk is the first one that is being processed: processed_first = True data_name = [] data_set = [] data_end_index = [] fea_dict = [] lab_dict = [] arch_dict = [] # --------FORWARD--------# ep = N_ep - 1 # KMC modification for forward_data in forward_data_lst: # Compute the number of chunks N_ck_forward = compute_n_chunks(out_folder, forward_data, ep, N_ep_str_format, "forward") N_ck_str_format = "0" + str(max(math.ceil(np.log10(N_ck_forward)), 1)) + "d" processes = list() info_files = list() for ck in range(N_ck_forward): if not is_production: print("Testing %s chunk = %i / %i" % (forward_data, ck + 1, N_ck_forward)) else: print("Forwarding %s chunk = %i / %i" % (forward_data, ck + 1, N_ck_forward)) # output file