def test_compare(system_dict): forward = True test = "compare_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ctf = compare(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "compare_Comparison()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ctf.Comparison("Sample-Comparison-1") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "compare_Add_Experiment()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ctf.Add_Experiment("sample-project-1", "sample-experiment-1") ctf.Add_Experiment("sample-project-1", "sample-experiment-2") ctf.Add_Experiment("sample-project-1", "sample-experiment-3") ctf.Add_Experiment("sample-project-1", "sample-experiment-4") ctf.Add_Experiment("sample-project-1", "sample-experiment-5") ctf.Add_Experiment("sample-project-1", "sample-experiment-6") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "compare_Generate_Statistics()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ctf.Generate_Statistics() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_compare(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "compare_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ctf = compare(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "compare_Comparison()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ctf.Comparison("Sample-Comparison-1"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "compare_Add_Experiment()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ctf.Add_Experiment("sample-project-1", "sample-experiment-1"); ctf.Add_Experiment("sample-project-1", "sample-experiment-2"); ctf.Add_Experiment("sample-project-1", "sample-experiment-3"); ctf.Add_Experiment("sample-project-1", "sample-experiment-4"); ctf.Add_Experiment("sample-project-1", "sample-experiment-5"); ctf.Add_Experiment("sample-project-1", "sample-experiment-6"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "compare_Generate_Statistics()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ctf.Generate_Statistics(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict;
def Analyse_Optimizers(self, analysis_name, optimizer_list, percent_data, num_epochs=2, state="keep_all"): from pytorch_prototype import prototype project = analysis_name; self.custom_print(""); self.custom_print("Running Optimizer analysis"); #Change 1 self.custom_print("Analysis Name : {}".format(project)); self.custom_print(""); for i in range(len(optimizer_list)): #Change 2 ptf_ = prototype(verbose=0); self.custom_print("Running experiment : {}/{}".format(i+1, len(optimizer_list))); #Change 3 experiment = "Optimizer_" + str(optimizer_list[i]); #Change 4, 5 self.custom_print("Experiment name : {}".format(experiment)) ptf_.Prototype(project, experiment, pseudo_copy_from=[self.system_dict["project_name"], self.system_dict["experiment_name"]]); ptf_.Dataset_Percent(percent_data); dataset_type = ptf_.system_dict["dataset"]["dataset_type"]; dataset_train_path = ptf_.system_dict["dataset"]["train_path"]; dataset_val_path = ptf_.system_dict["dataset"]["val_path"]; csv_train = ptf_.system_dict["dataset"]["csv_train"]; csv_val = ptf_.system_dict["dataset"]["csv_val"]; if(dataset_type=="train"): ptf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="train-val"): ptf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); elif(dataset_type=="csv_train"): ptf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="csv_train-val"): ptf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); lr = ptf_.system_dict["hyper-parameters"]["learning_rate"] if(optimizer_list[i] == "adagrad"): #Change 6 ptf_.optimizer_adagrad(lr); elif(optimizer_list[i] == "adadelta"): ptf_.optimizer_adadelta(lr); elif(optimizer_list[i] == "adam"): ptf_.optimizer_adam(lr); elif(optimizer_list[i] == "adamw"): ptf_.optimizer_adamw(lr); elif(optimizer_list[i] == "sparseadam"): ptf_.optimizer_sparseadam(lr); elif(optimizer_list[i] == "adamax"): ptf_.optimizer_adamax(lr); elif(optimizer_list[i] == "asgd"): ptf_.optimizer_asgd(lr); elif(optimizer_list[i] == "rmsprop"): ptf_.optimizer_rmsprop(lr); elif(optimizer_list[i] == "rprop"): ptf_.optimizer_rprop(lr); elif(optimizer_list[i] == "sgd"): ptf_.optimizer_sgd(lr); ptf_.Reload(); #Change 7 ptf_.update_num_epochs(num_epochs); ptf_.update_display_progress_realtime(False) ptf_.update_save_intermediate_models(False); total_time_per_epoch = ptf_.get_training_estimate(); total_time = total_time_per_epoch*num_epochs; if(int(total_time//60) == 0): self.custom_print("Estimated time : {} sec".format(total_time)); else: self.custom_print("Estimated time : {} min".format(int(total_time//60)+1)); ptf_.Train(); self.custom_print("Experiment Complete"); self.custom_print("\n"); self.custom_print("Comparing Experiments"); from compare_prototype import compare ctf_ = compare(verbose=0); ctf_.Comparison("Comparison_" + analysis_name); self.custom_print("Comparison ID: {}".format("Comparison_" + analysis_name)); training_accuracies = []; validation_accuracies = []; training_losses = []; validation_losses = []; tabular_data = []; for i in range(len(optimizer_list)): #Change 8 project = analysis_name; experiment = "Optimizer_" + str(optimizer_list[i]); #Change 9, 10 ctf_.Add_Experiment(project, experiment) tmp = []; tmp.append(experiment); training_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_acc_history.npy"; tmp.append(np.load(training_accuracy_file, allow_pickle=True)[-1]); validation_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_acc_history.npy"; tmp.append(np.load(validation_accuracy_file, allow_pickle=True)[-1]); training_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_loss_history.npy"; tmp.append(np.load(training_loss_file, allow_pickle=True)[-1]); validation_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_loss_history.npy"; tmp.append(np.load(validation_loss_file, allow_pickle=True)[-1]); tabular_data.append(tmp) ctf_.Generate_Statistics(); self.custom_print("Generated statistics post all epochs"); self.custom_print(tabulate(tabular_data, headers=['Experiment Name', 'Train Acc', 'Val Acc', 'Train Loss', 'Val Loss'], tablefmt='orgtbl')); self.custom_print(""); return_dict = {}; for i in range(len(tabular_data)): return_dict[tabular_data[i][0]] = {}; return_dict[tabular_data[i][0]]["training_accuracy"] = tabular_data[i][1]; return_dict[tabular_data[i][0]]["validation_accuracy"] = tabular_data[i][2]; return_dict[tabular_data[i][0]]["training_loss"] = tabular_data[i][3]; return_dict[tabular_data[i][0]]["validation_loss"] = tabular_data[i][4]; fname = self.system_dict["master_systems_dir_relative"] + analysis_name + "/" + tabular_data[i][0] + "/experiment_state.json"; system_dict = read_json(fname); return_dict[tabular_data[i][0]]["training_time"] = system_dict["training"]["outputs"]["training_time"]; if(state=="keep_none"): shutil.rmtree(self.system_dict["master_systems_dir_relative"] + analysis_name); return return_dict ###############################################################################################################################################
import os import sys sys.path.append("../../../monk/") import psutil from compare_prototype import compare ctf = compare(verbose=1) ctf.Comparison("Sample-Comparison-1") ctf.Add_Experiment("sample-project-1", "sample-experiment-1") ctf.Add_Experiment("sample-project-1", "sample-experiment-2") ctf.Add_Experiment("sample-project-1", "sample-experiment-3") ctf.Add_Experiment("sample-project-1", "sample-experiment-4") ctf.Add_Experiment("sample-project-1", "sample-experiment-5") ctf.Add_Experiment("sample-project-1", "sample-experiment-6") ctf.Generate_Statistics()
def Analyse_Models(self, analysis_name, model_list, percent_data, num_epochs=2, state="keep_all"): from gluon_prototype import prototype project = analysis_name; self.custom_print(""); self.custom_print("Running Model analysis"); #Change 1 self.custom_print("Analysis Name : {}".format(project)); self.custom_print(""); for i in range(len(model_list)): #Change 2 gtf_ = prototype(verbose=0); self.custom_print("Running experiment : {}/{}".format(i+1, len(model_list))); #Change 3 if(model_list[i][1]): experiment = "Model_" + str(model_list[i][0]) + "_freeze_base"; #Change 4, 5 else: experiment = "Model_" + str(model_list[i][0]) + "_unfreeze_base"; if(model_list[i][2]): experiment += "_pretrained"; else: experiment += "_uninitialized"; self.custom_print("Experiment name : {}".format(experiment)) gtf_.Prototype(project, experiment, pseudo_copy_from=[self.system_dict["project_name"], self.system_dict["experiment_name"]]); gtf_.Dataset_Percent(percent_data); dataset_type = gtf_.system_dict["dataset"]["dataset_type"]; dataset_train_path = gtf_.system_dict["dataset"]["train_path"]; dataset_val_path = gtf_.system_dict["dataset"]["val_path"]; csv_train = gtf_.system_dict["dataset"]["csv_train"]; csv_val = gtf_.system_dict["dataset"]["csv_val"]; if(dataset_type=="train"): gtf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="train-val"): gtf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); elif(dataset_type=="csv_train"): gtf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="csv_train-val"): gtf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); gtf_.update_model_name(model_list[i][0]) #Change 6 gtf_.update_freeze_base_network(model_list[i][1]) gtf_.update_use_pretrained(model_list[i][2]) gtf_.Reload(); #Change 7 gtf_.update_num_epochs(num_epochs); gtf_.update_display_progress_realtime(False) gtf_.update_save_intermediate_models(False); total_time_per_epoch = gtf_.get_training_estimate(); total_time = total_time_per_epoch*num_epochs; if(int(total_time//60) == 0): self.custom_print("Estimated time : {} sec".format(total_time)); else: self.custom_print("Estimated time : {} min".format(int(total_time//60)+1)); gtf_.Train(); self.custom_print("Experiment Complete"); self.custom_print("\n"); self.custom_print("Comparing Experiments"); from compare_prototype import compare ctf_ = compare(verbose=0); ctf_.Comparison("Comparison_" + analysis_name); self.custom_print("Comparison ID: {}".format("Comparison_" + analysis_name)); training_accuracies = []; validation_accuracies = []; training_losses = []; validation_losses = []; tabular_data = []; for i in range(len(model_list)): #Change 8 project = analysis_name; if(model_list[i][1]): experiment = "Model_" + str(model_list[i][0]) + "_freeze_base"; #Change 9, 10 else: experiment = "Model_" + str(model_list[i][0]) + "_unfreeze_base"; if(model_list[i][2]): experiment += "_pretrained"; else: experiment += "_uninitialized"; ctf_.Add_Experiment(project, experiment) tmp = []; tmp.append(experiment); training_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_acc_history.npy"; tmp.append(np.load(training_accuracy_file)[-1]); validation_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_acc_history.npy"; tmp.append(np.load(validation_accuracy_file)[-1]); training_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_loss_history.npy"; tmp.append(np.load(training_loss_file)[-1]); validation_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_loss_history.npy"; tmp.append(np.load(validation_loss_file)[-1]); tabular_data.append(tmp) ctf_.Generate_Statistics(); self.custom_print("Generated statistics post all epochs"); self.custom_print(tabulate(tabular_data, headers=['Experiment Name', 'Train Acc', 'Val Acc', 'Train Loss', 'Val Loss'], tablefmt='orgtbl')); self.custom_print(""); return_dict = {}; for i in range(len(tabular_data)): return_dict[tabular_data[i][0]] = {}; return_dict[tabular_data[i][0]]["training_accuracy"] = tabular_data[i][1]; return_dict[tabular_data[i][0]]["validation_accuracy"] = tabular_data[i][2]; return_dict[tabular_data[i][0]]["training_loss"] = tabular_data[i][3]; return_dict[tabular_data[i][0]]["validation_loss"] = tabular_data[i][4]; fname = self.system_dict["master_systems_dir_relative"] + analysis_name + "/" + tabular_data[i][0] + "/experiment_state.json"; system_dict = read_json(fname); return_dict[tabular_data[i][0]]["training_time"] = system_dict["training"]["outputs"]["training_time"]; if(state=="keep_none"): shutil.rmtree(self.system_dict["master_systems_dir_relative"] + analysis_name); return return_dict