def test_block_resnet_v2_bottleneck(system_dict): forward = True; test = "test_block_resnet_v2_bottleneck"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.resnet_v2_bottleneck_block(output_channels=32, stride=1, downsample=True)); network.append(gtf.resnet_v2_bottleneck_block(output_channels=32, stride=1, downsample=False)); gtf.Compile_Network(network, data_shape=(1, 64, 64), use_gpu=False); x = torch.randn(1, 1, 64, 64); y = gtf.system_dict["local"]["model"](x); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_block_inception_e(system_dict): forward = True test = "test_block_inception_e" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: gtf = prototype(verbose=0) gtf.Prototype("sample-project-1", "sample-experiment-1") network = [] network.append(gtf.inception_e_block(pool_type="avg")) network.append(gtf.inception_e_block(pool_type="max")) gtf.Compile_Network(network, data_shape=(1, 64, 64), use_gpu=False) x = torch.randn(1, 1, 64, 64) y = gtf.system_dict["local"]["model"](x) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_block_squeezenet_fire(system_dict): forward = True; test = "test_block_squeezenet_fire"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.squeezenet_fire_block(squeeze_channels=16, expand_channels_1x1=32, expand_channels_3x3=64)); gtf.Compile_Network(network, data_shape=(1, 64, 64), use_gpu=False); x = torch.randn(1, 1, 64, 64); y = gtf.system_dict["local"]["model"](x); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_loss_multimargin(system_dict): forward = True test = "test_loss_multimargin" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: gtf = prototype(verbose=0) gtf.Prototype("sample-project-1", "sample-experiment-1") label = torch.empty(1, dtype=torch.long).random_(5) y = torch.randn(1, 5) gtf.loss_multimargin() load_loss(gtf.system_dict) loss_obj = gtf.system_dict["local"]["criterion"] loss_val = loss_obj(y, label) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_loss_multilabelmargin(system_dict): forward = True; test = "test_loss_multilabelmargin"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); label = torch.tensor([[3, 0, -1, 1, 2]]) y = torch.randn(1, 5); gtf.loss_multilabel_margin(); load_loss(gtf.system_dict); loss_obj = gtf.system_dict["local"]["criterion"]; loss_val = loss_obj(y, label); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_layer_convolution1d(system_dict): forward = True test = "test_layer_convolution1d" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: gtf = prototype(verbose=0) gtf.Prototype("sample-project-1", "sample-experiment-1") network = [] network.append(gtf.convolution1d(output_channels=3, kernel_size=3)) gtf.Compile_Network(network, data_shape=(3, 128), use_gpu=False) x = torch.randn(1, 3, 128) y = gtf.system_dict["local"]["model"](x) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_loss_binary_crossentropy(system_dict): forward = True; test = "test_loss_binary_crossentropy"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); label = torch.empty((1, 5)).random_(2); y = torch.randn(1, 5); m = torch.nn.Sigmoid(); y = m(y); gtf.loss_binary_crossentropy(); load_loss(gtf.system_dict); loss_obj = gtf.system_dict["local"]["criterion"]; loss_val = loss_obj(y, label); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_block_densenet(system_dict): forward = True; test = "test_block_densenet"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.densenet_block(bottleneck_size=4, growth_rate=16, dropout=0.2)); gtf.Compile_Network(network, data_shape=(1, 64, 64), use_gpu=False); x = torch.randn(1, 1, 64, 64); y = gtf.system_dict["local"]["model"](x); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_optimizer_adagrad(system_dict): forward = True; test = "test_optimizer_adagrad"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); gtf.Default(dataset_path="../../system_check_tests/datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2); gtf.optimizer_adagrad(0.01, weight_decay=0.0001, learning_rate_decay=0.001); gtf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_block_mobilenet_v2_inverted_linear_bottleneck(system_dict): forward = True test = "test_block_mobilenet_v2_inverted_linear_bottleneck" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: gtf = prototype(verbose=0) gtf.Prototype("sample-project-1", "sample-experiment-1") network = [] network.append( gtf.mobilenet_v2_inverted_linear_bottleneck_block( output_channels=64, bottleneck_width=4, stride=1)) gtf.Compile_Network(network, data_shape=(64, 64, 64), use_gpu=False) x = torch.randn(1, 64, 64, 64) y = gtf.system_dict["local"]["model"](x) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_activation_logsoftmax(system_dict): forward = True; test = "test_activation_logsoftmax"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.logsoftmax()); gtf.Compile_Network(network, data_shape=(3, 64, 64), use_gpu=False); x = torch.randn(1, 3, 64, 64); y = gtf.system_dict["local"]["model"](x); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_optimizer_adam(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "test_optimizer_adam"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); gtf.Default(dataset_path="../../system_check_tests/datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2); gtf.optimizer_adam(0.01, weight_decay=0.0001, beta1=0.9, beta2=0.999, amsgrad=True); gtf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_optimizer_adam(system_dict): forward = True test = "test_optimizer_adam" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: gtf = prototype(verbose=0) gtf.Prototype("sample-project-1", "sample-experiment-1") gtf.Default( dataset_path= "../../system_check_tests/datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2) gtf.optimizer_adam(0.01, weight_decay=0.0001, beta1=0.9, beta2=0.999, amsgrad=True) gtf.Train() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_update_normal(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "update_normal_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf = prototype(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_Prototype()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Prototype("sample-project-1", "sample-experiment-3"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_Default()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Default(dataset_path="datasets/dataset_csv_id/train", path_to_csv="datasets/dataset_csv_id/train.csv", delimiter=",", model_name="resnet18", freeze_base_network=True, num_epochs=10); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_model_name()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_model_name("resnet50"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_use_gpu()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_use_gpu(False); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_use_pretrained()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_use_pretrained(True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_freeze_base_network()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_freeze_base_network(False); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_freeze_layers()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_freeze_layers(10); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_num_epochs()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_num_epochs(2); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_display_progress_realtime()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_display_progress_realtime(False); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_display_progress()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_display_progress(False); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_save_intermediate_models()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_save_intermediate_models(False); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_update_save_training_logs()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.update_save_training_logs(True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_lr_fixed()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.lr_fixed(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_Reload()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Reload(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_EDA()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.EDA(check_missing=True, check_corrupt=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_Estimate_Train_Time()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Estimate_Train_Time(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "update_normal_Train()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_expert_eval_infer(system_dict): forward = True if (not os.path.isdir("datasets")): os.system( "! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt" ) os.system("! unzip -qq datasets.zip") test = "expert_eval_infer_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-4", eval_infer=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params(dataset_path="datasets/dataset_csv_id/test", path_to_csv="datasets/dataset_csv_id/test.csv", delimiter=",") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_reset_transforms()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.reset_transforms(test=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Evaluate()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: accuracy, class_based_accuracy = ptf.Evaluate() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_update_copy_from(system_dict): forward = True test = "update_copy_from_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype( "sample-project-1", "sample-experiment-2", copy_from=["sample-project-1", "sample-experiment-1"]) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_reset_transforms()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.reset_transforms() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_apply_transforms()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.apply_random_resized_crop(256, train=True, val=True, test=True) ptf.apply_random_perspective(train=True, val=True) ptf.apply_random_vertical_flip(train=True, val=True) ptf.apply_random_horizontal_flip(train=True, val=True) ptf.apply_normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], train=True, val=True, test=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_dataset(dataset_path=[ "../datasets/dataset_cats_dogs_train", "../datasets/dataset_cats_dogs_eval" ]) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_input_size()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_input_size(256) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_batch_size()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_batch_size(6) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_shuffle_data()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_shuffle_data(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_num_processors()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_num_processors(16) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_update_trainval_split()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_trainval_split(0.6) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_Reload()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Reload() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_EDA()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.EDA(check_missing=True, check_corrupt=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_copy_from_Train()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Train() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_update_normal(system_dict): forward = True test = "update_normal_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-3") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Default()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Default(dataset_path="../datasets/dataset_csv_id/train", path_to_csv="../datasets/dataset_csv_id/train.csv", delimiter=",", model_name="resnet18", freeze_base_network=True, num_epochs=10) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_model_name()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_model_name("resnet50") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_use_gpu()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_use_gpu(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_use_pretrained()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_use_pretrained(True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_freeze_base_network()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_freeze_base_network(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_freeze_layers()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_freeze_layers(10) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_num_epochs()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_num_epochs(2) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_display_progress_realtime()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_display_progress_realtime(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_display_progress()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_display_progress(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_save_intermediate_models()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_save_intermediate_models(False) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_save_training_logs()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_save_training_logs(True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_lr_fixed()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.lr_fixed() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Reload()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Reload() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_EDA()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.EDA(check_missing=True, check_corrupt=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Estimate_Train_Time()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Estimate_Train_Time() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Train()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Train() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_switch_default(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "switch_default_object_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf = prototype(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_object_Prototype()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Prototype("sample-project-1", "sample-experiment-5"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Default()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Default(dataset_path=["datasets/dataset_cats_dogs_train", "datasets/dataset_cats_dogs_eval"], model_name="resnet18", freeze_base_network=True, num_epochs=10); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_EDA()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.EDA(check_missing=True, check_corrupt=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Switch_Mode()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Switch_Mode(eval_infer=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Dataset_Params()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset_Params(dataset_path="datasets/dataset_cats_dogs_eval"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Dataset()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Evaluate()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: accuracy, class_based_accuracy = ptf.Evaluate(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Switch_Mode()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Switch_Mode(train=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Train()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
import sys sys.path.append("../../../../monk_v1/"); sys.path.append("../../../monk/"); import psutil from pytorch_prototype import prototype from compare_prototype import compare from common import print_start from common import print_status import torch import numpy as np from pytorch.losses.return_loss import load_loss gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.transposed_convolution3d(output_channels=3, kernel_size=3)); gtf.Compile_Network(network, data_shape=(3, 3, 64, 64), use_gpu=False); x = torch.randn(1, 3, 3, 64, 64); y = gtf.system_dict["local"]["model"](x); def test_layer_transposed_convolution3d(system_dict): forward = True; test = "test_layer_transposed_convolution3d";
def test_expert_train(system_dict): forward = True test = "expert_train_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-4") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params(dataset_path=[ "../datasets/dataset_csv_id/train", "../datasets/dataset_csv_id/val" ], path_to_csv=[ "../datasets/dataset_csv_id/train.csv", "../datasets/dataset_csv_id/val.csv" ], split=0.9, input_size=224, batch_size=16, shuffle_data=True, num_processors=3) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Model_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model_Params(model_name="resnet18", freeze_base_network=True, use_gpu=True, use_pretrained=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_append_layer()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.append_dropout(probability=0.1) ptf.append_linear(final_layer=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Model()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_lr_step_decrease()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.lr_step_decrease(1, gamma=0.9) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_optimizer_sgd()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.optimizer_sgd(0.001, momentum=0.9) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_loss_softmax_crossentropy()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.loss_softmax_crossentropy() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Training_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Training_Params( num_epochs=3, display_progress=True, display_progress_realtime=True, save_intermediate_models=True, intermediate_model_prefix="intermediate_model_", save_training_logs=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Train()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Train() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_expert_eval_infer(system_dict): forward = True test = "expert_eval_infer_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-4", eval_infer=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params( dataset_path="../datasets/dataset_csv_id/test", path_to_csv="../datasets/dataset_csv_id/test.csv", delimiter=",") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_reset_transforms()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.reset_transforms(test=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_eval_infer_Evaluate()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: accuracy, class_based_accuracy = ptf.Evaluate() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_switch_expert(system_dict): forward = True if (not os.path.isdir("datasets")): os.system( "! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt" ) os.system("! unzip -qq datasets.zip") test = "switch_expert_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-6") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_switch_mode()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Switch_Mode(eval_infer=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Model_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model_Params( model_path= "workspace/sample-project-1/sample-experiment-5/output/models/intermediate_model_9", use_gpu=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Model()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_update_input_size()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_input_size(224) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Infer-Img()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: img_name = "datasets/dataset_cats_dogs_test/0.jpg" predictions = ptf.Infer(img_name=img_name, return_raw=True) img_name = "datasets/dataset_cats_dogs_test/84.jpg" predictions = ptf.Infer(img_name=img_name, return_raw=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Infer-Folder()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: inference_dataset = "datasets/dataset_cats_dogs_test/" output = ptf.Infer(img_dir=inference_dataset, return_raw=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params(dataset_path="datasets/dataset_cats_dogs_eval", input_size=224) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_Evaluate()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: accuracy, class_based_accuracy = ptf.Evaluate() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "switch_expert_switch_mode()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Switch_Mode(train=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params(dataset_path="datasets/dataset_cats_dogs_train", split=0.9, input_size=224, batch_size=16, shuffle_data=True, num_processors=3) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_apply_transforms()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.apply_random_resized_crop(256, train=True, val=True, test=True) ptf.apply_random_perspective(train=True, val=True) ptf.apply_random_vertical_flip(train=True, val=True) ptf.apply_random_horizontal_flip(train=True, val=True) ptf.apply_normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225], train=True, val=True, test=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Model_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model_Params(model_name="resnet18", freeze_base_network=True, use_gpu=True, use_pretrained=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Model()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Model() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_lr_multistep_decrease()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.lr_multistep_decrease([1, 3], gamma=0.9) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_optimizer_sgd()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.optimizer_sgd(0.001, momentum=0.9) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_loss_softmax_crossentropy()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.loss_softmax_crossentropy() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Training_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Training_Params( num_epochs=4, display_progress=True, display_progress_realtime=True, save_intermediate_models=True, intermediate_model_prefix="intermediate_model_", save_training_logs=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "expert_train_Train()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Train() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_switch_default(system_dict): forward = True; test = "switch_default_object_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf = prototype(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_object_Prototype()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Prototype("sample-project-1", "sample-experiment-5"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Default()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Default(dataset_path=["../datasets/dataset_cats_dogs_train", "../datasets/dataset_cats_dogs_eval"], model_name="resnet18", freeze_base_network=True, num_epochs=10); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_EDA()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.EDA(check_missing=True, check_corrupt=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Switch_Mode()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Switch_Mode(eval_infer=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Dataset_Params()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset_Params(dataset_path="../datasets/dataset_cats_dogs_eval"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Dataset()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Evaluate()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: accuracy, class_based_accuracy = ptf.Evaluate(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Switch_Mode()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Switch_Mode(train=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "switch_default_Train()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_expert_train(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "expert_train_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf = prototype(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Prototype()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Prototype("sample-project-1", "sample-experiment-4"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Dataset_Params()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset_Params(dataset_path=["datasets/dataset_csv_id/train", "datasets/dataset_csv_id/val"], path_to_csv=["datasets/dataset_csv_id/train.csv", "datasets/dataset_csv_id/val.csv"], split=0.9, input_size=224, batch_size=16, shuffle_data=True, num_processors=3); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Dataset()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Dataset(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Model_Params()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Model_Params(model_name="resnet18", freeze_base_network=True, use_gpu=True, use_pretrained=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_append_layer()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.append_dropout(probability=0.1); ptf.append_linear(final_layer=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Model()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Model(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_lr_step_decrease()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.lr_step_decrease(1, gamma=0.9); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_optimizer_sgd()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.optimizer_sgd(0.001, momentum=0.9); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_loss_softmax_crossentropy()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.loss_softmax_crossentropy(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Training_Params()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Training_Params(num_epochs=3, display_progress=True, display_progress_realtime=True, save_intermediate_models=True, intermediate_model_prefix="intermediate_model_", save_training_logs=True); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "expert_train_Train()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Train(); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def Analyse_Optimizers(self, analysis_name, optimizer_list, percent_data, num_epochs=2, state="keep_all"): from pytorch_prototype import prototype project = analysis_name; self.custom_print(""); self.custom_print("Running Optimizer analysis"); #Change 1 self.custom_print("Analysis Name : {}".format(project)); self.custom_print(""); for i in range(len(optimizer_list)): #Change 2 ptf_ = prototype(verbose=0); self.custom_print("Running experiment : {}/{}".format(i+1, len(optimizer_list))); #Change 3 experiment = "Optimizer_" + str(optimizer_list[i]); #Change 4, 5 self.custom_print("Experiment name : {}".format(experiment)) ptf_.Prototype(project, experiment, pseudo_copy_from=[self.system_dict["project_name"], self.system_dict["experiment_name"]]); ptf_.Dataset_Percent(percent_data); dataset_type = ptf_.system_dict["dataset"]["dataset_type"]; dataset_train_path = ptf_.system_dict["dataset"]["train_path"]; dataset_val_path = ptf_.system_dict["dataset"]["val_path"]; csv_train = ptf_.system_dict["dataset"]["csv_train"]; csv_val = ptf_.system_dict["dataset"]["csv_val"]; if(dataset_type=="train"): ptf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="train-val"): ptf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); elif(dataset_type=="csv_train"): ptf_.update_dataset(dataset_path=dataset_train_path, path_to_csv="sampled_dataset_train.csv"); elif(dataset_type=="csv_train-val"): ptf_.update_dataset(dataset_path=[dataset_train_path, dataset_val_path], path_to_csv=["sampled_dataset_train.csv", "sampled_dataset_val.csv"]); lr = ptf_.system_dict["hyper-parameters"]["learning_rate"] if(optimizer_list[i] == "adagrad"): #Change 6 ptf_.optimizer_adagrad(lr); elif(optimizer_list[i] == "adadelta"): ptf_.optimizer_adadelta(lr); elif(optimizer_list[i] == "adam"): ptf_.optimizer_adam(lr); elif(optimizer_list[i] == "adamw"): ptf_.optimizer_adamw(lr); elif(optimizer_list[i] == "sparseadam"): ptf_.optimizer_sparseadam(lr); elif(optimizer_list[i] == "adamax"): ptf_.optimizer_adamax(lr); elif(optimizer_list[i] == "asgd"): ptf_.optimizer_asgd(lr); elif(optimizer_list[i] == "rmsprop"): ptf_.optimizer_rmsprop(lr); elif(optimizer_list[i] == "rprop"): ptf_.optimizer_rprop(lr); elif(optimizer_list[i] == "sgd"): ptf_.optimizer_sgd(lr); ptf_.Reload(); #Change 7 ptf_.update_num_epochs(num_epochs); ptf_.update_display_progress_realtime(False) ptf_.update_save_intermediate_models(False); total_time_per_epoch = ptf_.get_training_estimate(); total_time = total_time_per_epoch*num_epochs; if(int(total_time//60) == 0): self.custom_print("Estimated time : {} sec".format(total_time)); else: self.custom_print("Estimated time : {} min".format(int(total_time//60)+1)); ptf_.Train(); self.custom_print("Experiment Complete"); self.custom_print("\n"); self.custom_print("Comparing Experiments"); from compare_prototype import compare ctf_ = compare(verbose=0); ctf_.Comparison("Comparison_" + analysis_name); self.custom_print("Comparison ID: {}".format("Comparison_" + analysis_name)); training_accuracies = []; validation_accuracies = []; training_losses = []; validation_losses = []; tabular_data = []; for i in range(len(optimizer_list)): #Change 8 project = analysis_name; experiment = "Optimizer_" + str(optimizer_list[i]); #Change 9, 10 ctf_.Add_Experiment(project, experiment) tmp = []; tmp.append(experiment); training_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_acc_history.npy"; tmp.append(np.load(training_accuracy_file, allow_pickle=True)[-1]); validation_accuracy_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_acc_history.npy"; tmp.append(np.load(validation_accuracy_file, allow_pickle=True)[-1]); training_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/train_loss_history.npy"; tmp.append(np.load(training_loss_file, allow_pickle=True)[-1]); validation_loss_file = self.system_dict["master_systems_dir_relative"] + "/" + project + "/" + experiment + "/output/logs/val_loss_history.npy"; tmp.append(np.load(validation_loss_file, allow_pickle=True)[-1]); tabular_data.append(tmp) ctf_.Generate_Statistics(); self.custom_print("Generated statistics post all epochs"); self.custom_print(tabulate(tabular_data, headers=['Experiment Name', 'Train Acc', 'Val Acc', 'Train Loss', 'Val Loss'], tablefmt='orgtbl')); self.custom_print(""); return_dict = {}; for i in range(len(tabular_data)): return_dict[tabular_data[i][0]] = {}; return_dict[tabular_data[i][0]]["training_accuracy"] = tabular_data[i][1]; return_dict[tabular_data[i][0]]["validation_accuracy"] = tabular_data[i][2]; return_dict[tabular_data[i][0]]["training_loss"] = tabular_data[i][3]; return_dict[tabular_data[i][0]]["validation_loss"] = tabular_data[i][4]; fname = self.system_dict["master_systems_dir_relative"] + analysis_name + "/" + tabular_data[i][0] + "/experiment_state.json"; system_dict = read_json(fname); return_dict[tabular_data[i][0]]["training_time"] = system_dict["training"]["outputs"]["training_time"]; if(state=="keep_none"): shutil.rmtree(self.system_dict["master_systems_dir_relative"] + analysis_name); return return_dict ###############################################################################################################################################
import os import sys sys.path.append("../../../monk/") import psutil from pytorch_prototype import prototype ptf = prototype(verbose=1) ptf.Prototype("sample-project-1", "sample-experiment-1") ptf.Default( dataset_path= "../../../monk/system_check_tests/datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2) ################################################### Dataset Updates ###################################################################### ptf.update_input_size(256) ptf.update_batch_size(6) ptf.update_shuffle_data(True) ptf.update_num_processors(psutil.cpu_count()) ptf.update_trainval_split(0.6) ptf.update_dataset(dataset_path=[ "../../../monk/system_check_tests/datasets/dataset_cats_dogs_train", "../../../monk/system_check_tests/datasets/dataset_cats_dogs_eval" ]) ################################################################################################################################################# ################################################### Transforms Updates ###################################################################### # Reset Transforms if required
def test_analyse(system_dict): forward = True test = "analyse_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-1") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Default()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Default(dataset_path="../datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Analyse_Learning_Rates()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: analysis_name = "analyse_learning_rates" lrs = [0.1, 0.05] epochs = 2 percent_data = 40 analysis = ptf.Analyse_Learning_Rates(analysis_name, lrs, percent_data, num_epochs=epochs, state="keep_none") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Analyse_Input_Sizes()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: analysis_name = "analyse_input_sizes" input_sizes = [128, 256] epochs = 2 percent_data = 40 analysis = ptf.Analyse_Input_Sizes(analysis_name, input_sizes, percent_data, num_epochs=epochs, state="keep_none") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Analyse_Batch_Sizes()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: analysis_name = "analyse_batch_sizes" batch_sizes = [2, 3] epochs = 2 percent_data = 40 analysis = ptf.Analyse_Batch_Sizes(analysis_name, batch_sizes, percent_data, num_epochs=epochs, state="keep_none") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Analyse_Models()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: analysis_name = "analyse_models" models = [["resnet18", True, True], ["resnet34", False, True]] percent_data = 40 analysis = ptf.Analyse_Models(analysis_name, models, percent_data, num_epochs=epochs, state="keep_none") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "analyse_Analyse_Optimizers()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: analysis_name = "analyse_optimizers" optimizers = ["sgd", "adam"] epochs = 2 percent_data = 40 analysis = ptf.Analyse_Optimizers(analysis_name, optimizers, percent_data, num_epochs=epochs, state="keep_none") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict
def test_analyse(system_dict): forward = True; if(not os.path.isdir("datasets")): os.system("! wget --load-cookies /tmp/cookies.txt \"https://docs.google.com/uc?export=download&confirm=$(wget --save-cookies /tmp/cookies.txt --keep-session-cookies --no-check-certificate 'https://docs.google.com/uc?export=download&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2' -O- | sed -rn 's/.*confirm=([0-9A-Za-z_]+).*/\1\n/p')&id=1rG-U1mS8hDU7_wM56a1kc-li_zHLtbq2\" -O datasets.zip && rm -rf /tmp/cookies.txt") os.system("! unzip -qq datasets.zip") test = "analyse_object_creation"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf = prototype(verbose=0); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Prototype()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Prototype("sample-project-1", "sample-experiment-1"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Default()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: ptf.Default(dataset_path="datasets/dataset_cats_dogs_train", model_name="resnet18", freeze_base_network=True, num_epochs=2); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Analyse_Learning_Rates()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: analysis_name = "analyse_learning_rates" lrs = [0.1, 0.05]; epochs=2 percent_data=40 analysis = ptf.Analyse_Learning_Rates(analysis_name, lrs, percent_data, num_epochs=epochs, state="keep_none"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Analyse_Input_Sizes()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: analysis_name = "analyse_input_sizes"; input_sizes = [128, 256]; epochs=2; percent_data=40; analysis = ptf.Analyse_Input_Sizes(analysis_name, input_sizes, percent_data, num_epochs=epochs, state="keep_none"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Analyse_Batch_Sizes()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: analysis_name = "analyse_batch_sizes"; batch_sizes = [2, 3]; epochs = 2; percent_data = 40; analysis = ptf.Analyse_Batch_Sizes(analysis_name, batch_sizes, percent_data, num_epochs=epochs, state="keep_none"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Analyse_Models()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: analysis_name = "analyse_models"; models = [["resnet18", True, True], ["resnet34", False, True]]; percent_data=40; analysis = ptf.Analyse_Models(analysis_name, models, percent_data, num_epochs=epochs, state="keep_none"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); test = "analyse_Analyse_Optimizers()"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: analysis_name = "analyse_optimizers"; optimizers = ["sgd", "adam"]; epochs = 2; percent_data = 40; analysis = ptf.Analyse_Optimizers(analysis_name, optimizers, percent_data, num_epochs=epochs, state="keep_none"); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_layer_concatenate(system_dict): forward = True; test = "test_layer_concatenate"; system_dict["total_tests"] += 1; print_start(test, system_dict["total_tests"]) if(forward): try: gtf = prototype(verbose=0); gtf.Prototype("sample-project-1", "sample-experiment-1"); network = []; network.append(gtf.convolution(output_channels=16)); network.append(gtf.batch_normalization()); network.append(gtf.relu()); network.append(gtf.max_pooling()); subnetwork = []; branch1 = []; branch1.append(gtf.convolution(output_channels=16)); branch1.append(gtf.batch_normalization()); branch1.append(gtf.convolution(output_channels=16)); branch1.append(gtf.batch_normalization()); branch2 = []; branch2.append(gtf.convolution(output_channels=16)); branch2.append(gtf.batch_normalization()); branch3 = []; branch3.append(gtf.identity()) subnetwork.append(branch1); subnetwork.append(branch2); subnetwork.append(branch3); subnetwork.append(gtf.concatenate()) network.append(subnetwork); network.append(gtf.convolution(output_channels=16)); network.append(gtf.batch_normalization()); network.append(gtf.relu()); network.append(gtf.max_pooling()); network.append(gtf.flatten()); network.append(gtf.fully_connected(units=1024)); network.append(gtf.dropout(drop_probability=0.2)); network.append(gtf.fully_connected(units=2)); gtf.Compile_Network(network, data_shape=(3, 64, 64), use_gpu=False); x = torch.randn(1, 3, 64, 64); y = gtf.system_dict["local"]["model"](x); system_dict["successful_tests"] += 1; print_status("Pass"); except Exception as e: system_dict["failed_tests_exceptions"].append(e); system_dict["failed_tests_lists"].append(test); forward = False; print_status("Fail"); else: system_dict["skipped_tests_lists"].append(test); print_status("Skipped"); return system_dict
def test_update_eval_infer(system_dict): forward = True test = "update_normal_object_creation" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf = prototype(verbose=0) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Prototype()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Prototype("sample-project-1", "sample-experiment-3", eval_infer=True) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Dataset_Params()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset_Params( dataset_path="../datasets/dataset_csv_id/test", path_to_csv="../datasets/dataset_csv_id/test.csv", delimiter=",") system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "default_eval_infer_Dataset()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Dataset() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_update_model_path()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.update_model_path( "workspace/sample-project-1/sample-experiment-3/output/models/best_model" ) system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "update_normal_Reload()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: ptf.Reload() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") test = "default_eval_infer_Evaluate()" system_dict["total_tests"] += 1 print_start(test, system_dict["total_tests"]) if (forward): try: accuracy, class_based_accuracy = ptf.Evaluate() system_dict["successful_tests"] += 1 print_status("Pass") except Exception as e: system_dict["failed_tests_exceptions"].append(e) system_dict["failed_tests_lists"].append(test) forward = False print_status("Fail") else: system_dict["skipped_tests_lists"].append(test) print_status("Skipped") return system_dict