def run_lr(X, y, kf=None): tag = "Acc/LR" clf_cv = LogisticRegression(max_iter=100000) get_prediction_kf(kf, clf_cv, X, y, tag) log.info("")
def test_model(model, test_loader, iterator=0): correct = 0 total = len(test_loader.dataset) prediction_list, label_list = [], [] # set the model into evaluation mode model = model.eval() # behavior of the batch norm layer so that it is not sensitive to batch size with torch.no_grad(): # Iterate through test set mini batches for e, (images, labels) in enumerate(tqdm(test_loader)): # Forward pass inputs = images.to(device) labels = labels.to(device) y = model(inputs) predictions = torch.argmax(y, dim=1) prediction_list.extend([p.item() for p in predictions]) label_list.extend([l.item() for l in labels]) truths = torch.sum((predictions == labels).float()).item() correct += truths acc = (correct / total) log.info('\nTest accuracy: {}'.format(acc)) if iterator != 0: writer.add_scalar(MODEL_NAME[0] + "/Acc/Validation", acc, iterator) conf_matrix = confusion_matrix(label_list, prediction_list) log.info("Confusion Matrix:\n_________\n| {} | {} | \n|---|---|\n| {} | {} |\n_________" .format(conf_matrix[0, 0], conf_matrix[0, 1], conf_matrix[1, 0], conf_matrix[1, 1])) return 100 * acc
def run_svm(X, y, kf=None): tag = "Acc/SVM" svc_cv = SVC(probability=True) get_prediction_kf(kf, svc_cv, X, y, tag) log.info("")
def run_knn(X, y, kf=None): tag = "Acc/KNN" neigh_cv = KNeighborsClassifier(n_neighbors=len(set(y))) get_prediction_kf(kf, neigh_cv, X, y, tag) log.info("")
def main(model_name, dataset_folder, seed, cv=5, img_size=112, normalize=True): kf = KFold(n_splits=cv, shuffle=True, random_state=seed) log.info("Constructing datasets and arrays") X, y = get_dataset(dataset_folder, img_size, normalize, divide=False) log.info("Calling the model: " + model_name) run_model(model_name, X, y, kf) collect_garbage()
def validate_model(model, test_loader, metric, iterator, save): correct = 0 total = len(test_loader.dataset) # set the model into evaluation mode model = model.eval() metric = metric.eval() # behavior of the batch norm layer so that it is not sensitive to batch size with torch.no_grad(): # Iterate through test set mini batches for e, (images, labels) in enumerate(tqdm(test_loader)): # Forward pass inputs = images.to(device) labels = labels.to(device) outputs = model(inputs) predictions = torch.argmax(outputs, dim=1) truths = torch.sum((predictions == labels).float()).item() correct += truths loss = metric(outputs, labels).item() val_acc = correct / total writer.add_scalar(MODEL_NAME[0] + "/Loss/Validation", loss, iterator) writer.add_scalar(MODEL_NAME[0] + "/Acc/Validation", val_acc, iterator) log.info("{}th Validation --> Loss: {} - Accuracy: {}".format( iterator, round(loss, 6), round(val_acc, 6))) if save and is_verified(100 * val_acc): exist_files = path_exists(ROOT_DIR, SAVE_FILE[0], "contains") better = len(exist_files) == 0 if not better: exist_acc = [] for file in exist_files: exist_acc.append(float(file.split("_")[0].replace(",", "."))) better = all(100 * val_acc > acc for acc in exist_acc) if better: save_model(model=model, path=str(round(100 * val_acc, 2)) + "_" + SAVE_FILE[0])
def weighted_model(model_name, pretrain_file, use_actual_num_classes=False): out_file = ROOT_DIR + "/" + pretrain_file + ".pth" if model_name == models.alexnet.__name__: model = models.alexnet( num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.resnet18.__name__: model = models.resnet18( num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.resnet50.__name__: model = models.resnet50( num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.resnet152.__name__: model = models.resnet152( num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.vgg16.__name__: model = models.vgg16(num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.vgg19.__name__: model = models.vgg19(num_classes=2 if use_actual_num_classes else 1000) elif model_name == models.densenet169.__name__: model = models.densenet169( num_classes=2 if use_actual_num_classes else 1000) else: log.fatal("model name is not known: " + model_name) sys.exit(1) try: log.info("Using class size as: {}".format( 2 if use_actual_num_classes else 1000)) return load_model(model, out_file) except RuntimeError as re: log.error(re) return weighted_model(model_name, pretrain_file, True)
def run_model(model_name, X, y, kf): collect_garbage() if model_name == "svm": run_svm(X=X, y=y, kf=kf) elif model_name == "lr": run_lr(X=X, y=y, kf=kf) elif model_name == "knn": run_knn(X=X, y=y, kf=kf) elif model_name == "all": log.info("Running ML model: svm") run_svm(X=X, y=y, kf=kf) log.info("Running ML model: lr") run_lr(X=X, y=y, kf=kf) log.info("Running ML model: knn") run_knn(X=X, y=y, kf=kf) else: log.fatal("ML model name is not known: " + model_name) sys.exit(1)
def get_dataset(dataset_folder, img_size, normalize, divide=False): log.info("Reading dataset") X, y = read_dataset(dataset_folder=dataset_folder, resize_value=(img_size, img_size), to_crop=True) if normalize: X = StandardScaler().fit_transform(X) if divide: log.info("Dividing dataset into train and test data") X_tr, y_tr, X_ts, y_ts = divide_dataset(X, y) log.info("Train data length: %d" % len(y_tr)) log.info("Test data length: %d" % len(y_ts)) return X_tr, y_tr, X_ts, y_ts return X, y
def main(save=False, dataset_folder="dataset", batch_size=20, img_size=112, test_without_train=False, pretrain_file=None, num_workers=4, model_name='alexnet', optimizer_name='Adam', is_pre_trained=False, fine_tune=False, num_epochs=18, update_lr=True, normalize=None, validation_freq=0.1, lr=0.001, momentum=0.9, partial=0.125, betas=(0.9, 0.99), weight_decay=0.025): if not is_pre_trained and fine_tune: fine_tune = False if test_without_train and pretrain_file is None: log.fatal( "Pretrained weight file is a must on test without train approach") sys.exit(1) log.info("Constructing datasets and loaders") train_data, train_loader, test_data, test_loader = set_dataset_and_loaders( dataset_folder, batch_size, img_size, num_workers, normalize) set_0, set_1 = 0, 0 for imgs, labels in test_loader: if set_0 == 3 and set_1 == 3: break for e, label in enumerate(labels.tolist()): if label == 0 and set_0 != 3: writer.add_image( "{} - class image sample {}".format( train_data.classes[0], set_0), inv_normalize_tensor(imgs[e], normalize)) set_0 += 1 elif label == 1 and set_1 != 3: writer.add_image( "{} - class image sample {}".format( train_data.classes[1], set_1), inv_normalize_tensor(imgs[e], normalize)) set_1 += 1 if set_0 == 3 and set_1 == 3: break log.info("Calling the model: " + model_name) if test_without_train: model = weighted_model(model_name, pretrain_file) test_model(model, test_loader, 0) else: run_model(model_name=model_name, optimizer_name=optimizer_name, is_pre_trained=is_pre_trained, fine_tune=fine_tune, train_loader=train_loader, test_loader=test_loader, num_epochs=num_epochs, save=save, update_lr=update_lr, dataset_folder=dataset_folder, validation_freq=validation_freq, lr=lr, momentum=momentum, partial=partial, betas=betas, weight_decay=weight_decay) collect_garbage() writer.close()
run_model(model_name=model_name, optimizer_name=optimizer_name, is_pre_trained=is_pre_trained, fine_tune=fine_tune, train_loader=train_loader, test_loader=test_loader, num_epochs=num_epochs, save=save, update_lr=update_lr, dataset_folder=dataset_folder, validation_freq=validation_freq, lr=lr, momentum=momentum, partial=partial, betas=betas, weight_decay=weight_decay) collect_garbage() writer.close() if __name__ == '__main__': save = False log.info("Process Started") main(model_name="resnet18", is_pre_trained=True, pretrain_file="84.35_PreTrained_resnet18_Adam_dataset_out", img_size=112, test_without_train=True) log.info("Process Finished")
def main(transfer_learning, method="", ml_model_name="", cv=10, dataset_folder="dataset", pretrain_file=None, batch_size=8, img_size=112, num_workers=4, cnn_model_name="", optimizer_name='Adam', validation_freq=0.1, lr=0.001, momentum=0.9, partial=0.125, betas=(0.9, 0.99), weight_decay=0.025, update_lr=True, is_pre_trained=False, fine_tune=False, num_epochs=16, normalize=True, seed=17): if not transfer_learning: if method.lower() == "ml": run_ML.main(model_name=ml_model_name, dataset_folder=dataset_folder, seed=seed, cv=cv, img_size=img_size, normalize=normalize) elif method.lower() == "cnn": run_CNN.main(save=False, dataset_folder=dataset_folder, batch_size=batch_size, test_without_train=False, img_size=img_size, num_workers=num_workers, num_epochs=num_epochs, model_name=cnn_model_name, optimizer_name=optimizer_name, is_pre_trained=is_pre_trained, fine_tune=fine_tune, update_lr=update_lr, normalize=normalize, validation_freq=validation_freq, lr=lr, momentum=momentum, partial=partial, betas=betas, weight_decay=weight_decay) else: log.fatal("method name is not known: " + method) sys.exit(1) else: log.info("Constructing datasets and loaders") train_data, train_loader, test_data, test_loader = set_dataset_and_loaders(dataset_folder=dataset_folder, batch_size=batch_size, img_size=img_size, num_workers=num_workers, normalize=normalize) if is_pre_trained and pretrain_file is not None and \ cnn_model_name in pretrain_file.lower(): log.info("Getting PreTrained CNN model: " + cnn_model_name + " from the Weights of " + pretrain_file) model = cnn_model.weighted_model(cnn_model_name, pretrain_file) else: log.info("Running CNN model: " + cnn_model_name) model = cnn_model.run_model(model_name=cnn_model_name, optimizer_name=optimizer_name, fine_tune=fine_tune, is_pre_trained=is_pre_trained, train_loader=train_loader, num_epochs=num_epochs, test_loader=test_loader, validation_freq=validation_freq, lr=lr, momentum=momentum, partial=partial, betas=betas, weight_decay=weight_decay, update_lr=update_lr, save=False, dataset_folder=dataset_folder) log.info("Feature extractor is being created") feature_extractor = get_feature_extractor(cnn_model_name, model.eval()) log.info("Feature extractor is setting to device: " + str(device)) feature_extractor = feature_extractor.to(device) log.info("Merging CNN train&test datasets") dataset = train_data + test_data log.info("Constructing loader for merged dataset") data_loader = set_loader(dataset=dataset, batch_size=int(len(dataset) / 5), shuffle=False, num_workers=num_workers) log.info("Extracting features as X_cnn array and labels as general y vector") X_cnn, y = extract_features(data_loader, feature_extractor) class_dist = {i: y.count(i) for i in y} class0_size = class_dist[0] class1_size = class_dist[1] log.info("Total class 0 size: " + str(class0_size)) log.info("Total class 1 size: " + str(class1_size)) if normalize: X_cnn = Normalizer().fit_transform(X_cnn) X_cnn = StandardScaler().fit_transform(X_cnn) log.info("Number of features in X_cnn: " + str(len(X_cnn[0]))) kf = KFold(n_splits=cv, shuffle=True, random_state=seed) ml_model.run_model(ml_model_name, X_cnn, y, kf) collect_garbage()
def run_model(model_name, optimizer_name, is_pre_trained, fine_tune, train_loader, test_loader, validation_freq, lr, momentum, partial, betas, weight_decay, update_lr=True, num_epochs=25, save=False, dataset_folder="dataset"): collect_garbage() MODEL_NAME[0] = model_name num_classes = len(train_loader.dataset.classes) log.info("Instantiate the model") if model_name == models.alexnet.__name__: model = prepare_alexnet(is_pre_trained, fine_tune, num_classes) elif model_name in (models.resnet18.__name__, models.resnet50.__name__, models.resnet152.__name__): model = prepare_resnet(model_name, is_pre_trained, fine_tune, num_classes) elif model_name in (models.vgg16.__name__, models.vgg19.__name__): model = prepare_vgg(model_name, is_pre_trained, fine_tune, num_classes) elif model_name == models.densenet169.__name__: model = prepare_densenet(is_pre_trained, fine_tune, num_classes) else: log.fatal("model name is not known: " + model_name) sys.exit(1) log.info("Setting the model to device") model = model.to(device) if "densenet" not in model_name: log.info("The summary:") get_summary(model, train_loader) collect_garbage() log.info("Setting the loss function") metric = nn.CrossEntropyLoss() model_parameters = get_grad_update_params(model, fine_tune) if optimizer_name == optim.Adam.__name__: optimizer = optim.Adam(model_parameters, lr=lr) elif optimizer_name == optim.SGD.__name__: optimizer = optim.SGD(model_parameters, lr=lr, momentum=momentum) elif optimizer_name == padam.Padam.__name__: optimizer = padam.Padam(model_parameters, lr=lr, partial=partial, weight_decay=weight_decay, betas=betas) else: log.fatal("not implemented optimizer name: {}".format(optimizer_name)) sys.exit(1) log.info("Setting the optimizer as: {}".format(optimizer_name)) SAVE_FILE[0] = ( "" if not is_pre_trained else "PreTrained_" ) + model_name + "_" + optimizer_name + "_" + dataset_folder + "_out.pth" last_val_iterator = train_model(model, train_loader, test_loader, metric, optimizer, lr=lr, num_epochs=num_epochs, update_lr=update_lr, validation_freq=validation_freq, save=save) log.info("Testing the model") test_acc = test_model(model, test_loader, last_val_iterator) if save and is_verified(test_acc): exist_files = path_exists(ROOT_DIR, SAVE_FILE[0], "contains") better = len(exist_files) == 0 if not better: exist_acc = [] for file in exist_files: exist_acc.append(float(file.split("_")[0].replace(",", "."))) better = all(test_acc > acc for acc in exist_acc) if better: save_model(model=model, path=str(round(test_acc, 2)) + "_" + SAVE_FILE[0]) return model
def get_prediction_kf(kf, model, X, y, tag=None): cv = kf.n_splits ratios = [] conf_matrices = [] roc_list = [] for e, (train, test) in enumerate(kf.split(X, y)): if not isinstance(X, np.ndarray): X = np.array(X) X_train, X_test, y_train, y_test = X[train], X[test], np.array( y)[train], np.array(y)[test] model.fit(X_train, y_train) success_ratio = model.score(X_test, y_test) log.info( str(cv) + "-Fold CV -- Iteration " + str(e) + " Test Success Ratio: " + str(100 * success_ratio) + "%") ratios.append(success_ratio) test_prob = model.predict_proba(X_test) auc = roc_auc_score(y_test, test_prob[:, 1]) log.info( str(cv) + "-Fold CV -- Iteration " + str(e) + " AUC Score: " + str(auc)) roc_list.append(auc) conf_matrix = confusion_matrix(y_test.tolist(), model.predict(X_test).tolist()) log.info( str(cv) + "-Fold CV -- Iteration " + str(e) + " Confusion Matrix:\n" + str(conf_matrix)) conf_matrices.append(conf_matrix) if tag is not None: writer.add_scalar(tag, success_ratio, e) log.info( str(cv) + "-Fold CV Average Test Success Ratio: " + str(100 * np.average(np.array(ratios))) + "%") log.info( str(cv) + "-Fold CV Average AUC Score: " + str(np.average(np.array(roc_list)))) log.info( str(cv) + "-Fold CV Average Confusion Matrix:\n" + str(np.mean(conf_matrices, axis=0)))
def get_prediction(model, X_ts, y_ts): log.info("Test Success Ratio: " + str(100 * model.score(X_ts, y_ts)) + '%')
import os import torch from util.logger_util import log ROOT_DIR = str(os.path.dirname(os.path.abspath(__file__))) # os.environ["KERAS_BACKEND"] = "plaidml.keras.backend" # log.info("envion is set as: %s" % str(os.environ["KERAS_BACKEND"])) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") log.info("Device is selected as %s" % device) SAVE_FILE = [""] MODEL_NAME = [""]
def set_dataset_and_loaders(dataset_folder, batch_size, img_size, num_workers, normalize=None): dataset_dir = ROOT_DIR.split("cnn")[0] log.info("Setting train data") train_data = set_dataset(folder=dataset_dir + dataset_folder + '/train', size=img_size, normalize=normalize) log.info("Train data length: %d" % len(train_data)) log.info("Setting test data") test_data = set_dataset(folder=dataset_dir + dataset_folder + '/test', size=img_size, normalize=normalize) log.info("Test data length: %d" % len(test_data)) log.info("Setting train loader") train_loader = set_loader(dataset=train_data, batch_size=batch_size, shuffle=True, num_workers=num_workers) log.info("Setting test loader") test_loader = set_loader(dataset=test_data, batch_size=batch_size, shuffle=False, num_workers=num_workers) return train_data, train_loader, test_data, test_loader
def train_model(model, train_loader, test_loader, metric, optimizer, lr, validation_freq, save, num_epochs=25, update_lr=False): total_loss_history = [] total_acc_history = [] validate_every = max(1, math.floor(num_epochs * validation_freq)) last_validate_iter = 0 log.info("Training the model") # Iterate through train set mini batches for epoch in trange(num_epochs): correct = 0 total = len(train_loader.dataset) update = update_lr loss_history = [] for e, (images, labels) in enumerate(train_loader): # tqdm # zero the parameter gradients optimizer.zero_grad() inputs = images.to(device) labels = labels.to(device) # Do the forward pass outputs = model(inputs) predictions = torch.argmax(outputs, dim=1) truths = torch.sum((predictions == labels).float()) correct += truths.item() loss = metric(outputs, labels) loss_history.append(loss.item()) if update \ and (epoch != 0 and epoch != num_epochs - 1)\ and e == len(train_loader) - 1 \ and (epoch + 1) % int(num_epochs / 4) == 0: update = False lr = float(lr / 10) log.info("learning rate is updated to " + str(lr)) optimizer = optim.Adam(optimizer.param_groups, lr=lr) # Calculate gradients and step loss.backward() optimizer.step() log.info("\nIteration number on epoch %d / %d is %d" % (epoch + 1, num_epochs, len(loss_history))) epoch_loss = sum(loss_history) / len(loss_history) writer.add_scalar(MODEL_NAME[0] + "/Loss/Train", epoch_loss, epoch) total_loss_history.append(epoch_loss) epoch_acc = correct / total writer.add_scalar(MODEL_NAME[0] + "/Acc/Train", epoch_acc, epoch) total_acc_history.append(epoch_acc) log.info("Epoch {} --> training loss: {} - training acc: {}".format( epoch + 1, round(epoch_loss, 4), round(epoch_acc, 4))) if epoch % validate_every == 0 and epoch != (num_epochs - 1): last_validate_iter = int(epoch / validate_every) validate_model(model, test_loader, metric, last_validate_iter, save) model = model.train() metric = metric.train() log.info("\nTotal training iteration: %d" % len(total_loss_history)) total_loss = sum(total_loss_history) / len(total_loss_history) total_acc = sum(total_acc_history) / len(total_acc_history) log.info("Average --> training loss: {} - training acc: {} ".format( round(total_loss, 6), round(total_acc, 6))) return last_validate_iter