def __init__(self, channels, classes, imagesize, **kwargs): super(ModelCnn, self).__init__() self.layers = Cnn.get_layers(channels, classes, imagesize) self.distills = torch.nn.ModuleList([ models.GlobalSumPool( h=models.DenseNet(headsize=32, layers=1, dropout=0.2), c=models.Classifier(32, classes + 1, useprototype=1, usenorm=0, p=2), ), models.GlobalSumPool( h=models.DenseNet(headsize=64, layers=1, dropout=0.2), c=models.Classifier(64, classes + 1, useprototype=1, usenorm=0, p=2), ), models.GlobalSumPool( h=models.DenseNet(headsize=64, layers=1, dropout=0.2), c=models.Classifier(64, classes + 1, useprototype=1, usenorm=0, p=2), ) ])
def make_distillpools(self, classes): return [ models.GlobalSumPool( h=models.DenseNet(headsize=64, bodysize=256, tailsize=self.squash[3], layers=self.layers, dropout=0.2, activation=self.act, bias=self.usebias), c=models.Classifier(self.squash[3], classes + self.optout, useprototype=self.useprototype, usenorm=self.usenorm, p=self.p), ), models.GlobalSumPool( h=models.DenseNet(headsize=128, bodysize=256, tailsize=self.squash[5], layers=self.layers, dropout=0.2, activation=self.act, bias=self.usebias), c=models.Classifier(self.squash[5], classes + self.optout, useprototype=self.useprototype, usenorm=self.usenorm, p=self.p), ), models.GlobalSumPool( h=models.DenseNet(headsize=256, bodysize=1024, tailsize=self.squash[7], layers=self.layers, dropout=0.2, activation=self.act, bias=self.usebias), c=models.Classifier(self.squash[7], classes + self.optout, useprototype=self.useprototype, usenorm=self.usenorm, p=self.p), ) ]
def test(in_dataset, out_dataset, wide, epsilon, temperature): testsetout = torchvision.datasets.ImageFolder(os.path.expanduser( "./data/{}".format(out_dataset)), transform=transform) testloaderOut = torch.utils.data.DataLoader(testsetout, batch_size=100, shuffle=False, num_workers=2) if in_dataset == "cifar100": testset = torchvision.datasets.CIFAR100(root='./data', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) elif in_dataset == "cifar10": testset = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform) testloaderIn = torch.utils.data.DataLoader(testset, batch_size=100, shuffle=False, num_workers=2) for fold in range(1, 6): print(f"Processing fold {fold}") nclasses = int(in_dataset[5:]) if wide: net = models.WideResNet(int(nclasses * 4 / 5)) ck = torch.load( f"./checkpoints/{in_dataset}_fold_{fold}_wide_checkpoint/model_best.pth.tar" ) else: net = models.DenseNet(int(nclasses * 4 / 5)) ck = torch.load( f"./checkpoints/{in_dataset}_fold_{fold}_dense_checkpoint/model_best.pth.tar" ) net.load_state_dict(ck['state_dict']) net.cuda() net.eval() d.testData(net, criterion, testloaderIn, testloaderOut, in_dataset, out_dataset, epsilon, temperature, fold) m.test(in_dataset, out_dataset, plot=True)
def set_model(self): if args.wide: self.g = models.WideResNet().cuda() self.c1 = models.Classifier(self.g.nChannels, self.args.num_classes).cuda() self.c2 = models.Classifier(self.g.nChannels, self.args.num_classes).cuda() else: self.g = models.DenseNet().cuda() self.c1 = models.Classifier(self.g.in_planes, self.args.num_classes).cuda() self.c2 = models.Classifier(self.g.in_planes, self.args.num_classes).cuda()
def main(): logger = setup_logger(filename='log.txt') train_epoch_number = 10 batch_size = 100 device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") # dataset = np.loadtxt(models.Config.dataset_path, delimiter=',') # np.random.shuffle(dataset[:5000]) transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)) ]) # trainset = models.DataSet(dataset[:5000]) trainset = torchvision.datasets.CIFAR10(root='./data/cifar10', train=True, download=False, transform=transform) trainloader = DataLoader(dataset=trainset, batch_size=batch_size, shuffle=True, num_workers=24) # testset = models.DataSet(dataset[5000:]) testset = torchvision.datasets.CIFAR10(root='./data/cifar10', train=False, download=False, transform=transform) testloader = DataLoader(dataset=testset, batch_size=1, shuffle=False, num_workers=24) prototypes = {} # net = models.CNNNet(device=device) net = models.DenseNet(device=device, number_layers=8, growth_rate=12, drop_rate=0.0) logger.info("DenseNet Channels: %d", net.channels) gcpl = models.GCPLLoss(threshold=models.Config.threshold, gamma=models.Config.gamma, b=models.Config.threshold, tao=1.0, beta=0.5, lambda_=0.001) sgd = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) if not os.path.exists("pkl"): os.mkdir("pkl") if os.path.exists(models.Config.pkl_path): state_dict = torch.load(models.Config.pkl_path) try: net.load_state_dict(state_dict) logger.info("Load state from file %s.", models.Config.pkl_path) except RuntimeError: logger.error("Loading state from file %s failed.", models.Config.pkl_path) for epoch in range(train_epoch_number): logger.info("Trainset size: %d, Epoch number: %d", len(trainset), epoch + 1) running_loss = 0.0 for i, (features, labels) in enumerate(trainloader): features = features.to(net.device) sgd.zero_grad() features = net(features).view(batch_size, 1, -1) loss = gcpl(features, labels, prototypes) loss.backward() sgd.step() running_loss += loss.item() / batch_size logger.debug("[%3d, %5d] loss: %7.4f", epoch + 1, i + 1, loss.item() / batch_size) torch.save(net.state_dict(), models.Config.pkl_path) prototype_count = 0 for c in prototypes: prototype_count += len(prototypes[c]) logger.info("Prototypes Count: %d", prototype_count) # if (epoch + 1) % 5 == 0: distance_sum = 0.0 correct = 0 for i, (feature, label) in enumerate(testloader): feature = net(feature.to(net.device)).view(1, -1) predicted_label, probability, min_distance = models.predict( feature, prototypes) if label == predicted_label: correct += 1 distance_sum += min_distance logger.debug( "%5d: Label: %d, Prediction: %d, Probability: %7.4f, Distance: %7.4f, Accuracy: %7.4f", i + 1, label, predicted_label, probability, min_distance, correct / (i + 1)) logger.info("Distance Average: %7.4f", distance_sum / len(testloader)) logger.info("Accuracy: %7.4f\n", correct / len(testloader))
def set_model(self): if args.wide: self.model = models.WideResNet(int(args.num_classes * 4 / 5), dropRate=0.3).cuda() else: self.model = models.DenseNet(int(args.num_classes * 4 / 5)).cuda()
def stream(config, trainset, streamset): logger = logging.getLogger(__name__) net = models.DenseNet(device=torch.device(config.device), tensor_view=trainset.tensor_view, number_layers=config.number_layers, growth_rate=config.growth_rate, drop_rate=config.drop_rate) logger.info("densenet channel: %d", net.channels) try: net.load(config.net_path) except FileNotFoundError: pass else: logger.info("Load model from file '%s'.", config.net_path) criterion = models.CPELoss(gamma=config.gamma, tao=config.tao, b=config.b, beta=config.beta, lambda_=config.lambda_) optimizer = optim.SGD(net.parameters(), lr=config.learning_rate, momentum=0.9) prototypes = models.Prototypes(threshold=config.threshold) # load saved prototypes try: prototypes.load(config.prototypes_path) except FileNotFoundError: pass else: logger.info("load prototypes from file '%s'.", config.prototypes_path) logger.info("original prototype count: %d", len(prototypes)) detector = None def train(train_dataset): logger.info('---------------- train ----------------') dataloader = DataLoader(dataset=train_dataset, batch_size=1, shuffle=True) for epoch in range(config.epoch_number): logger.info('---------------- epoch: %d ----------------', epoch + 1) logger.info("threshold: %.4f, gamma: %.4f, tao: %.4f, b: %.4f", config.threshold, config.gamma, config.tao, config.b) logger.info("prototypes count before training: %d", len(prototypes)) net.train() for i, (feature, label) in enumerate(dataloader): feature, label = feature.to(net.device), label.to(net.device) optimizer.zero_grad() feature, out = net(feature) loss, distance = criterion(feature, out, label, prototypes) loss.backward() optimizer.step() logger.debug("[train %d, %d] %7.4f %7.4f", epoch + 1, i + 1, loss.item(), distance) logger.info("prototypes count after training: %d", len(prototypes)) prototypes.update() logger.info("prototypes count after update: %d", len(prototypes)) net.save(config.net_path) logger.info("net has been saved.") prototypes.save(config.prototypes_path) logger.info("prototypes has been saved.") intra_distances = [] with torch.no_grad(): net.eval() for i, (feature, label) in enumerate(dataloader): feature, label = feature.to(net.device), label.item() feature, out = net(feature) closest_prototype, distance = prototypes.closest( feature, label) intra_distances.append((label, distance)) novelty_detector = models.Detector(intra_distances, train_dataset.label_set, config.std_coefficient) logger.info("distance average: %s", novelty_detector.average_distances) logger.info("distance std: %s", novelty_detector.std_distances) logger.info("detector threshold: %s", novelty_detector.thresholds) novelty_detector.save(config.detector_path) logger.info("detector has been saved.") return novelty_detector def test(test_dataset, novelty_detector): logger.info('---------------- test ----------------') dataloader = DataLoader(dataset=test_dataset, batch_size=1, shuffle=False) logger.info("known labels: %s", novelty_detector.known_labels) logger.info("distance average: %s", novelty_detector.average_distances) logger.info("distance std: %s", novelty_detector.std_distances) logger.info("detector threshold: %s", novelty_detector.thresholds) detection_results = [] with torch.no_grad(): net.eval() for i, (feature, label) in enumerate(dataloader): feature, label = feature.to(net.device), label.item() feature, out = net(feature) predicted_label, distance = models.predict(feature, prototypes) prob = models.probability(feature, predicted_label, prototypes, gamma=config.gamma) detected_novelty = novelty_detector(predicted_label, distance) real_novelty = label not in novelty_detector.known_labels detection_results.append( (label, predicted_label, real_novelty, detected_novelty)) logger.debug("[test %5d]: %d, %d, %7.4f, %7.4f, %5s, %5s", i + 1, label, predicted_label, prob, distance, real_novelty, detected_novelty) tp, fp, fn, tn, cm, acc, acc_all = novelty_detector.evaluate( detection_results) precision = tp / (tp + fp + 1) recall = tp / (tp + fn + 1) logger.info("accuracy of known labels: %.4f", acc) logger.info("accuracy of all labels: %.4f", acc_all) logger.info("true positive: %d", tp) logger.info("false positive: %d", fp) logger.info("false negative: %d", fn) logger.info("true negative: %d", tn) logger.info("precision: %7.4f", precision) logger.info("recall: %7.4f", recall) logger.info("confusion matrix: \n%s", cm) def stream_train(train_dataset, stream_dataset): logger.info('---------------- stream train ----------------') logger.info('---------------- initial train ----------------') novelty_detector = train(trainset) logger.info('---------------- initial test ----------------') test(stream_dataset, novelty_detector) novelty_dataset = dataset.NoveltyDataset(train_dataset) iter_streamloader = enumerate( DataLoader(dataset=stream_dataset, batch_size=1, shuffle=True)) buffer = [] for i, (feature, label) in iter_streamloader: sample = (feature.squeeze(dim=0), label.squeeze(dim=0)) with torch.no_grad(): net.eval() feature, label = feature.to(net.device), label.item() feature, out = net(feature) predicted_label, distance = models.predict(feature, prototypes) prob = models.probability(feature, predicted_label, prototypes, gamma=config.gamma) detected_novelty = novelty_detector(predicted_label, distance) real_novelty = label not in novelty_detector.known_labels if detected_novelty: buffer.append(sample) logger.debug("[stream %5d]: %d, %d, %7.4f, %7.4f, %5s, %5s, %4d", i + 1, label, predicted_label, prob, distance, real_novelty, detected_novelty, len(buffer)) if len(buffer) == 1000: logger.info("novelty dataset size before extending: %d", len(novelty_dataset)) novelty_dataset.extend(buffer, config.novelty_buffer_sample_rate) logger.info("novelty dataset size after extending: %d", len(novelty_dataset)) logger.info( '---------------- incremental train ----------------') novelty_detector = train(novelty_dataset) buffer.clear() return novelty_detector if config.train: for period in range(config.period): logger.info('---------------- period: %d ----------------', period + 1) detector = stream_train(trainset, streamset) test(streamset, detector) else: test(streamset, detector)
print(args) args.cuda = not args.no_cuda and torch.cuda.is_available() print("Random Seed: ", args.seed) torch.manual_seed(args.seed) if args.cuda: torch.cuda.manual_seed(args.seed) kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {} print('Load model') if args.model == 'resnet': model = models.resnet18(num_classes=args.num_classes) elif args.model == 'densenet': model = models.DenseNet(growth_rate=48, drop_rate=0.2, block_config=(6, 6, 6), num_init_features=96) else: raise Exception('invalid model selected') model.load_state_dict(torch.load(args.pre_trained_net)) print('load target data: ', args.dataset) _, test_loader = data_loader.getTargetDataSet(args.dataset, args.batch_size, args.imageSize, args.dataroot) print('load non target data: ', args.out_dataset) nt_test_loader = data_loader.getNonTargetDataSet(args.out_dataset, args.batch_size, args.imageSize, args.dataroot) if args.cuda:
def set_model(self): self.model = models.DenseNet(self.args.num_classes).cuda()
if configuration.down_samp == 'Convolution': down_samp = False else: down_samp = True if configuration.up_samp == 'UpConvolution': up_samp = False else: up_samp = True if configuration.model_type == 'DenseNet': network = models.DenseNet(configuration.input_channels, configuration.n_classes, ignore_class=configuration.mask_class, k0=32, Theta=0.5, Dropout=0.2, Growth_rate=configuration.gr) elif configuration.model_type == 'DenseUNet': network = models.DenseUNet(configuration.input_channels, configuration.n_classes, ignore_class=configuration.mask_class, k0=32, Theta=0.5, Dropout=0.2, Growth_rate=configuration.gr) else: network = models.DenseNetStack(configuration.input_channels, configuration.n_classes, ignore_class=configuration.mask_class,
def set_model(self): self.g = models.DenseNet().cuda() self.c1 = models.Classifier(self.g.in_planes, self.args.num_classes).cuda() self.c2 = models.Classifier(self.g.in_planes, self.args.num_classes).cuda()
def fit(max_epochs, patience, batch_size): """fitting the model""" #defining the split for cross-validation skf = KFold(n_splits=5, shuffle=True, random_state=47 ) #random_state set, such that the split is reproducible #list for saving the output logs logs = [] #training for each of the above defined folds for fold, (train_idx, val_idx) in enumerate( skf.split(X=np.zeros(len(train_df)), y=train_df['target'], groups=train_df['patient_id'].tolist()), 1): print("FOLD: " + str(fold)) logs.append("FOLD: " + str(fold)) #creating the chosen model model = models.DenseNet(32, [6, 12, 24, 16], len(meta_features)).to(device) print("model initialized!") #setting the optimizer and learning rate scheduler optimizer = torch.optim.Adam(model.parameters(), lr=0.001) lr_scheduler = ReduceLROnPlateau(optimizer=optimizer, mode='min', patience=1, verbose=True, factor=0.2) #getting the indices for the respective train-validation split train_data = train_df.iloc[train_idx].reset_index(drop=True) valid_data = train_df.iloc[val_idx].reset_index(drop=True) #creating the dataset with the respective train-validation splits train_set = MelanomaDataset(df=train_data, imfolder='train', train=True, transforms=train_transform, meta_features=meta_features) val_set = MelanomaDataset(df=valid_data, imfolder='train', train=True, transforms=test_transform, meta_features=meta_features) #initializing the dataloaders train_loader = DataLoader(dataset=train_set, batch_size=batch_size, shuffle=True, num_workers=4) val_loader = DataLoader(dataset=val_set, batch_size=batch_size, shuffle=False, num_workers=4) #variables to save the best loss and best ROC (Area Under Receiving Operator Characteristic) best_loss = 100000000 best_roc = 0.0 #setting the loss functios, BCEWithLogitsLoss includes a sigmoid layer into the Binary Cross-Entropy loss criterion = torch.nn.BCEWithLogitsLoss() criterion_val = torch.nn.BCEWithLogitsLoss() for epoch in range(max_epochs): print(f"Epoch {epoch + 1}/{max_epochs}") # Training one epoch print("") print("training...") epoch_loss, train_acc, train_roc = run_epoch( model, optimizer, train_loader, train=True, loss_fnc=criterion, length_dataset=len(train_set), batch_size=batch_size) print(f"Train loss: {epoch_loss}") print(f"ACC on training set: {train_acc}") print(f"ROC on training set: {train_roc}") print("") #saving the logs to the list logs.append(("Fold: " + str(fold) + ", epoch: " + str(epoch) + ": train loss: " + str(epoch_loss) + ", train acc: " + str(train_acc) + ", train roc: " + str(train_roc))) # Validating one epoch print("validating...") val_loss, val_acc, val_roc = run_epoch(model, None, val_loader, train=False, loss_fnc=criterion_val, length_dataset=len(val_set), batch_size=batch_size) print(f"Test loss: {val_loss}") print(f"Val Acc: {val_acc}") print(f"Val Roc: {val_roc}") print("") #saving the logs to the list logs.append(("Fold: " + str(fold) + ", epoch: " + str(epoch) + ": val loss: " + str(val_loss) + ", val acc: " + str(val_acc) + ", val roc: " + str(val_roc))) #learning rate scheduler step, minimizing the learning rate when the train loss does not decrease for a epoch lr_scheduler.step(epoch_loss) # saving best weights if val_roc >= best_roc: best_epoch = epoch best_loss = val_loss best_acc = val_acc best_roc = val_roc best_model_weights = copy.deepcopy(model.state_dict()) torch.save(model.state_dict(), "best_model" + str(fold)) # Early stopping if epoch - best_epoch >= patience: break #saving the log to a .json file with open("results" + str(fold) + ".json", "w") as file: json.dump(logs, file)
def visualization(config, trainset): logger = logging.getLogger(__name__) trainloader = DataLoader(dataset=trainset, batch_size=1, shuffle=False, num_workers=4) # net = models.CNNNet(device=device) net = models.DenseNet(device=torch.device(config.device), in_channels=config.in_channels, number_layers=config.number_layers, growth_rate=config.growth_rate, drop_rate=0.0) logger.info("DenseNet Channels: %d", net.channels) if config.loss_type == 'gcpl': criterion = models.GCPLLoss(threshold=config.threshold, gamma=config.gamma, lambda_=config.lambda_) elif config.loss_type == 'pdce': criterion = models.PairwiseDCELoss(threshold=config.threshold, tao=config.tao, b=config.b, lambda_=config.lambda_) else: raise RuntimeError('Cannot find "{}" loss type.'.format( config.loss_type)) # load saved model state dict if os.path.exists(config.model_path): state_dict = torch.load(config.model_path) try: net.load_state_dict(state_dict) logger.info("Load model from file '%s'.", config.model_path) except RuntimeError: logger.error("Loading model from file '%s' failed.", config.model_path) # load saved prototypes if os.path.exists(config.prototypes_path): try: criterion.load_prototypes(config.prototypes_path) logger.info("Load prototypes from file '%s'.", config.prototypes_path) except RuntimeError: logger.error("Loading prototypes from file '%s' failed.", config.prototypes_path) # original_features = [] features = [] labels = [] for i, (feature, label) in enumerate(trainloader): feature, label = net(feature.to(net.device)).view(-1), label.item() features.append(feature.data.cpu().numpy()) labels.append(label) features = np.array(features) labels = np.array(labels) feature_tsne = TSNE(n_components=2, random_state=30) features = feature_tsne.fit_transform(features, labels) # original_features = feature_tsne.fit_transform(original_features, labels) plt.figure(figsize=(6, 4)) colors = 'r', 'g', 'b', 'c', 'm', 'y', 'k', 'gray', 'orange', 'purple' for c, label in zip(colors, sorted(list(trainset.label_set))): # print(c, label) plt.scatter(features[labels == label, 0], features[labels == label, 1], c=c, label=label) plt.legend() plt.show()
def run_cel(config, trainset, testset): logger = logging.getLogger(__name__) trainloader = DataLoader(dataset=trainset, batch_size=1, shuffle=True, num_workers=0) testloader = DataLoader(dataset=testset, batch_size=1, shuffle=False, num_workers=0) device = torch.device(config.device) net = models.DenseNet(device=device, in_channels=config.in_channels, number_layers=config.number_layers, growth_rate=12, drop_rate=0.0) logger.info("DenseNet Channels: %d", net.channels) fc_net = models.LinearNet(device=device, in_features=net.channels * (config.tensor_view[1] // 8) * (config.tensor_view[2] // 8)) cel = nn.CrossEntropyLoss() sgd = optim.SGD(net.parameters(), lr=config.learning_rate, momentum=0.9) # load saved model state dict if os.path.exists(config.model_path): state_dict = torch.load(config.model_path) try: net.load_state_dict(state_dict) logger.info("Load state from file '%s'.", config.model_path) except RuntimeError: logger.error("Loading state from file '%s' failed.", config.model_path) for epoch in range(config.epoch_number): logger.info("Epoch number: %d", epoch + 1) probs = [] # train if not config.testonly: running_loss = 0.0 for i, (feature, label) in enumerate(trainloader): feature, label = feature.to(net.device), label.to(net.device) sgd.zero_grad() feature = net(feature).view(1, -1) feature = fc_net(feature) loss = cel(feature, label) loss.backward() sgd.step() feature = feature.data.squeeze() running_loss += loss.item() probs.append((label.item(), feature[label.item()])) logger.debug("[%d, %d] %7.4f", epoch + 1, i + 1, loss.item()) torch.save(net.state_dict(), config.model_path) torch.save(probs, config.probs_path) # end train # load saved probs if config.testonly: if os.path.exists(config.probs_path): try: probs = torch.load(config.probs_path) except RuntimeError: logger.error("Loading probs from file '%s' failed.", config.probs_path) else: logger.info("Load probs from file '%s'.", config.probs_path) # test detector = models.SoftmaxDetector(probs, config.std_coefficient, trainset.label_set) if (epoch + 1) % config.testfreq == 0 or config.testonly: detection_results = [] with torch.no_grad(): for i, (feature, label) in enumerate(testloader): feature, label = feature.to(net.device), label.item() feature = net(feature).view(1, -1) probability, predicted_label = fc_net(feature).max(dim=1) detected_novelty = detector(predicted_label, probability) real_novelty = label not in trainset.label_set detection_results.append( (label, predicted_label.item(), probability, real_novelty, detected_novelty)) logger.debug("%5d: %d, %d, %7.4f, %s, %s", i + 1, label, predicted_label, probability, real_novelty, detected_novelty) true_positive, false_positive, false_negative = detector.evaluate( detection_results) precision = true_positive / (true_positive + false_positive + 1) recall = true_positive / (true_positive + false_negative + 1) cm = confusion_matrix(detector.results['true_label'], detector.results['predicted_label'], sorted(list(testset.label_set))) results = detector.results[np.isin(detector.results['true_label'], list(trainset.label_set))] logger.info( "Accuracy: %7.4f", accuracy_score(results['true_label'], results['predicted_label'])) logger.info("True Positive: %d", true_positive) logger.info("False Positive: %d", false_positive) logger.info("False Negative: %d", false_negative) logger.info("Precision: %7.4f", precision) logger.info("Recall: %7.4f", recall) logger.info("Confusion Matrix: \n%s", cm)
def run(config, trainset, testset): logger = logging.getLogger(__name__) trainloader = DataLoader(dataset=trainset, batch_size=1, shuffle=True, num_workers=4) testloader = DataLoader(dataset=testset, batch_size=1, shuffle=False, num_workers=4) # net = models.CNNNet(device=device) net = models.DenseNet(device=torch.device(config.device), in_channels=config.in_channels, number_layers=config.number_layers, growth_rate=config.growth_rate, drop_rate=0.0) logger.info("DenseNet Channels: %d", net.channels) if config.loss_type == 'gcpl': criterion = models.GCPLLoss(threshold=config.threshold, gamma=config.gamma, lambda_=config.lambda_) elif config.loss_type == 'pdce': criterion = models.PairwiseDCELoss(threshold=config.threshold, tao=config.tao, b=config.b, lambda_=config.lambda_) else: raise RuntimeError('Cannot find "{}" loss type.'.format( config.loss_type)) sgd = optim.SGD(net.parameters(), lr=config.learning_rate, momentum=0.9) # adam = optim.Adam(net.parameters(), lr=config.learning_rate) # load saved optim # if os.path.exists(config.model_path): # state_dict = torch.load(config.optim_path) # try: # net.load_state_dict(state_dict) # logger.info("Load optim from file '%s'.", config.optim_path) # except RuntimeError: # logger.error("Loading optim from file '%s' failed.", config.optim_path) # load saved model state dict if os.path.exists(config.model_path): state_dict = torch.load(config.model_path) try: net.load_state_dict(state_dict) logger.info("Load model from file '%s'.", config.model_path) except RuntimeError: logger.error("Loading model from file '%s' failed.", config.model_path) # load saved prototypes if os.path.exists(config.prototypes_path): try: criterion.load_prototypes(config.prototypes_path) logger.info("Load prototypes from file '%s'.", config.prototypes_path) except RuntimeError: logger.error("Loading prototypes from file '%s' failed.", config.prototypes_path) for epoch in range(config.epoch_number): intra_class_distances = [] # train if not config.testonly: logger.info("Epoch number: %d", epoch + 1) logger.info("threshold: %.4f, gamma: %.4f, tao: %.4f, b: %.4f", config.threshold, config.gamma, config.tao, config.b) running_loss = 0.0 distance_sum = 0.0 if len(criterion.prototypes) > len(trainset.label_set): criterion.clear_prototypes() else: criterion.upgrade_prototypes() for i, (feature, label) in enumerate(trainloader): feature, label = feature.to(net.device), label.item() sgd.zero_grad() feature = net(feature).view(1, -1) loss, distance = criterion(feature, label) loss.backward() sgd.step() running_loss += loss.item() distance_sum += distance intra_class_distances.append((label, distance)) logger.debug("[%d, %d] %7.4f, %7.4f", epoch + 1, i + 1, loss.item(), distance) distances = np.array(intra_class_distances, dtype=[('label', np.int32), ('distance', np.float32)]) average_distance = np.average(distances['distance']).item() std_distance = distances['distance'].std().item() config.threshold = (average_distance + 3 * std_distance) config.gamma = 2 / average_distance config.tao = average_distance + std_distance config.b = std_distance criterion.set_threshold(config.threshold) criterion.set_gamma(config.gamma) criterion.set_tao(config.tao) criterion.set_b(config.b) torch.save(net.state_dict(), config.model_path) criterion.save_prototypes(config.prototypes_path) torch.save(intra_class_distances, config.intra_class_distances_path) logger.info("Prototypes Count: %d", len(criterion.prototypes)) # end train # test if config.testonly: # load saved intra class distances if os.path.exists(config.intra_class_distances_path): try: intra_class_distances = torch.load( config.intra_class_distances_path) logger.info("Load intra class distances from file '%s'.", config.intra_class_distances_path) except RuntimeError: logger.error( "Loading intra class distances from file '%s' failed.", config.intra_class_distances_path) detector = models.Detector(intra_class_distances, config.std_coefficient, trainset.label_set) logger.info("Distance Average: %s", detector.average_distances) logger.info("Distance Std: %s", detector.std_distances) logger.info("Distance Threshold: %s", detector.thresholds) if (epoch + 1) % config.testfreq == 0 or config.testonly: detection_results = [] for i, (feature, label) in enumerate(testloader): feature, label = net(feature.to(net.device)).view( 1, -1), label.item() predicted_label, probability, distance = criterion.predict( feature) detected_novelty = detector(predicted_label, probability, distance) real_novelty = label not in trainset.label_set detection_results.append( (label, predicted_label, probability, distance, real_novelty, detected_novelty)) logger.debug("%5d: %d, %d, %7.4f, %7.4f, %s, %s", i + 1, label, predicted_label, probability, distance, real_novelty, detected_novelty) true_positive, false_positive, false_negative = detector.evaluate( detection_results) precision = true_positive / (true_positive + false_positive + 1) recall = true_positive / (true_positive + false_negative + 1) cm = confusion_matrix(detector.results['true_label'], detector.results['predicted_label'], sorted(list(testset.label_set))) results = detector.results[np.isin(detector.results['true_label'], list(trainset.label_set))] logger.info( "Accuracy: %7.4f", accuracy_score(results['true_label'], results['predicted_label'])) # logger.info("Accuracy: %7.4f", accuracy_score(detector.results['true_label'], detector.results['predicted_label'])) logger.info("True Positive: %d", true_positive) logger.info("False Positive: %d", false_positive) logger.info("False Negative: %d", false_negative) logger.info("Precision: %7.4f", precision) logger.info("Recall: %7.4f", recall) logger.info("Confusion Matrix: \n%s", cm)
def main(): logger = setup_logger(level=logging.DEBUG, filename='log.txt') train_epoch_number = 200 batch_size = 1 pairwise = False device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") dataset = np.loadtxt(models.Config.dataset_path, delimiter=',') np.random.shuffle(dataset[:5000]) trainset = models.DataSet(dataset[:5000], pairwise=pairwise) trainloader = DataLoader(dataset=trainset, batch_size=batch_size, shuffle=True, num_workers=24) testset = models.DataSet(dataset[5000:]) testloader = DataLoader(dataset=testset, batch_size=1, shuffle=False, num_workers=2) # net = models.CNNNet(device=device) net = models.DenseNet(device=device, number_layers=8, growth_rate=12, drop_rate=0.0) logger.info("DenseNet Channels: %d", net.channels) # cel = torch.nn.CrossEntropyLoss() gcpl = functions.GCPLLoss(threshold=models.Config.threshold, gamma=models.Config.gamma, lambda_=0.01) pwl = functions.PairwiseLoss(tao=10.0, b=2.0, beta=0.5) sgd = optim.SGD(net.parameters(), lr=0.001, momentum=0.9) if not os.path.exists("pkl"): os.mkdir("pkl") if os.path.exists(models.Config.pkl_path): state_dict = torch.load(models.Config.pkl_path) try: net.load_state_dict(state_dict) logger.info("Load state from file %s.", models.Config.pkl_path) except RuntimeError: logger.error("Loading state from file %s failed.", models.Config.pkl_path) for epoch in range(train_epoch_number): logger.info("Trainset size: %d, Epoch number: %d", len(trainset), epoch + 1) # CPL train prototypes = train(net, trainloader, (gcpl, pwl), sgd, pairwise=pairwise) torch.save(net.state_dict(), models.Config.pkl_path) prototype_count = 0 for c in prototypes: prototype_count += len(prototypes[c]) logger.info("Prototype Count: %d", prototype_count) accuracy, average_distance = test(net, testloader, prototypes, gcpl.gamma) models.Config.threshold = average_distance * 1.5 logger.info("Distance Average: %7.4f", average_distance) logger.info("Accuracy: %7.4f\n", accuracy)