def test(config): dataset = img_dataset("./dataset/test", "test") dataloader = torch.utils.data.DataLoader( dataset=dataset, batch_size=config.bs ) net = Classifier(num_classes=13).cuda() net.load_state_dict( torch.load( join( f"{config.weight_path}", f"{config.test_epoch}_classifier.pth" ) ) ) net.eval() with open("output.csv", "w", newline="") as csvfile: writer = csv.writer(csvfile, delimiter=",") writer.writerow(["id", "label"]) for _, data in enumerate(dataloader): with torch.no_grad(): inputs, img_name = data inputs = inputs.cuda() outputs = net(inputs) _, predicted = torch.max(outputs.data, 1) writer.writerow([img_name[0], class_name[predicted.data]])
def evaluate_max_prediction(valSlideList, netAcc): predicted_labels = [] for slide in valSlideList.getSlideList(): # CREATE ITERATOR number_of_patches = len(slide) val_dataset = dataset.img_dataset( slide, batch_size=netAcc.getBatchSize(), shuffle_buffer_size=netAcc.getShuffleBufferSize(), shuffle=False) val_iterator = val_dataset.make_initializable_iterator() # PREDICT LIST OF PATCHES batch_pred_y_pred, batch_pred_y_pred_prob, batch_pred_y_argmax = predict.predict_given_net( iterator_handle=netAcc.getIteratorHandle(), pred_iterator_handle=val_iterator, pred_iterator_len=number_of_patches, ) # MAJORITY VOTE predict.predict_vote(batch_pred_y_argmax, valSlideList.getLabelEncoder()) # ADD TO PREDICTED LABELS # calculate accuracy return #accuracy and confusion matrix
def test_existing_net(slide_datapath, net_loadpath, model_name, dropout_ratio, batch_size, do_augment=False, shuffle_buffer_size=2000): slide_list, slide_dimensions, num_patches, slide_label = data_tf.collect_data( slide_datapath, batch_size) patches = dataset.slidelist_to_patchlist(slide_list) with tf.Session() as sess: if do_augment: input_dataset = dataset.img_dataset_augment(patches, batch_size=batch_size, shuffle_buffer_size=shuffle_buffer_size, shuffle=True) else: input_dataset = dataset.img_dataset(patches, batch_size=batch_size, shuffle_buffer_size=shuffle_buffer_size, shuffle=True) input_iterator = input_dataset.make_initializable_iterator() iterator_handle = sess.run(input_iterator.string_handle()) proxy_iterator_handle_ph = tf.placeholder(tf.string, shape=[]) proxy_iterator = tf.data.Iterator.from_string_handle(proxy_iterator_handle_ph, output_types=input_iterator.output_types, output_shapes=input_iterator.output_shapes) x, y = proxy_iterator.get_next() train_op, loss_op, y, accuracy_op, x, keep_prob_ph, learning_rate_ph, is_training_ph, y_pred_op, y_argmax_op = \ netutil.build_model(model_name, x, y, use_bn_1=True, use_bn_2=True, use_dropout_1=True, use_dropout_2=True) # model saver saver = tf.train.Saver() ######################## # load model from disc saver.restore(sess, net_loadpath) validate.validate_existing_net(iterator_handle, input_iterator, num_patches, dropout_ratio, batch_size, loss_op, accuracy_op, keep_prob_ph, is_training_ph, proxy_iterator_handle_ph, sess) # net_loadpath = "/home/oole/tfnetsave/tfnet_em_full_premod" # slide_datapath = "/home/oole/Data/training/patient_patches_premod_jpg" # model_name="model" # dropout_ratio=0.5 # batch_size=64 # shuffle_buffer_size=2000 # # test_existing_net(slide_datapath, net_loadpath, model_name, dropout_ratio, batch_size, do_augment=False)
def train(config): lera.log_hyperparams({ "title": "hw1", "epoch": config.epochs, "lr": config.lr }) dataset = img_dataset("./dataset/train", "train") dataloader = torch.utils.data.DataLoader(dataset=dataset, batch_size=config.bs, shuffle=True, drop_last=True) net = Classifier(num_classes=13).cuda() net.load_state_dict( torch.load( join(f"{config.weight_path}", f"{config.pre_epochs}_classifier.pth"))) criterion = nn.CrossEntropyLoss().cuda() optimizer = optim.Adam(net.parameters(), lr=config.lr) for epoch in range(config.epochs): for _, data in enumerate(dataloader, 0): optimizer.zero_grad() net.train() inputs, labels = data inputs = inputs.cuda() labels = labels.cuda() outputs = net(inputs) loss = criterion(outputs, labels) loss.backward() optimizer.step() _, predicted = torch.max(outputs.data, 1) correct_counts = predicted.eq(labels.data.view_as(predicted)) train_acc = torch.sum(correct_counts).item() / predicted.size(0) lera.log({"loss": loss.item(), "acc": train_acc}) print("epoch:{}/{}, loss:{}, acc:{:02f}".format( epoch + 1 + config.pre_epochs, config.epochs + config.pre_epochs, loss.item(), train_acc, )) if (epoch + 1 + config.pre_epochs) % 10 == 0: torch.save( net.state_dict(), join( f"{config.weight_path}", f"{epoch + 1 + config.pre_epochs}_classifier.pth", ), )
lr = 0.0001 #### log dir log_directory = "/home/oole/tf_log" """ LOAD DATA """ # training_data_path = 'D:/Data/tf_test_data/validation' training_data_path = '/home/oole/tf_test_data/validation' train_slidelist, train_slide_dimensions, train_num_patches, train_slide_label = data_tf.collect_data( training_data_path, batch_size) patches = dataset.slidelist_to_patchlist(train_slidelist) no_patches = len(patches) """ CREATE TRAINING AND VALIDATION DATASET """ train_dataset = dataset.img_dataset(patches, batch_size, shuffle=True, shuffle_buffer_size=no_patches) train_iterator = train_dataset.make_initializable_iterator() val_dataset = dataset.img_dataset(patches, batch_size, shuffle=False, shuffle_buffer_size=no_patches) val_iterator = val_dataset.make_initializable_iterator() with tf.Session() as sess: iterator_handle, iterator_access, proxy_iterator = dataset.proxy_iterator( sess, train_iterator, val_iterator) train_iterator_handle = iterator_access[0] val_iterator_handle = iterator_access[1]
def train_on_discriminative_patches(trainSlideData, valSlideData, netAccess, H, initial_epochnum, num_epochs, num_patches, dropout_ratio, learning_rate, sess, do_augment=False, runName="", logregSavePath=None, discriminativePatchFinderPredict=None, verbose=2, do_simple_validation=True): slideList = trainSlideData.getSlideList() batchSize = netAccess.getBatchSize() train_patches = dataset.slidelist_to_patchlist(slideList, H=H) # Important shuffle!!! (shuffle buffer may be too small to properly shuffle np.random.shuffle(train_patches) val_patches = dataset.slidelist_to_patchlist(valSlideData.getSlideList()) if len(train_patches) != num_patches: raise Exception("H did not work correctly") if do_augment: train_dataset = dataset.img_dataset_augment( train_patches, batch_size=batchSize, shuffle_buffer_size=shuffle_buffer_size, shuffle=True, getlabel=trainSlideData.getLabelFunc(), labelEncoder=trainSlideData.getLabelEncoder(), parseFunctionAugment=trainSlideData.getparseFunctionAugment()) else: train_dataset = dataset.img_dataset( train_patches, batch_size=batchSize, shuffle_buffer_size=shuffle_buffer_size, shuffle=True) val_dataset = dataset.img_dataset( val_patches, batch_size=batchSize, getlabel=valSlideData.getLabelFunc(), labelEncoder=valSlideData.getLabelEncoder(), parseFunction=valSlideData.getparseFunctionNormal()) val_iterator = val_dataset.make_initializable_iterator() train_iterator = train_dataset.make_initializable_iterator() actualEpoch = initial_epochnum train_accuracy = train.train_given_net( netAccess, len(train_patches), train_iterator, val_iterator_len=len(val_patches), val_iterator=val_iterator, num_epochs=num_epochs, batch_size=batchSize, dropout_ratio=dropout_ratio, learning_rate=learning_rate, sess=sess, runName=runName, actualEpoch=actualEpoch, verbose=verbose, do_simple_validation=do_simple_validation) # Train logreg model with current net logregModel = train_logreg.train_logreg(netAccess, logregSavePath, trainSlideData, dropout_ratio, sess, discriminativePatchFinderPredict) evaluate.evaluateNet( netAccess, logregModel, valSlideData, actualEpoch, sess=sess, dropout=dropout_ratio, runName=runName, discriminativePatchFinder=discriminativePatchFinderPredict) actualEpoch += num_epochs return train_accuracy