def run(data_train, data_test, kernel='rbf', C=1e-7, confusion=False):
"""Run the training script and yield accuracy, confusion
:param data_train: Featurized training data, contains X and Y
:param data_test: Featurized test data, contains X and Y
:param kernel: kernel to apply during classification
:param C: Weight of misclassification during classification
:param confusion: Whether or not to print confusion matrix
:return: None
"""
model = train(data_train, C=C, kernel=kernel)
pred_train = predict(model, data_train)
pred_test = predict(model, data_test)
train_accuracy = accuracy_score(data_train['Y'], pred_train)
test_accuracy = accuracy_score(data_test['Y'], pred_test)
print('[%s][%.2E] \t Accuracy: \t (Train) %.4f \t (Val) %.4f' %
(kernel, C, train_accuracy, test_accuracy))
if confusion:
cnf_matrix = confusion_matrix(data_test['Y'], pred_test)
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=CLASS_NAMES,
title='Confusion matrix, without normalization')
plt.savefig('../data/generated/cnf-%s.png' % str(time.time())[-6:])
# Plot normalized confusion matrix
plt.figure()
plot_confusion_matrix(cnf_matrix,
classes=CLASS_NAMES,
normalize=True,
title='Normalized confusion matrix')
plt.savefig('../data/generated/nm-cnf-%s.png' % str(time.time())[-6:])
return {
'model': model,
'train': train_accuracy,
'validation': test_accuracy
}
def on_epoch_end(self, epoch, logs=None):
batch_size = self.model.BATCH_SIZE
class_color_map = self.model.CLS_COLOR_MAP
# get first batch of dataset
(lidar_input,
lidar_mask), label, _ = self.dataset.take(1).get_single_element()
probabilities, predictions = self.model([lidar_input, lidar_mask])
label = label[:self.num_images, :, :]
predictions = predictions[:self.num_images, :, :].numpy()
# label and prediction visualizations
label_image = class_color_map[label.numpy().reshape(-1)].reshape(
[self.num_images, label.shape[1], label.shape[2], 3])
pred_image = class_color_map[predictions.reshape(-1)].reshape(
[self.num_images, label.shape[1], label.shape[2], 3])
# confusion matrix visualization
figure = plot_confusion_matrix(
self.model.miou_tracker.total_cm.numpy(),
class_names=self.model.mc.CLASSES)
cm_image = plot_to_image(figure)
with self.custom_tb_writer.as_default():
tf.summary.image('Images/Depth Image',
lidar_input.numpy()[:self.num_images, :, :, [4]],
max_outputs=batch_size,
step=epoch)
tf.summary.image('Images/Label Image',
label_image,
max_outputs=batch_size,
step=epoch)
tf.summary.image('Images/Prediction Image',
pred_image,
max_outputs=batch_size,
step=epoch)
tf.summary.image("Confusion Matrix", cm_image, step=epoch)
# Save IoU, Precision, Recall
iou, recall, precision = confusion_matrix_to_iou_recall_precision(
self.model.miou_tracker.total_cm)
with self.custom_tb_writer.as_default():
for i, cls in enumerate(self.model.mc.CLASSES):
tf.summary.scalar('IoU/' + cls, iou[i], step=epoch)
tf.summary.scalar('Recall/' + cls, recall[i], step=epoch)
tf.summary.scalar('Precision/' + cls, precision[i], step=epoch)
super().on_epoch_end(epoch, logs)
def main(config, resume, config_testdata=None):
#PREFERENCES
outputOverlaycsv = False
showHeatMap = False
THRESHOLD = 0.45
TRUE_CLASS = 1
PRED_CLASS = 8
# set visualization preference
print("GPUs available: " + str(torch.cuda.device_count()))
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
# setup data_loader instances
data_loader = get_instance(module_data, 'data_loader_test',
config_testdata)
'''
data_loader = getattr(module_data, config['data_loader']['type'])(
config['data_loader']['args']['data_dir'],
batch_size=512,
shuffle=False,
validation_split=0.0,
training=False,
num_workers=2
)
'''
# build model architecture
#config['arch']['args']['num_feature'] = 76
model = get_instance(module_arch, 'arch', config)
print(model)
if torch.cuda.is_available():
print("Using GPU: " + torch.cuda.get_device_name(0))
else:
print("Using CPU to test")
# get function handles of loss and metrics
loss_fn = getattr(module_loss, config['loss'])
criterion = loss_fn(None) # for imbalanced datasets
#criterion = loss_fn(data_loader.dataset.weight.to(device)) # for imbalanced datasets
metric_fns = [getattr(module_metric, met) for met in config['metrics']]
# load state dict
checkpoint = torch.load(resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()
total_loss = 0.0
total_metrics = torch.zeros(len(metric_fns))
#classes = ('endothelium', 'pct', 'vasculature')
classes = ('s1', 's2s3', 'tal', 'dct', 'cd', 'cd45', 'nestin', 'cd31_glom',
'cd31_inter', 'cd45_1', 'cd45_2')
all_pred = []
all_pred_k = []
all_true = []
all_softmax = []
thresh_true = []
thresh_pred = []
thresh_true_bad = []
thresh_pred_bad = []
below_thresh = 0
if showHeatMap:
hm_layers = {
'final_layer': 'layer',
'fc_layer': 'fc_layer',
'conv_num': 17,
'fc_num': 3
} #need to set based on model
heatmapper = classActivationMap.CAMgenerator(hm_layers, config, model)
#heatmapper = classActivationMap.CAMgenerator3d(hm_layers, config, model) #for 3d data
heatmapper.generateImage(num_images=10)
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader)):
k = 2
data, target = data.to(device), target.to(device)
#output = model(data)
output = triple_prediction(data, model)
#image = np.squeeze(data[0].cpu().data.numpy())
label = np.squeeze(target[0].cpu().data.numpy())
all_true.extend(target.cpu().data.numpy())
all_pred.extend(np.argmax(output.cpu().data.numpy(), axis=1))
mypred = torch.topk(output, k, dim=1)[1]
all_pred_k.extend(mypred.cpu().data.numpy())
m = torch.nn.Softmax(dim=0)
for i, row in enumerate(output.cpu()):
sm = m(row)
all_softmax.append(sm.data.numpy())
#print(np.max(sm.numpy(), axis=0))
if np.amax(sm.numpy(), axis=0) > THRESHOLD:
thresh_true.append(target.cpu().data.numpy()[i])
thresh_pred.append(np.argmax(sm))
else:
below_thresh += 1
thresh_true_bad.append(target.cpu().data.numpy()[i])
thresh_pred_bad.append(np.argmax(sm))
if i < 2:
m = torch.nn.Softmax(dim=0)
print("prediction percentages")
print(m(output.cpu()[0]))
print(all_true[i])
#all_softmax.extend(m(output.cpu()))
#if i > 50: break
#
# save sample images, or do something with output here
#
# computing loss, metrics on test set
loss = criterion(output, target)
batch_size = data.shape[0]
total_loss += loss.item() * batch_size
for i, metric in enumerate(metric_fns):
total_metrics[i] += metric(output.cpu(),
target.cpu()) * batch_size
correct = 0
all_pred_k = np.array(all_pred_k)
all_true_k = np.array(all_true)
#CREATE 8 CLASS SOFTMAX
#very ugly please fix
softmax_combined = np.zeros((len(all_softmax), len(all_softmax[0]) - 3))
for i, row in enumerate(all_softmax):
max_pct = np.amax([row[0], row[1]]) #can use amax instead of sum
max_cd45 = np.amax([row[5], row[9],
row[10]]) #can use amax instead of sum
myrow = row[1:len(all_softmax[0]) - 3 + 1]
myrow[0] = max_pct
myrow[4] = max_cd45
softmax_combined[i] = myrow
all_pred_k_combined = torch.topk(
torch.from_numpy(softmax_combined), k, dim=1)[1] + 1
all_pred_k = torch.from_numpy(all_pred_k)
all_true_k = torch.from_numpy(np.array(all_true))
for i in range(k):
correct += torch.sum(all_pred_k[:, i] == all_true_k).item()
print("TOP K all classes = {}".format(correct / len(all_pred_k)))
all_pred_k = np.array(all_pred_k)
all_true_k = np.array(all_true)
#APPLY THRESHOLD - in development
softmax_above_threshold = np.amax(softmax_combined, axis=1)
softmax_above_threshold_mask = softmax_above_threshold > THRESHOLD
thresh_pred_v2 = np.array(all_pred)[softmax_above_threshold_mask]
thresh_true_v2 = np.array(all_true)[softmax_above_threshold_mask]
#REASSIGN LABELS
all_results = [
all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad,
thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2
]
all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels(
all_results, 0, 1)
all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels(
all_results, 9, 5)
all_pred, all_true, thresh_true, thresh_pred, thresh_true_bad, thresh_pred_bad, all_pred_k, all_true_k, thresh_pred_v2, thresh_true_v2 = reassign_labels(
all_results, 10, 5)
#VIEW MISTAKE CLASS
interogate_CM(all_pred, all_true, TRUE_CLASS, PRED_CLASS, data_loader,
classes)
correct = 0
all_pred_k = torch.from_numpy(all_pred_k)
all_true_k = torch.from_numpy(np.array(all_true))
for i in range(k):
correct += torch.sum(all_pred_k_combined[:, i] == all_true_k).item()
print("TOP K Combined classes = {}".format(correct /
len(all_pred_k_combined)))
# SHOW PERCENTAGE OF IMAGES BELOW THRESHOLD AS CONFUSION MATRIX
cm_all = confusion_matrix(all_true, all_pred)
cm_below_thresh = confusion_matrix(thresh_true_bad, thresh_pred_bad)
cm_above_thresh = confusion_matrix(thresh_true, thresh_pred)
compare_CM(cm_above_thresh, cm_all, classes)
if outputOverlaycsv:
ids = data_loader.dataset.getIds()
#softmax = pd.DataFrame(all_softmax)
softmax = pd.DataFrame(softmax_combined)
#ids = ids[:,1].reshape(ids.shape[0], 1)
print(ids[0:5])
print(ids.shape)
print(softmax.shape)
frames = [ids, softmax, pd.DataFrame(all_true)]
output_data = np.concatenate(frames, axis=1)
print(output_data.shape)
output_df = pd.DataFrame(output_data)
output_df.to_csv('overlaycsv.csv', index=False, header=False)
n_samples = len(data_loader.sampler)
print("num test images = " + str(n_samples))
log = {'loss': total_loss / n_samples}
log.update({
met.__name__: total_metrics[i].item() / n_samples
for i, met in enumerate(metric_fns)
})
for key in log:
print("{} = {:.4f}".format(key, log[key]))
#print(log)
log['classes'] = classes
log['test_targets'] = all_true
log['test_predictions'] = all_pred
print("My_metric is accuracy")
print("Number of images below threshold: {}".format(below_thresh))
print("Number of images below threshold v2: {}".format(
len(all_true) - len(thresh_pred_v2)))
util.plot_confusion_matrix(all_true,
all_pred,
classes=classes,
normalize=False)
util.plot_confusion_matrix(thresh_true,
thresh_pred,
classes=classes,
normalize=False)
util.plot_confusion_matrix(thresh_true_bad,
thresh_pred_bad,
classes=classes,
normalize=False)
def main(config, resume):
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print("GPUs available: " + str(torch.cuda.device_count()))
os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
#data_loader = get_instance(module_data, 'data_loader', config) --> cant use this because it will auto split a validation set
data_loader = hdf5_3d_dataloader(
config['data_loader']['args']['hdf5_path'],
batch_size=128,
shuffle=False,
validation_split=0.0,
training=True,
mean=config['data_loader']['args']['mean'],
stdev=config['data_loader']['args']['stdev'])
print(len(data_loader.dataset))
data_loader_test = get_instance(module_data, 'data_loader_test', config)
# Load trained model, including fully connected
model = get_instance(module_arch, 'arch', config)
print(model)
if torch.cuda.is_available():
print("Using GPU: " + torch.cuda.get_device_name(0))
else:
print("Using CPU to test")
loss_fn = getattr(module_loss, config['loss'])
criterion = loss_fn(None)
# load state dict
checkpoint = torch.load(resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
model = model.to(device)
model.eval()
classes = ('S1', 'PCT', 'TAL', 'DCT', 'CD', 'CD45', 'Nestin', 'CD31_glom',
'CD31_inter')
#identify feature layer to separate
save_output = SaveOutput()
#print(model.module.conv_layer4)
handle = model.module.fc6.register_forward_hook(save_output)
#generate feature set
all_true = []
feature_set = []
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader)):
data, target = data.to(device), target.to(device)
output = model(data)
all_true.extend(target.cpu().data.numpy())
print(i * 128)
all_true = np.array(all_true)
all_features = np.array(save_output.outputs)
for item in all_features:
feature_set.extend(item)
all_features = np.array(feature_set)
#all_features = np.reshape(all_features, (feature_shape[0]*feature_shape[1], feature_shape[2]))
print(all_true.shape)
print(all_features.shape)
#create + train logistic model
minmaxscaler = MinMaxScaler()
all_features_scaled = minmaxscaler.fit_transform(all_features)
clf = LogisticRegression(random_state=0,
class_weight='balanced',
verbose=1,
solver='lbfgs',
n_jobs=-1).fit(all_features_scaled, all_true)
#solver options: saga, lbfgs
#test logistic model
save_output_test = SaveOutput()
handle = model.module.fc6.register_forward_hook(save_output_test)
all_true_test = []
feature_set_test = []
all_softmax_cnn = []
with torch.no_grad():
for i, (data, target) in enumerate(tqdm(data_loader_test)):
data, target = data.to(device), target.to(device)
output = model(data)
all_true_test.extend(target.cpu().data.numpy())
#all_pred_cnn.extend(np.argmax(output.cpu().data.numpy(), axis=1))
m = torch.nn.Softmax(dim=0)
for i, row in enumerate(output.cpu()):
sm = m(row)
all_softmax_cnn.append(sm.data.numpy())
all_true_test = np.array(all_true_test)
all_features_test = np.array(save_output_test.outputs)
for item in all_features_test:
feature_set_test.extend(item)
all_features_test = np.array(feature_set_test)
print(all_true_test.shape)
print(all_features_test.shape)
all_pred = clf.predict(minmaxscaler.transform(all_features_test))
all_softmax_lr = clf.predict_proba(all_features_test)
all_softmax_combined = all_softmax_lr + all_softmax_cnn
all_pred_combined = np.argmax(all_softmax_combined, axis=1)
#REASSIGN LABELS
all_results = [all_pred, all_true_test, all_pred_combined]
all_pred, all_true_test, all_pred_combined = reassign_labels(
all_results, 0, 1)
all_pred, all_true_test, all_pred_combined = reassign_labels(
all_results, 9, 5)
all_pred, all_true_test, all_pred_combined = reassign_labels(
all_results, 10, 5)
#display results
util.plot_confusion_matrix(all_true_test,
all_pred,
classes=classes,
normalize=False) #log regression
util.plot_confusion_matrix(
all_true_test, all_pred_combined, classes=classes,
normalize=False) #average of log regression and cnn
def main():
parser = argparse.ArgumentParser(
description="PyTorch DeeplabV3Plus Training")
# cuda, seed and logging
parser.add_argument('--gpu-ids',
type=str,
default='0',
help='use which gpu to train, must be a \
comma-separated list of integers only (default=0)')
# evaluation option
parser.add_argument('--num_image',
type=int,
default=1,
help='visualization number of image')
parser.add_argument('--shuffle',
type=bool,
default=True,
help='data list shuffle')
parser.add_argument('--result',
type=str,
default='eachimage',
choices=['eachimage', 'matrix'],
help='data list shuffle')
args = parser.parse_args()
# default settings for epochs, batch_size and lr
print(args)
deeplab = Deeplab('cfg/deeplab.cfg')
with open('./dataset/val.txt', 'r') as f:
datalist = []
for data in f.readlines():
if '\n' in data:
data = data[:-1]
datalist.append(data)
with open('./dataset/output/class_names.txt', 'r') as f:
ClassName = []
for cl in f.readlines():
if '\n' in cl:
cl = cl[:-1]
ClassName.append(cl)
if args.shuffle:
random.shuffle(datalist)
if args.result == 'matrix':
start = time.time()
for data in tqdm(datalist):
src = Image.open('./dataset/output/JPEGImages/' + data +
'.jpg').convert('RGB')
tar = Image.open('./dataset/output/SegmentationClassPNG/' + data +
'.png')
deeplab.validation_matrix(src, data, tar)
cm = deeplab.evaluator.confusion_matrix
end = time.time()
plot_confusion_matrix(cm / len(datalist), ClassName)
plt.savefig('result/all_CM.png')
print('Image : %d | Time : %.2f | fps : %.1f' %
(len(datalist), end - start, len(datalist) / (end - start)))
elif args.result == 'eachimage':
for i in range(args.num_image):
data = datalist[i]
src = Image.open('./dataset/output/JPEGImages/' + data +
'.jpg').convert('RGB')
tar = Image.open('./dataset/output/SegmentationClassPNG/' + data +
'.png')
deeplab.validation(src, data, ClassName, tar)
def validation(self, src, data, ClassName, tar=None):
self.model.eval()
self.evaluator.reset()
test_loss = 0.0
w, h = src.size
sample = {'image': src, 'label': tar}
sample = self.transform(sample)
image, target = sample['image'], sample['label']
# for the dimension
image = torch.unsqueeze(image, 0)
target = torch.unsqueeze(target, 0)
image, target = image.cuda(), target.cuda()
with torch.no_grad():
output = self.model(image)
imgs = image.data.cpu()
lbl_pred = output.data.max(1)[1].cpu().numpy()[:, :, :]
lbl_true = target.data.cpu() #cpu()
for img, lt, lp in zip(imgs, lbl_true, lbl_pred):
img, lt = self.untransform(img, lt)
viz = fcn.utils.visualize_segmentation(
lbl_pred=lp,
lbl_true=None,
img=img,
n_class=10,
label_names=[
' ', '11_pforceps', '12_mbforceps', '13_mcscissors',
'15_pcapplier', '18_pclip', '20_sxir', '19_mtclip',
'17_mtcapplier', '14_graspers'
])
width = int(viz.shape[1] / 3 * 2)
#Image.fromarray(viz[:,width:,:]).save(str(data)+'.jpg')
def hide(plt):
ax = plt.gca()
ax.axes.xaxis.set_visible(False)
ax.axes.yaxis.set_visible(False)
plt.figure(figsize=(20, 10))
plt.subplot(2, 2, 1)
plt.imshow(src)
plt.title('Image')
hide(plt)
plt.subplot(2, 2, 2)
plt.imshow(tar)
plt.title('SegmentationClass')
hide(plt)
plt.subplot(2, 2, 3)
plt.imshow(
Image.open('./dataset/output/SegmentationClassVisualization/' +
data + '.jpg'))
plt.title('SegmentationVisuallization')
hide(plt)
plt.subplot(2, 2, 4)
plt.imshow(
cv2.resize(np.float32(Image.fromarray(viz[:, width:, :])) /
255, (w, h),
interpolation=cv2.INTER_LINEAR))
plt.title('Prediction')
hide(plt)
if not os.path.isdir('result'):
os.makedirs('result')
plt.savefig('result/%s.png' % (data))
if tar is not None:
loss = self.criterion(output, target)
test_loss += loss.item()
#tbar.set_description('Test loss: %.3f' % (test_loss))
pred = output.data.cpu().numpy()
target = target.cpu().numpy()
pred = np.argmax(pred, axis=1)
# Add batch sample into evaluator
self.evaluator.add_batch(target[:, 145:815, :], pred[:,
145:815, :])
# Fast test during the training
Acc = self.evaluator.Pixel_Accuracy()
Acc_class = self.evaluator.Pixel_Accuracy_Class()
mIoU = self.evaluator.Mean_Intersection_over_Union()
FWIoU = self.evaluator.Frequency_Weighted_Intersection_over_Union()
print('Validation:')
#print('[Epoch: %d, numImages: %5d]' % (epoch, self.args.batch_size + image.data.shape[0]))
print("Acc:{}, Acc_class:{}, mIoU:{}, fwIoU: {}".format(
Acc, Acc_class, mIoU, FWIoU))
print('Loss: %.3f' % test_loss)
plot_confusion_matrix(self.evaluator.confusion_matrix, ClassName)
plt.savefig('result/%s_CM.png' % (data))