def segmentation_mymethod(train_data,
train_labels,
test_data,
test_labels,
task='brain',
method='nearest neighbour',
testDice=True):
# segments the image based on your own method!
# Input:
# train_data_matrix num_pixels x num_features x num_subjects matrix of
# features
# train_labels_matrix num_pixels x num_subjects matrix of labels
# test_data num_pixels x num_features test data
# task String corresponding to the segmentation task: either 'brain' or 'tissue'
# Output:
# predicted_labels Predicted labels for the test slice
#------------------------------------------------------------------#
#TODO: Implement your method here
if method == 'kmeans':
predicted_labels = seg.kmeans_clustering(test_data, K=4)
elif method == 'nearest neighbour':
predicted_labels = seg.nn_classifier(train_data, train_labels,
test_data)
elif method == 'knn':
predicted_labels = seg.knn_classifier(train_data,
train_labels,
test_data,
k=4)
elif method == 'atlas':
predicted_labels = seg.segmentation_atlas(train_data, train_labels,
test_data)
predicted_labels = predicted_labels.astype(bool)
test_labels = test_labels.astype(bool)
err = util.classification_error(test_labels, predicted_labels)
print('Error:\n{}'.format(err))
if testDice:
dice = util.dice_multiclass(test_labels, predicted_labels)
print('Dice coefficient:\n{}'.format(dice))
#------------------------------------------------------------------#
return predicted_labels
def segmentation_demo():
# Data name specification
train_subject = 1
test_subject = 2
train_slice = 1
test_slice = 1
task = 'tissue'
# Load data
train_data, train_labels, train_feature_labels = util.create_dataset(
train_subject, train_slice, task)
test_data, test_labels, test_feature_labels = util.create_dataset(
test_subject, test_slice, task)
# find the predicted labels (here: the train_labels)
predicted_labels = seg.segmentation_atlas(None, train_labels, None)
# Calculate the error and dice score of these predicted labels in comparison to test labels
err = util.classification_error(test_labels, predicted_labels)
dice = util.dice_overlap(test_labels, predicted_labels)
# Display results
true_mask = test_labels.reshape(240, 240)
predicted_mask = predicted_labels.reshape(240, 240)
fig = plt.figure(figsize=(8, 8))
ax1 = fig.add_subplot(111)
ax1.imshow(true_mask, 'gray')
ax1.imshow(predicted_mask, 'viridis', alpha=0.5)
print('Subject {}, slice {}.\nErr {}, dice {}'.format(
test_subject, test_slice, err, dice))
# COMPARE METHODS
num_images = 5
num_methods = 3
im_size = [240, 240]
# make space for error and dice data
all_errors = np.empty([num_images, num_methods])
all_errors[:] = np.nan
all_dice = np.empty([num_images, num_methods])
all_dice[:] = np.nan
# data name specification
all_subjects = np.arange(num_images)
train_slice = 1
task = 'tissue'
# make space for data
all_data_matrix = np.empty(
[train_data.shape[0], train_data.shape[1], num_images])
# all_labels_matrix = np.empty([train_labels.size, num_images], dtype=bool)
all_labels_matrix = np.empty([train_labels.size, num_images])
# Load datasets once
print('Loading data for ' + str(num_images) + ' subjects...')
for i in all_subjects:
sub = i + 1
train_data, train_labels, train_feature_labels = util.create_dataset(
sub, train_slice, task)
all_data_matrix[:, :, i] = train_data
all_labels_matrix[:, i] = train_labels.flatten()
print('Finished loading data.\nStarting segmentation...')
# Go through each subject, taking i-th subject as the test
for i in np.arange(num_images):
sub = i + 1
# Define training subjects as all, except the test subject
train_subjects = all_subjects.copy()
train_subjects = np.delete(train_subjects, i)
# Obtain data about the chosen amount of subjects
train_data_matrix = all_data_matrix[:, :, train_subjects]
train_labels_matrix = all_labels_matrix[:, train_subjects]
test_data = all_data_matrix[:, :, i]
test_labels = all_labels_matrix[:, i]
test_shape_1 = test_labels.reshape(im_size[0], im_size[1])
fig = plt.figure(figsize=(15, 5))
# Get predicted labels from atlas method
predicted_labels = seg.segmentation_combined_atlas(train_labels_matrix)
all_errors[i, 0] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 0] = util.dice_overlap(test_labels, predicted_labels)
# Plot atlas method
predicted_mask_1 = predicted_labels.reshape(im_size[0], im_size[1])
ax1 = fig.add_subplot(131)
ax1.imshow(test_shape_1, 'gray')
ax1.imshow(predicted_mask_1, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 0],
all_dice[i, 0])
ax1.set_xlabel(text_str)
ax1.set_title('Subject {}: Combined atlas'.format(sub))
# Get predicted labels from kNN method
predicted_labels = seg.segmentation_combined_knn(
train_data_matrix, train_labels_matrix, test_data)
all_errors[i, 1] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 1] = util.dice_overlap(test_labels, predicted_labels)
# Plot kNN method
predicted_mask_2 = predicted_labels.reshape(im_size[0], im_size[1])
ax2 = fig.add_subplot(132)
ax2.imshow(test_shape_1, 'gray')
ax2.imshow(predicted_mask_2, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 1],
all_dice[i, 1])
ax2.set_xlabel(text_str)
ax2.set_title('Subject {}: Combined k-NN'.format(sub))
# Get predicted labels from my own method
predicted_labels = segmentation_mymethod(train_data_matrix,
train_labels_matrix,
test_data, task)
print(predicted_labels.shape)
print(np.unique(predicted_labels))
all_errors[i, 2] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 2] = util.dice_overlap(test_labels, predicted_labels)
# Plot my own method
predicted_mask_3 = predicted_labels.reshape(im_size[0], im_size[1])
ax3 = fig.add_subplot(133)
ax3.imshow(test_shape_1, 'gray')
ax3.imshow(predicted_mask_3, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax3.set_xlabel(text_str)
ax3.set_title('Subject {}: My method'.format(sub))
def segmentation_demo():
train_subject = 1
test_subject = 2
train_slice = 1
test_slice = 1
task = 'brain'
#Load data
train_data, train_labels, train_feature_labels = util.create_dataset(
train_subject, train_slice, task)
test_data, test_labels, test_feature_labels = util.create_dataset(
test_subject, test_slice, task)
predicted_labels = seg.segmentation_atlas(None, train_labels, None)
err = util.classification_error(test_labels, predicted_labels)
dice = util.dice_overlap(test_labels, predicted_labels)
#Display results
true_mask = test_labels.reshape(240, 240)
predicted_mask = predicted_labels.reshape(240, 240)
# fig = plt.figure(figsize=(8,8))
# ax1 = fig.add_subplot(111)
# ax1.imshow(true_mask, 'gray')
# ax1.imshow(predicted_mask, 'viridis', alpha=0.5)
# print('Subject {}, slice {}.\nErr {}, dice {}'.format(test_subject, test_slice, err, dice))
## Compare methods
num_images = 5
num_methods = 3
im_size = [240, 240]
all_errors = np.empty([num_images, num_methods])
all_errors[:] = np.nan
all_dice = np.empty([num_images, num_methods])
all_dice[:] = np.nan
all_subjects = np.arange(num_images)
train_slice = 1
task = 'brain'
all_data_matrix = np.empty(
[train_data.shape[0], train_data.shape[1], num_images])
all_labels_matrix = np.empty([train_labels.size, num_images], dtype=bool)
#Load datasets once
print('Loading data for ' + str(num_images) + ' subjects...')
for i in all_subjects:
sub = i + 1
train_data, train_labels, train_feature_labels = util.create_dataset(
sub, train_slice, task)
all_data_matrix[:, :, i] = train_data
all_labels_matrix[:, i] = train_labels.flatten()
print('Finished loading data.\nStarting segmentation...')
#Go through each subject, taking i-th subject as the test
for i in np.arange(num_images):
sub = i + 1
#Define training subjects as all, except the test subject
train_subjects = all_subjects.copy()
train_subjects = np.delete(train_subjects, i)
train_data_matrix = all_data_matrix[:, :, train_subjects]
train_labels_matrix = all_labels_matrix[:, train_subjects]
test_data = all_data_matrix[:, :, i]
test_labels = all_labels_matrix[:, i]
test_shape_1 = test_labels.reshape(im_size[0], im_size[1])
fig = plt.figure(figsize=(15, 5))
predicted_labels = seg.segmentation_combined_atlas(train_labels_matrix)
all_errors[i, 0] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 0] = util.dice_overlap(test_labels, predicted_labels)
predicted_mask_1 = predicted_labels.reshape(im_size[0], im_size[1])
ax1 = fig.add_subplot(131)
ax1.imshow(test_shape_1, 'gray')
ax1.imshow(predicted_mask_1, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 0],
all_dice[i, 0])
ax1.set_xlabel(text_str)
ax1.set_title('Subject {}: Combined atlas'.format(sub))
predicted_labels = seg.segmentation_combined_knn(
train_data_matrix, train_labels_matrix, test_data)
all_errors[i, 1] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 1] = util.dice_overlap(test_labels, predicted_labels)
predicted_mask_2 = predicted_labels.reshape(im_size[0], im_size[1])
ax2 = fig.add_subplot(132)
ax2.imshow(test_shape_1, 'gray')
ax2.imshow(predicted_mask_2, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 1],
all_dice[i, 1])
ax2.set_xlabel(text_str)
ax2.set_title('Subject {}: Combined k-NN'.format(sub))
predicted_labels = segmentation_mymethod(train_data_matrix,
train_labels_matrix,
test_data, task)
all_errors[i, 2] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 2] = util.dice_overlap(test_labels, predicted_labels)
predicted_mask_3 = predicted_labels.reshape(im_size[0], im_size[1])
ax3 = fig.add_subplot(133)
ax3.imshow(test_shape_1, 'gray')
ax3.imshow(predicted_mask_3, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax3.set_xlabel(text_str)
ax3.set_title('Subject {}: My method'.format(sub))
def segmentation_demo():
#only SECTION 2 is needed for what we want to do
train_subject = 1
test_subject = 2
train_slice = 1
test_slice = 1
task = 'tissue'
#SECTION 1 (this seciton has nothing to do with SECTION 2)
#Load data from a train and testsubject
train_data, train_labels, train_feature_labels = util.create_dataset(
train_subject, train_slice, task)
test_data, test_labels, test_feature_labels = util.create_dataset(
test_subject, test_slice, task)
util.scatter_data(train_data, train_labels, 0, 6)
util.scatter_data(test_data, test_labels, 0, 6)
predicted_labels = seg.segmentation_atlas(None, train_labels, None)
err = util.classification_error(test_labels, predicted_labels)
dice = util.dice_overlap(test_labels, predicted_labels)
#Display results
true_mask = test_labels.reshape(240, 240)
predicted_mask = predicted_labels.reshape(240, 240)
fig = plt.figure(figsize=(8, 8))
ax1 = fig.add_subplot(111)
ax1.imshow(true_mask, 'gray')
ax1.imshow(predicted_mask, 'viridis', alpha=0.5)
print('Subject {}, slice {}.\nErr {}, dice {}'.format(
test_subject, test_slice, err, dice))
## SECTION 2:Compare methods
num_images = 5
num_methods = 3
im_size = [240, 240]
all_errors = np.empty([num_images, num_methods])
all_errors[:] = np.nan
all_dice = np.empty([num_images, num_methods])
all_dice[:] = np.nan
all_subjects = np.arange(5) #list of all subjects [0, 1, 2, 3, 4]
train_slice = 2
task = 'tissue'
all_data_matrix = np.empty(
[train_data.shape[0], train_data.shape[1], num_images])
all_labels_matrix = np.empty([train_labels.size, num_images])
#Load datasets once
print('Loading data for ' + str(num_images) + ' subjects...')
for i in all_subjects:
sub = i + 1
train_data, train_labels, train_feature_labels = util.create_dataset(
sub, train_slice, task)
all_data_matrix[:, :, i] = train_data
all_labels_matrix[:, i] = train_labels.flatten()
print('Finished loading data.\nStarting segmentation...')
#Go through each subject, taking i-th subject as the test
for i in all_subjects:
sub = i + 1
#Define training subjects as all, except the test subject
train_subjects = all_subjects.copy()
train_subjects = np.delete(train_subjects, i)
train_data_matrix = all_data_matrix[:, :, train_subjects]
train_labels_matrix = all_labels_matrix[:, train_subjects]
test_data = all_data_matrix[:, :, i]
test_labels = all_labels_matrix[:, i]
test_shape_1 = test_labels.reshape(im_size[0], im_size[1])
fig = plt.figure(figsize=(15, 5))
predicted_labels, predicted_labels2 = seg.segmentation_combined_atlas(
train_labels_matrix)
all_errors[i, 0] = util.classification_error(test_labels,
predicted_labels2)
all_dice[i, 0] = util.dice_multiclass(test_labels, predicted_labels2)
predicted_mask_1 = predicted_labels2.reshape(im_size[0], im_size[1])
ax1 = fig.add_subplot(131)
ax1.imshow(test_shape_1, 'gray')
ax1.imshow(predicted_mask_1, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 0],
all_dice[i, 0])
ax1.set_xlabel(text_str)
ax1.set_title('Subject {}: Combined atlas'.format(sub))
predicted_labels, predicted_labels2 = seg.segmentation_combined_knn(
train_data_matrix, train_labels_matrix, test_data)
all_errors[i, 1] = util.classification_error(test_labels,
predicted_labels2)
all_dice[i, 1] = util.dice_multiclass(test_labels, predicted_labels2)
predicted_mask_2 = predicted_labels2.reshape(im_size[0], im_size[1])
ax2 = fig.add_subplot(132)
ax2.imshow(test_shape_1, 'gray')
ax2.imshow(predicted_mask_2, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 1],
all_dice[i, 1])
ax2.set_xlabel(text_str)
ax2.set_title('Subject {}: Combined k-NN'.format(sub))
#OUR METHOD
#predict the labels using our method
predicted_labels_mymethod = segmentation_mymethod(train_data_matrix,
train_labels_matrix,
test_data,
num_iter=100,
mu=0.1)
#determine error and dice (multiclass, since there are more classes)
all_errors[i,
2] = util.classification_error(test_labels,
predicted_labels_mymethod)
all_dice[i, 2] = util.dice_multiclass(test_labels,
predicted_labels_mymethod)
#reshape the predicted labels in order to plot the results
predicted_mask_3 = predicted_labels_mymethod.reshape(
im_size[0], im_size[1])
#plot the predicted image over the real image
plt.imshow(predicted_mask_3, 'viridis')
ax3 = fig.add_subplot(133)
ax3.imshow(test_shape_1, 'gray')
ax3.imshow(predicted_mask_3, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax3.set_xlabel(text_str)
ax3.set_title('Subject {}: My method'.format(sub))
#save the figure after every loop (3 subimages/plots)
fig.savefig("Results for test subject {}".format(sub), )
def segmentation_demo():
# Data name specification
train_subject = 1
test_subject = 2
train_slice = 1
test_slice = 1
task = 'tissue'
# Load data
train_data, train_labels, train_feature_labels = util.create_dataset(
train_subject, train_slice, task)
test_data, test_labels, test_feature_labels = util.create_dataset(
test_subject, test_slice, task)
# Normalize and feed data through X_pca
train_norm, _ = seg.normalize_data(train_data)
Xpca, v, w, fraction_variance, ix = seg.mypca(train_norm)
relevant_feature = int(np.sum(fraction_variance < 0.95)) + 1
train_norm_ord = train_norm[:, ix]
train_norm = train_norm_ord[:, :relevant_feature]
# find the predicted labels (here: the train_labels)
predicted_labels = seg.segmentation_atlas(None, train_labels, None)
# Calculate the error and dice score of these predicted labels in comparison to test labels
err = util.classification_error(test_labels, predicted_labels)
dice = util.dice_multiclass(test_labels, predicted_labels)
# Display results
true_mask = test_labels.reshape(240, 240)
predicted_mask = predicted_labels.reshape(240, 240)
fig = plt.figure(figsize=(8, 8))
ax1 = fig.add_subplot(111)
ax1.imshow(true_mask, 'gray')
ax1.imshow(predicted_mask, 'viridis', alpha=0.5)
print('Subject {}, slice {}.\nErr {}, dice {}'.format(
test_subject, test_slice, err, dice))
# COMPARE METHODS
num_images = 5
num_methods = 3
im_size = [240, 240]
# make space for error and dice data
all_errors = np.empty([num_images, num_methods])
all_errors[:] = np.nan
all_dice = np.empty([num_images, num_methods])
all_dice[:] = np.nan
# data name specification
all_subjects = np.arange(num_images)
train_slice = 1
task = 'tissue'
# make space for data
all_data_matrix = np.empty(
[train_norm.shape[0], train_norm.shape[1], num_images])
all_labels_matrix = np.empty([train_labels.size, num_images])
all_data_matrix_kmeans = np.empty(
[train_norm.shape[0], train_norm.shape[1], num_images])
all_labels_matrix_kmeans = np.empty([train_labels.size, num_images])
# Load datasets once
print('Loading data for ' + str(num_images) + ' subjects...')
for i in all_subjects:
sub = i + 1
train_data, train_labels, train_feature_labels = util.create_dataset(
sub, train_slice, task)
train_norm, _ = seg.normalize_data(train_data)
Xpca, v, w, fraction_variance, ix = seg.mypca(train_norm)
relevant_labels = int(np.sum(fraction_variance < 0.95)) + 1
train_norm_ord = train_norm[:, ix]
train_norm = train_norm_ord[:, :relevant_labels]
all_data_matrix[:, :, i] = train_norm
all_labels_matrix[:, i] = train_labels.flatten()
# Load datasets for kmeans
print('Loading data for ' + str(num_images) + ' subjects...')
for i in all_subjects:
sub = i + 1
train_data_kmeans, train_labels_kmeans, train_feature_labels_kmeans = create_dataset(
sub, train_slice, task)
train_norm_kmeans, _ = seg.normalize_data(train_data_kmeans)
all_data_matrix_kmeans[:, :, i] = train_norm_kmeans
all_labels_matrix_kmeans[:, i] = train_labels_kmeans.flatten()
print('Finished loading data.\nStarting segmentation...')
# Go through each subject, taking i-th subject as the test
for i in np.arange(num_images):
sub = i + 1
# Define training subjects as all, except the test subject
train_subjects = all_subjects.copy()
train_subjects = np.delete(train_subjects, i)
# Obtain data about the chosen amount of subjects
train_data_matrix = all_data_matrix[:, :, train_subjects]
train_labels_matrix = all_labels_matrix[:, train_subjects]
test_data = all_data_matrix[:, :, i]
test_labels = all_labels_matrix[:, i]
test_shape_1 = test_labels.reshape(im_size[0], im_size[1])
fig = plt.figure(figsize=(15, 5))
# Get predicted labels from atlas method
predicted_labels = seg.segmentation_combined_atlas(train_labels_matrix)
all_errors[i, 0] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 0] = util.dice_multiclass(test_labels, predicted_labels)
# Plot atlas method
predicted_mask_1 = predicted_labels.reshape(im_size[0], im_size[1])
ax1 = fig.add_subplot(151)
ax1.imshow(test_shape_1, 'gray')
ax1.imshow(predicted_mask_1, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 0],
all_dice[i, 0])
ax1.set_xlabel(text_str)
ax1.set_title('Subject {}: Combined atlas'.format(sub))
# Get predicted labels from kNN method
predicted_labels = seg.segmentation_combined_knn(train_data_matrix,
train_labels_matrix,
test_data,
k=10)
all_errors[i, 1] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 1] = util.dice_multiclass(test_labels, predicted_labels)
# Plot kNN method
predicted_mask_2 = predicted_labels.reshape(im_size[0], im_size[1])
ax2 = fig.add_subplot(152)
ax2.imshow(test_shape_1, 'gray')
ax2.imshow(predicted_mask_2, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 1],
all_dice[i, 1])
ax2.set_xlabel(text_str)
ax2.set_title('Subject {}: Combined k-NN'.format(sub))
# Get predicted labels from my own method
# all_data_matrix_bnb = np.empty([train_norm.shape[0], train_norm.shape[1], num_images])
# all_labels_matrix_bnb = np.empty([train_labels.size, num_images])
# for ii in all_subjects:
# sub = i + 1
# task = 'brain'
# train_data_bnb, train_labels_bnb, train_feature_labels_bnb = util.create_dataset(sub, train_slice, task)
# train_norm_bnb, _ = seg.normalize_data(train_data_bnb)
# Xpca, v, w, fraction_variance, ix = seg.mypca(train_norm_bnb)
# relevant_labels_bnb = int(np.sum(fraction_variance < 0.95)) + 1
# train_norm_ord_bnb = train_norm_bnb[:, ix]
# train_norm_bnb = train_norm_ord_bnb[:, :relevant_labels_bnb]
# all_data_matrix_bnb[:, :, ii] = train_norm_bnb
# all_labels_matrix_bnb[:, ii] = train_labels_bnb.flatten()
#
# qw, we, er = all_data_matrix.shape
# for iii in np.arange(qw):
# for j in np.arange(er):
# if all_labels_matrix_bnb[iii, j] == 0:
# for k in np.arange(we):
# all_data_matrix[iii, k, j] = 0
# train_data_matrix = all_data_matrix[:, :, train_subjects]
# test_data = all_data_matrix[:, :, i]
train_data_matrix_kmeans = all_data_matrix_kmeans[:, :, train_subjects]
train_labels_matrix_kmeans = all_labels_matrix[:, train_subjects]
test_data_kmeans = all_data_matrix_kmeans[:, :, i]
predicted_labels = segmentation_mymethod(train_data_matrix_kmeans,
train_labels_matrix_kmeans,
test_data_kmeans, task)
all_errors[i, 2] = util.classification_error(test_labels,
predicted_labels)
all_dice[i, 2] = util.dice_multiclass(test_labels, predicted_labels)
# Plot my own method
predicted_mask_3 = predicted_labels.reshape(im_size[0], im_size[1])
ax3 = fig.add_subplot(153)
ax3.imshow(test_shape_1, 'gray')
ax3.imshow(predicted_mask_3, 'viridis', alpha=0.5)
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax3.set_xlabel(text_str)
ax3.set_title('Subject {}: My method'.format(sub))
ax4 = fig.add_subplot(154)
ax4.imshow(predicted_mask_3, 'viridis')
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax4.set_xlabel(text_str)
ax4.set_title('Subject {}: My method'.format(sub))
ax5 = fig.add_subplot(155)
ax5.imshow(test_shape_1, 'gray')
text_str = 'Err {:.4f}, dice {:.4f}'.format(all_errors[i, 2],
all_dice[i, 2])
ax5.set_xlabel(text_str)
ax5.set_title('Subject {}: My method'.format(sub))