def __getitem__(self, i):
# Test
if self.mode == 'test':
# read data
left = cv2.imread(self.left_fps[i])
left = cv2.cvtColor(left, cv2.COLOR_BGR2RGB)
left = utils.normalize_img(left)
if self.augmentation:
left = self.augment_img(left)
return left, None
# Train
else:
# read data
left = cv2.imread(self.left_fps[i])
left = cv2.cvtColor(left, cv2.COLOR_BGR2RGB)
right = cv2.imread(self.right_fps[i])
right = cv2.cvtColor(right, cv2.COLOR_BGR2RGB)
left = utils.normalize_img(left)
right = utils.normalize_img(right)
# apply augmentations
if self.augmentation:
left, right = self.augment_pair(left, right)
return left, right
def inference(model_path, img_id):
model = load_model(model_path)
img = read_img(img_id)
normalized = normalize_img(img)
repeats_x = math.ceil(img.shape[0] / INPUT_SIZE)
repeats_y = math.ceil(img.shape[1] / INPUT_SIZE)
input_shape = (1, INPUT_SIZE, INPUT_SIZE, img.shape[2])
result = np.zeros((img.shape[0], img.shape[1], len(CLASSES)))
for idx_x in range(repeats_x):
for idx_y in range(repeats_y):
s_x = idx_x * INPUT_SIZE
s_y = idx_y * INPUT_SIZE
patch = normalized[s_x:s_x + INPUT_SIZE, s_y:s_y + INPUT_SIZE]
input_img = np.zeros(input_shape)
input_img[0, 0:patch.shape[0], 0:patch.shape[1]] = patch
pred = model.predict(input_img)
result[s_x:s_x + patch.shape[0], s_y:s_y + patch.shape[1]] \
= pred[0, 0:patch.shape[0], 0:patch.shape[1]]
result = np.where(result > INFERENCE_THRESHOLD, 1, 0)
return result
def create_dataset(config):
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = normalize_img(x_train)
x_test = normalize_img(x_test)
y_train_oh = scalar_to_onehot(y_train, 10)
y_test_oh = scalar_to_onehot(y_test, 10)
image_train = tf.data.Dataset.from_tensor_slices(x_train[..., np.newaxis])
label_train = tf.data.Dataset.from_tensor_slices(y_train_oh)
image_test = tf.data.Dataset.from_tensor_slices(x_test[..., np.newaxis])
label_test = tf.data.Dataset.from_tensor_slices(y_test_oh)
def tf_random_rotate(img_tensor):
im_shape = img_tensor.shape
random_rotate = partial(_random_rotate,
max_angle=config.data.test.rotation.max_angle)
[
image_tensor,
] = tf.py_function(random_rotate, [img_tensor], [tf.float32])
image_tensor.set_shape(im_shape)
return image_tensor
if config.data.augmentation:
gaussian_noise_aug = partial(_gaussain_noise, var=0.5)
image_train = image_train.map(tf_random_rotate)
image_train = image_train.map(gaussian_noise_aug)
if config.data.test.rotation.apply:
image_test = image_test.map(tf_random_rotate)
elif config.data.test.noise.apply:
gaussian_noise = partial(_gaussain_noise,
var=config.data.test.noise.variance)
image_test = image_test.map(gaussian_noise)
train_dataset = tf.data.Dataset.zip((image_train, label_train))
test_dataset = tf.data.Dataset.zip((image_test, label_test))
train_dataset = train_dataset.batch(config.trainer.batch_size)
test_dataset = test_dataset.batch(config.trainer.batch_size)
return train_dataset, test_dataset
def get_train_data(train_img_ids, df, gs):
x_train_list = []
y_train_list = []
for img_id in train_img_ids:
img = read_img(img_id)
x_train = normalize_img(img)
y_train = generate_mask(x_train, img_id, df, gs)
x_train_list.append(x_train)
y_train_list.append(y_train)
return (x_train_list, y_train_list)
def run(flags_obj):
"""Run nin model"""
if flags_obj.dataset == 'cifar-10':
nin = define_nin(input_shape=(32, 32, 3))
(x_train, y_train), (x_test,
y_test) = tf.keras.datasets.cifar10.load_data()
elif flags.FLAGS.dataset == 'mnist':
nin = define_nin(input_shape=(28, 28, 1))
(x_train, y_train), (x_test,
y_test) = tf.keras.datasets.mnist.load_data()
else:
raise Exception('Dataset must be "cifar-10" or "mnist"')
train_size = x_train.shape[0]
test_size = x_test.shape[0]
x_train = normalize_img(x_train.astype('float32'))
x_train = normalize_img(x_train.astype('float32'))
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))
test_dataset = tf.data.Dataset.from_tensor_slices((x_test, y_test))
train_dataset = train_dataset.shuffle(10000).batch(
flags_obj.batch_size).repeat()
test_dataset = test_dataset.batch(flags_obj.batch_size).repeat()
nin.compile(
optimizer=tf.keras.optimizers.Adam(lr=flags_obj.learning_rate),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
log_dir = './logs/fit' + datetime.datetime.now().strftime('%Y%m%d-%H%M%S')
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
nin.fit(train_dataset,
epochs=flags_obj.epoch,
steps_per_epoch=train_size // flags_obj.batch_size,
validation_data=test_dataset,
validation_steps=test_size // flags_obj.batch_size,
callbacks=[tensorboard_callback])
def get_data(path, center_cropping=True, normalization=True):
img_path_list = [os.path.join(path, file) for file in os.listdir(path)]
img_data = []
for img_path in img_path_list:
if center_cropping:
img = center_crop_img(cv2.imread(img_path), IMG_SIZE)
else:
img = cv2.resize(cv2.imread(img_path), (IMG_SIZE, IMG_SIZE),
interpolation=cv2.INTER_AREA)
if img is None:
continue
if normalization:
img = normalize_img(img)
img_data.append(img.reshape(-1))
return np.array(img_data)
def main_step(model, dl, device):
confusion_matrix = torch.zeros(3, 3, dtype=torch.int32).to(device)
for v, b in enumerate(dl):
X, y = b
if device == torch.device('cuda'):
X, y = X.to(device), y.to(device)
image = [X.squeeze_(0)] # remove the batch dimension
X = utils.normalize_img(image[0], device=device)
_, pred_scores, _ = model(image)
# predicted class scores
confusion_matrix[torch.nonzero(y.squeeze_(0) > 0).item(),
pred_scores[0]['final_scores'].argmax().item()] += 1
print("------------------------------------------")
print("Confusion Matrix for 3-class problem, a total of {0:d} images:".
format(len(dl)))
print("0: Control, 1: Normal Pneumonia, 2: COVID")
print(confusion_matrix)
print("------------------------------------------")
# confusion matrix
cm = confusion_matrix.float()
cm[0, :].div_(cm[0, :].sum())
cm[1, :].div_(cm[1, :].sum())
cm[2, :].div_(cm[2, :].sum())
print("------------------------------------------")
print("Class Sensitivity:")
print(cm)
print("------------------------------------------")
print('Overall accuracy:')
oa = confusion_matrix.diag().float().sum().div(confusion_matrix.sum())
print(oa)
cm_spec = confusion_matrix.float()
cm_spec[:, 0].div_(cm_spec[:, 0].sum())
cm_spec[:, 1].div_(cm_spec[:, 1].sum())
cm_spec[:, 2].div_(cm_spec[:, 2].sum())
# Class weights: 0, 1, 2
cw = torch.tensor([0.45, 0.35, 0.2], dtype=torch.float).to(device)
print("------------------------------------------")
print('F1 score:')
f1_score = 2 * cm.diag().mul(
cm_spec.diag()).div(cm.diag() + cm_spec.diag()).dot(cw).item()
print(f1_score)
# Confusion matrix, class sensitivity, overall accuracy and F1 score
return confusion_matrix, cm, oa, f1_score
def __next__(self):
with tf.device('/gpu:0'):
batch_image = np.zeros(
(cfg.BATCHSIZE, cfg.INPUTHEIGHT, cfg.INPUTWIDTH, 1))
batch_label = np.zeros(
(cfg.BATCHSIZE, cfg.INPUTHEIGHT // cfg.DOWNSCALE,
cfg.INPUTWIDTH // cfg.DOWNSCALE, 5))
if self.batch_done < self.batch_num:
cnt = 0
while cnt < cfg.BATCHSIZE:
prefix = self.file_list[self.batch_done * cfg.BATCHSIZE +
cnt]
curr_img = utils.normalize_img(
cv2.resize(cv2.imread(
os.path.join(cfg.DATAPATH, prefix + '.png'),
cv2.IMREAD_GRAYSCALE),
(cfg.INPUTWIDTH, cfg.INPUTHEIGHT),
interpolation=cv2.INTER_LANCZOS4))
curr_img = np.expand_dims(curr_img, -1)
batch_image[cnt] = curr_img
if prefix in self.json_list:
curr_jsn = json.load(
open(os.path.join(cfg.DATAPATH, prefix + '.json')))
curr_lbl = utils.jbox_2_label(
(cfg.INPUTWIDTH // cfg.DOWNSCALE,
cfg.INPUTHEIGHT // cfg.DOWNSCALE), curr_jsn,
self.anchor)
batch_label[cnt] = np.swapaxes(curr_lbl, 0, 1)
else:
batch_label[cnt] = np.zeros(
(cfg.INPUTHEIGHT // cfg.DOWNSCALE,
cfg.INPUTWIDTH // cfg.DOWNSCALE, 5))
cnt += 1
self.batch_done += 1
return batch_image, batch_label
else:
self.batch_done = 0
random.shuffle(self.file_list)
raise StopIteration
k = 0
IMG_HEIGHT = 256
IMG_WIDTH = 256
list_shapes = []
print('Reading train and val..')
for i in range(total_train):
print(i)
img_path = train_fold + '/' + train_ids[i] + '/images/' + train_ids[i] + '.png'
img = cv2.imread(img_path,1) #256 x 256 x 3?
# pdb.set_trace()
mask = create_mask(train_fold + '/' + train_ids[i], shape = (img.shape[0], img.shape[1],1))
print(mask.shape)
old_shape = img.shape
img = np.dstack([normalize_img(img[:,:,q]) for q in range(3) ])
if (img.shape != (IMG_HEIGHT, IMG_WIDTH, 3)): # resize it
img = resize_img(img, shape = (IMG_HEIGHT, IMG_WIDTH, 3))
mask = resize_img(mask, shape = (IMG_HEIGHT, IMG_WIDTH,1))
# if img.shape in list_shapes:
# pass
# else:
# list_shapes.append(img.shape)
# pdb.set_trace()
# cv2.imshow('image',img)
# cv2.waitKey(0)
# cv2.imshow('mask',mask)
# cv2.waitKey(0)
# pdb.set_trace()
if i<n_train:
def step(stage, e, max_loss):
epoch_loss_classification_indep = 0
epoch_loss_segmentation = 0
# repeat the total number of minibatches in the classifer dataset
total_batches = int(len(dataloader_classification_covid) / batch_size)
for r in range(total_batches):
total_classifier_loss = 0
optimizer.zero_grad()
rand_im_segment_ind = torch.randint(len(dataloader_covid),
size=(1, )).item()
b = dataloader_covid.dataset[rand_im_segment_ind]
X, y = b
if device == torch.device('cuda'):
X, y['labels'], y['boxes'], y['masks'] = X.to(
device), y['labels'].to(device), y['boxes'].to(
device), y['masks'].to(device)
X = utils.normalize_img(X, device=device)
images = [X]
targets = []
lab = {}
lab['boxes'] = y['boxes']
lab['labels'] = y['labels']
lab['masks'] = y['masks']
# train segmentation and classification or only classification?
targets.append(lab)
if len(targets[0]['boxes']):
# pass
attn_classifier.train()
attn_classifier.attn_layer.eval()
loss, _, _ = attn_classifier(images, targets)
total_seg_loss = 0
for k in loss.keys():
total_seg_loss += loss[k]
# exclude all classification box and mask layers in RoI
for _n, _p in attn_classifier.named_parameters():
if 's2' in _n or 'attn' in _n:
_p.grad = None
# only if better
epoch_loss_segmentation += total_seg_loss.item()
if total_seg_loss.item() < max_loss:
utils.copy_weights(attn_classifier)
total_seg_loss.backward()
max_loss = total_seg_loss.item()
optimizer.step()
optimizer.zero_grad()
torch.cuda.empty_cache()
else:
total_seg_loss = 0
# set all to eval, then switch back parameters upgrade
# compute the image loss
rand_im_class_ind = torch.randint(len(dataloader_classification_covid),
size=(1, )).item()
bt = dataloader_classification_covid.dataset[rand_im_class_ind]
attn_classifier.eval()
attn_classifier.backbone.train()
attn_classifier.attn_layer.train()
X_cl, y_cl = bt
# print('targ', y_cl)
if device == torch.device('cuda'):
X_cl, y_cl = X_cl.to(device), y_cl.to(device)
X_cl = X_cl.squeeze(0)
X_cl = utils.normalize_img(X_cl, device=device)
image = [X_cl] # remove the batch dimension
lab = dict(
image_label=y_cl.squeeze(0)
) # This is not used within thge model, keep it for the correct operations
_, predict_score, _ = attn_classifier(image, targets=lab)
# append the feature vector and its class
correct_class_str = lab['image_label']
#print('pred', predict_score)
classifier_loss = F.binary_cross_entropy_with_logits(
predict_score[0]['final_scores'].squeeze(0), y_cl)
total_classifier_loss += classifier_loss
total_classifier_loss.backward()
for _n, _p in attn_classifier.named_parameters():
if 'backbone' not in _n and 'attn' not in _n:
_p.grad = None
optimizer.step()
optimizer.zero_grad()
torch.cuda.empty_cache()
epoch_loss_classification_indep += total_classifier_loss
# save model?
if not e % save_every:
attn_classifier.eval()
state = {
'epoch': str(e),
'model_weights': attn_classifier.state_dict(),
'optimizer_state': optimizer.state_dict(),
'lrate': 1e-5
}
torch.save(
state,
os.path.join("saved_models",
bbn + "_16_4class_lstm_attn_ckpt_" + str(e) + ".pth"))
attn_classifier.train()
epoch_loss_classification_indep = epoch_loss_classification_indep / total_batches
epoch_loss_segmentation = epoch_loss_segmentation / len(dataloader_covid)
# print(epoch_loss)
# print('total!', tots_pos, tots_neg)
return epoch_loss_classification_indep, epoch_loss_segmentation, max_loss
attn_classifier.backbone.train()
attn_classifier.attn_layer.train()
if device == torch.device('cuda'):
attn_classifier = attn_classifier.to(device)
start = time.time()
batch_size = 1
confusion_matrix = torch.zeros(3, 3, dtype=torch.int32).to(device)
for id, b in enumerate(dataloader_classification_covid):
X, y = b
#print(X)
if device == torch.device('cuda'):
X, y = X.to(device), y.to(device)
image = [X.squeeze_(0)] #remove the batch dimension
utils.normalize_img(image[0], device=device)
_, predict_score, _ = attn_classifier(image, targets=None)
pred_class = predict_score[0]['final_scores'].argmax()
confusion_matrix[torch.nonzero(y.view(-1) > 0).item(), pred_class] += 1
end = time.time()
total = end - start
print(total)
cm = confusion_matrix.float()
print(cm, cm.sum())
print(cm.diagonal().sum().div(cm.sum()).item())
cm[0, :].div_(cm[0, :].sum())
cm[1, :].div_(cm[1, :].sum())
cm[2, :].div_(cm[2, :].sum())
print(cm)