def evaluate(self, x, y_true):
print('-------- Evaluation summary --------')
y_pred = self.predict(x)
precision, recall, f1 = metrics(y_pred, y_true)
print('Precision: {:.3f}, Recall: {:.3f}, F1-score: {:.3f}\n'.format(
precision, recall, f1))
return precision, recall, f1
def metrics_eval(self, X_test, y_true):
"""
Calculates the confusion matrix, recall, precision, accuracy
of model predictions.
:param X_test: 2-dimensional array of sparse matrix
:param y_true: 1-dimensional array of true target values
:return: 2-D arrays confusion matrix, floats of recall, precision and accuracy
"""
y_pred = self.predict(X_test)
return util.metrics(y_test=y_true, y_predict=y_pred)
def evaluate(ly_true, ly_pred, metrics=evaluation_metrics, n_jobs=-1):
print('evaluation (n_jobs={})'.format(n_jobs))
if n_jobs == 1:
return {
lang: metrics(ly_true[lang], ly_pred[lang])
for lang in ly_true.keys()
}
else:
langs = list(ly_true.keys())
evals = Parallel(n_jobs=n_jobs)(
delayed(metrics)(ly_true[lang], ly_pred[lang]) for lang in langs)
return {lang: evals[i] for i, lang in enumerate(langs)}
def evaluate_single_lang(polylingual_method,
X,
y,
lang,
predictor=None,
soft=False):
print('prediction for test in a single language')
if predictor is None:
predictor = polylingual_method.predict
metrics = evaluation_metrics
if soft is True:
metrics = soft_evaluation_metrics
ly_ = predictor({lang: X})
return metrics(y, ly_[lang])
def yolo_eval(dataset_path, ckpt_path):
"""Yolov3 evaluation."""
ds = create_yolo_dataset(dataset_path, is_training=False)
config = ConfigYOLOV3ResNet18()
net = yolov3_resnet18(config)
eval_net = YoloWithEval(net, config)
print("Load Checkpoint!")
param_dict = load_checkpoint(ckpt_path)
load_param_into_net(net, param_dict)
eval_net.set_train(False)
i = 1.
total = ds.get_dataset_size()
start = time.time()
pred_data = []
print("\n========================================\n")
print("total images num: ", total)
print("Processing, please wait a moment.")
for data in ds.create_dict_iterator():
img_np = data['image']
image_shape = data['image_shape']
annotation = data['annotation']
eval_net.set_train(False)
output = eval_net(Tensor(img_np), Tensor(image_shape))
for batch_idx in range(img_np.shape[0]):
pred_data.append({
"boxes": output[0].asnumpy()[batch_idx],
"box_scores": output[1].asnumpy()[batch_idx],
"annotation": annotation
})
percent = round(i / total * 100, 2)
print(' %s [%d/%d]' % (str(percent) + '%', i, total), end='\r')
i += 1
print(' %s [%d/%d] cost %d ms' %
(str(100.0) + '%', total, total, int((time.time() - start) * 1000)),
end='\n')
precisions, recalls = metrics(pred_data)
print("\n========================================\n")
for i in range(config.num_classes):
print("class {} precision is {:.2f}%, recall is {:.2f}%".format(
i, precisions[i] * 100, recalls[i] * 100))
def ssd_eval(dataset_path, ckpt_path):
"""SSD evaluation."""
ds = create_ssd_dataset(dataset_path,
batch_size=1,
repeat_num=1,
is_training=False)
net = SSD300(ssd_mobilenet_v2(), ConfigSSD(), is_training=False)
print("Load Checkpoint!")
param_dict = load_checkpoint(ckpt_path)
net.init_parameters_data()
load_param_into_net(net, param_dict)
net.set_train(False)
i = 1.
total = ds.get_dataset_size()
start = time.time()
pred_data = []
print("\n========================================\n")
print("total images num: ", total)
print("Processing, please wait a moment.")
for data in ds.create_dict_iterator():
img_np = data['image']
image_shape = data['image_shape']
annotation = data['annotation']
output = net(Tensor(img_np))
for batch_idx in range(img_np.shape[0]):
pred_data.append({
"boxes": output[0].asnumpy()[batch_idx],
"box_scores": output[1].asnumpy()[batch_idx],
"annotation": annotation,
"image_shape": image_shape
})
percent = round(i / total * 100, 2)
print(f' {str(percent)} [{i}/{total}]', end='\r')
i += 1
cost_time = int((time.time() - start) * 1000)
print(f' 100% [{total}/{total}] cost {cost_time} ms')
mAP = metrics(pred_data)
print("\n========================================\n")
print(f"mAP: {mAP}")
checkpoint = torch.load(a1_path)
optimal_model.load_state_dict(checkpoint['model_dict'])
optimal_model.eval()
bce_loss = nn.BCEWithLogitsLoss()
mask,pred = test(optimal_model,test_loader)
gt_map = np.concatenate(mask, axis=0)
pred_map = np.concatenate(pred, axis=0)
# In[ ]:
pred_map_sig = nn.Sigmoid()(torch.tensor(pred_map))
threshold = 0.5
predict = deepcopy(pred_map_sig)
predict[pred_map_sig>=threshold]=1
predict[pred_map_sig<threshold]=0
cut_pred1 = []
cut_gt = []
for i in range(predict.shape[0]):
cut_pred1.append(np.uint8(predict[i,0,132:252,180:300]))
cut_gt.append(np.uint8(gt_map[i,0,132:252,180:300]))
metrics(cut_gt,cut_pred1)
dtype=torch.float), sam['label'].to(device=device,
dtype=torch.long)
out, c = model(data)
loss = criterion(out, target) * 100.0
test_loss.append(loss.item())
out = sig(out)
tcla = torch.cat((tcla, label.cpu()), dim=0)
cla = torch.cat((cla, c.cpu()), dim=0)
mas = torch.cat((mas, target.cpu()), dim=0)
res = torch.cat((res, out.cpu()), dim=0)
avg_test_loss = sum(test_loss) / len(test_loss)
print('Test loss = {:.{prec}f}'.format(avg_test_loss, prec=4))
mas = np.array(mas)
res = np.array(res)
pred = deepcopy(res)
pred[res >= 0.5] = 1
pred[res < 0.5] = 0
cut_pred = []
cut_mas = []
for i in range(pred.shape[0]):
cut_pred.append(np.uint8(pred[i, 0, 132:252, 180:300]))
cut_mas.append(np.uint8(mas[i, 0, 132:252, 180:300]))
metrics(cut_mas, cut_pred)
def get_preprocessing(preprocessing_fn):
"""Construct preprocessing transform
Args:
preprocessing_fn (callbale): data normalization function
(can be specific for each pretrained neural network)
Return:
transform: albumentations.Compose
"""
_transform = [
albu.Lambda(image=preprocessing_fn),
albu.Lambda(image=to_tensor, mask=to_tensor),
]
return albu.Compose(_transform)
class WSOLDataset(Dataset):
"""Face Landmarks dataset."""
def __init__(self, data, label, grabcut, transform=None):
"""
Args:
csv_file (string): Path to the csv file with annotations.
root_dir (string): Directory with all the images.
transform (callable, optional): Optional transform to be applied
on a sample.
"""
self.grabcut = grabcut
self.data = data
self.label = label
def __len__(self):
return len(self.data)
def __getitem__(self, idx):
sample = augmentation(image=self.data[idx],mask=self.grabcut[idx])
sample1 = preprocessing(image=sample['image'],mask=sample['mask'].reshape(384,480,1))
out = {'image':sample1['image'],'mask':sample1['mask'],'label':torch.tensor(self.label[idx])}
return out
preprocess_input = get_preprocessing_fn('resnet101', pretrained='imagenet')
augmentation = get_validation_augmentation()
preprocessing = get_preprocessing(preprocess_input)
train_dataset = WSOLDataset(data=train_raw,label=train_labels,grabcut=train_ell)
val_dataset = WSOLDataset(data=val_raw,label=val_labels,grabcut=val_ell)
test_dataset = WSOLDataset(data=test_raw,label=test_labels,grabcut=test_grab)
BATCH_SIZE = 8
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=BATCH_SIZE, shuffle=True,
num_workers=1, drop_last=True)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=BATCH_SIZE, shuffle=True,
num_workers=1,drop_last=True)
test_loader = torch.utils.data.DataLoader(test_dataset,
batch_size=BATCH_SIZE, shuffle=False,
num_workers=1,drop_last=True)
new_coseg_model = model1().cuda()
checkpoint = torch.load(acoseg_path)
new_coseg_model.load_state_dict(checkpoint['model_dict'])
print('loaded pre-trained model')
sig = nn.Sigmoid()
criterion = nn.BCELoss()
test_loss=[]
mas = torch.tensor([])
res = torch.tensor([])
new_coseg_model.eval()
with torch.no_grad():
for batch_idx, sam in enumerate(test_loader):
#print(batch_idx)
data = sam['image'].float().cuda()
target = sam['mask'].float().cuda()
output1, output2 = new_coseg_model(data[0:4], data[4:8])
output1 = sig(output1)
output2 = sig(output2)
loss = criterion(output1.squeeze(), target[0:4].squeeze(1)) + criterion(output2.squeeze(), target[4:8].squeeze(1))
test_loss.append(loss.item())
mas = torch.cat((mas,target.cpu()),dim=0)
res = torch.cat((res,output1.cpu()),dim=0)
res = torch.cat((res,output2.cpu()),dim=0)
avg_test_loss = np.mean(test_loss)
print('Test loss = {:.{prec}f}'.format(avg_test_loss, prec=4))
mas = np.array(mas)
res = np.array(res)
pred = deepcopy(res)
pred[res>=0.5]=1 #check threshold
pred[res<0.5]=0
cut_pred = []
cut_mas = []
for i in range(pred.shape[0]):
cut_pred.append(np.uint8(pred[i,0,132:252,180:300]))
cut_mas.append(np.uint8(mas[i,0,132:252,180:300]))
metrics(cut_mas,cut_pred)