def main():
model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE)
optimizer = optim.Adam(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
loss_fn = YoloLoss()
train_dataset = VOCDataset("data/train.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR)
test_dataset = VOCDataset("data/test.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR)
train_loader=DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=1, pin_memory=PIN_MEMORY, shuffle=True,drop_last=True)
test_loader=DataLoader(dataset=test_dataset, batch_size=BATCH_SIZE, num_workers=1, pin_memory=PIN_MEMORY, shuffle=True,drop_last=True)
for epoch in range(EPOCHS):
pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4)
mAP = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5)
print(f"Train mAP:{mAP}")
train_fn(train_loader, model, optimizer, loss_fn)
if epoch > 99:
for x, y in test_loader:
x = x.to(DEVICE)
for idx in range(16):
bboxes = cellboxes_to_boxes(model(x))
bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4)
plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes)
if __name__ == "__main__":
main()
def test_alpha_by_use_map(args, docs, data):
br = prepare_bert_result(args.test)
maxAlpha = 5 * 10 + 1
fianlAlpha = 0
maxmAP = 0
mAPList = []
for alpha in range(1, maxAlpha):
alpha *= 0.1
mAPSub = 0
subList = []
for i in range(len(br)):
res = br[i]
gt = data[i]
#
rank = []
for j in range(len(res['doc_ids'])):
rank.append((res['doc_ids'][j], alpha * res['doc_scores'][j] +
gt['bm25_top1000_scores'][j]))
rank = sorted(rank, key=lambda s: s[1], reverse=True)
rank = [r[0] for r in rank]
tmp = mean_average_precision(rank, gt['pos_doc_ids'])
subList.append(tmp)
mAPSub += tmp
#
if mAPSub > maxmAP:
maxmAP = mAPSub
fianlAlpha = alpha
mAPList = subList
print(
f'average alpha : {fianlAlpha}, mAP : {maxmAP / len(br)}, mAPList : {mAPList}'
)
return fianlAlpha
def validate(valid_loader, model, criterion, func, targets):
losses = AverageMeter()
top1 = AverageMeter()
top5 = AverageMeter()
# switch to evaluate mode
model.eval()
outputs = []
for i, (input, target, video_names, frame_indexes) in enumerate(valid_loader):
b = target.size(0)
input = input.cuda(async=True)
target = target.cuda(async=True)
input_var = torch.autograd.Variable(input, volatile=True)
target_var = torch.autograd.Variable(target, volatile=True)
# compute output
output = model(input_var)
output = func(output) # softmax, (bxr)x101
# mean probabilities over r sequences
output = output.view(b, -1, num_classes).mean(1).squeeze(1)
loss = criterion(torch.log(output), target_var) # needs logsoftmax
outputs.append(output.data.cpu().numpy())
# measure accuracy and record loss
prec1, prec5 = accuracy(output.data, target, topk=(1,5))
losses.update(loss.data[0], target.size(0))
top1.update(prec1[0], target.size(0))
top5.update(prec5[0], target.size(0))
outputs = np.concatenate(outputs, axis=0)
mean_ap = mean_average_precision(outputs, targets, num_classes)
return losses.avg, top1.avg, top5.avg, mean_ap
def main():
model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE)
optimizer = optim.Adam(model.parameters(),
lr=config.LEARNING_RATE,
weight_decay=config.WEIGHT_DECAY)
loss_fn = YoloLoss()
scaler = torch.cuda.amp.GradScaler()
train_loader, test_loader, train_eval_loader = get_loaders(
train_csv_path=config.DATASET + "/train.csv",
test_csv_path=config.DATASET + "/test.csv")
if config.LOAD_MODEL:
load_checkpoint(config.CHECKPOINT_FILE, model, optimizer,
config.LEARNING_RATE)
print("loaded model")
scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor(
config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(config.DEVICE)
for epoch in range(config.NUM_EPOCHS):
print(f"\nEpoch [{epoch}]")
#plot_couple_examples(model, test_loader, 0.6, 0.5, scaled_anchors)
train_fn(train_loader, model, optimizer, loss_fn, scaler,
scaled_anchors)
#if config.SAVE_MODEL:
# save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar")
#print("On Train Eval loader:")
#print("On Train loader:")
#check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD)
if epoch > 0 and epoch % 4 == 0:
valid_fn(train_eval_loader, model, loss_fn, scaled_anchors)
check_class_accuracy(model,
test_loader,
threshold=config.CONF_THRESHOLD)
pred_boxes, true_boxes = get_evaluation_bboxes(
test_loader,
model,
iou_threshold=config.NMS_IOU_THRESH,
anchors=config.ANCHORS,
threshold=config.CONF_THRESHOLD,
)
mapval = mean_average_precision(
pred_boxes,
true_boxes,
iou_threshold=config.MAP_IOU_THRESH,
box_format="midpoint",
num_classes=config.NUM_CLASSES,
)
print(f"MAP: {mapval.item()}")
print("\nnResuming Training\n")
model.train()
def main():
model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE)
optimizer = optim.Adam(
model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
loss_fn = YoloLoss()
if LOAD_MODEL:
load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer)
train_dataset = VOCDataset(
"data/100examples.csv",
transform=transform,
img_dir=IMG_DIR,
label_dir=LABEL_DIR
)
test_dataset = VOCDataset(
"data/test.csv",
transform=transform,
img_dir=IMG_DIR,
label_dir=LABEL_DIR
)
train_loader = DataLoader(
dataset=train_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=True
)
test_loader = DataLoader(
dataset=test_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=True
)
for epoch in range(EPOCHS):
pred_boxes, target_boxes = get_bboxes(
train_loader, model, iou_threshold=0.5, threshold=0.4
)
mean_avg_prec = mean_average_precision(
pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint"
)
print(f"Train mAP in {epoch}: {mean_avg_prec}")
train_fn(train_loader, model, optimizer, loss_fn)
def main():
model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE)
optimizer = optim.Adam(
model.parameters(), lr=config.LEARNING_RATE, weight_decay=config.WEIGHT_DECAY
)
loss_fn = YoloLoss()
scaler = torch.cuda.amp.GradScaler()
train_loader, test_loader, train_eval_loader = get_loaders(
train_csv_path=config.DATASET + "/train.csv", test_csv_path=config.DATASET + "/test.csv"
)
if config.LOAD_MODEL:
load_checkpoint(
config.CHECKPOINT_FILE, model, optimizer, config.LEARNING_RATE
)
scaled_anchors = (
torch.tensor(config.ANCHORS)
* torch.tensor(config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)
).to(config.DEVICE)
for epoch in range(config.NUM_EPOCHS):
train_fn(train_loader, model, optimizer, loss_fn, scaler, scaled_anchors)
if epoch > 0 and epoch % 3 == 0:
check_class_accuracy(model, test_loader, threshold=config.CONF_THRESHOLD)
pred_boxes, true_boxes = get_evaluation_bboxes(
test_loader,
model,
iou_threshold=config.NMS_IOU_THRESH,
anchors=config.ANCHORS,
threshold=config.CONF_THRESHOLD,
)
mapval = mean_average_precision(
pred_boxes,
true_boxes,
iou_threshold=config.MAP_IOU_THRESH,
box_format="midpoint",
num_classes=config.NUM_CLASSES,
)
print(f"MAP: {mapval.item()}")
model.train()
if epoch > 99:
for x, y in train_loader:
x = x.to(DEVICE)
for idx in range(8):
bboxes = cellboxes_to_boxes(model(x))
bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4)
plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes, idx)
if __name__ == "__main__":
main()
def main():
model = YOLOv2_lite(grid_size=GRID_SIZE,
num_boxes=NUM_BOXES,
num_classes=NUM_CLASSES).to(DEVICE)
model.load_state_dict(torch.load(MODEL_PATH))
model.eval()
test_dataset = YOLOVOCDataset(
DATA_CSV,
transform=transform,
img_dir=IMG_DIR,
label_dir=LABEL_DIR,
S=GRID_SIZE,
B=NUM_BOXES,
C=NUM_CLASSES,
hflip_prob=0.5,
)
test_loader = DataLoader(
dataset=test_dataset,
batch_size=BATCH_SIZE,
shuffle=True,
drop_last=False,
)
images, predictions, labels = get_bboxes(
test_loader,
model,
iou_threshold=0.5,
prob_threshold=0.4,
S=GRID_SIZE,
C=NUM_CLASSES,
mode="all",
return_images=True,
)
mAP = mean_average_precision(predictions, labels, num_classes=NUM_CLASSES)
print("Mean Average Precision: %.3f" % (mAP))
img_to_plot = np.random.randint(0, len(images))
image = images[img_to_plot]
img_preds = [pred[1:] for pred in predictions if pred[0] == img_to_plot]
img_labels = [label[1:] for label in labels if label[0] == img_to_plot]
#plot_detections(image, img_preds)
plot_detections(image, img_labels)
def main():
model = Yolo(num_classes=config.NUM_CLASSES).to(config.DEVICE)
optimizer = optim.Adam(model.parameters(),
lr=config.LEARNING_RATE,
weight_decay=config.WEIGHT_DECAY)
lossfunction = Yololoss()
scaler = torch.cuda.amp.GradScaler()
train_loader, test_loader, train_eval_loader = get_loaders(
train_csv_path=config.DATASET + "/train.csv",
test_csv_path=config.DATASET + "/test.csv")
if config.LOAD_MODEL:
load_checkpoint(config.CHECKPOINT_FILE, model, optimizer,
config.LEARNING_RATE)
scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor(
config.GRID_SIZE).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(
config.DEVICE)
for epoch in range(config.NUM_EPOCHS):
train(train_loader, model, optimizer, lossfunction, scaler,
scaled_anchors)
if config.SAVE_MODEL:
save_checkpoint(model, optimizer, filename=config.CHECKPOINT_FILE)
if epoch > 0 and epoch % 3 == 0:
check_class_accuracy(model,
test_loader,
threshold=config.CONF_THRESHOLD)
pred_boxes, true_boxes = get_evaluation_bboxes(
test_loader,
model,
iou_threshold=config.NMS_IOU_THRESH,
anchors=config.ANCHORS,
threshold=config.CONF_THRESHOLD,
)
mapval = mean_average_precision(
pred_boxes,
true_boxes,
iou_threshold=config.MAP_IOU_THRESH,
box_format="midpoint",
num_classes=config.NUM_CLASSES,
)
print(f"MAP: {mapval.item()}")
model.train()
def main():
model = YOLOv1(split_size=7, num_boxes=2, num_classes=20).to(device)
optimizer = optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=wd)
loss_fn = YoloLoss()
if load_model:
load_checkpoint(torch.load(load_model_file), model, optimizer)
train_dataset = VOCDataset("data/8examples.csv",
transform=transform,
img_dir=img_dir,
label_dir=label_dir)
test_dataset = VOCDataset("data/test.csv",
transform=transform,
img_dir=img_dir,
label_dir=label_dir)
train_loader = DataLoader(dataset=train_dataset,
batch_size=bs,
num_workers=num_workers,
pin_memory=pin_mem,
shuffle=True,
drop_last=False)
test_loader = DataLoader(dataset=test_dataset,
batch_size=bs,
num_workers=num_workers,
pin_memory=pin_mem,
shuffle=True,
drop_last=True)
for epoch in range(epochs):
pred_boxes, target_boxes = get_bboxes(train_loader,
model,
iou_threshold=0.5,
threshold=0.4)
mean_avg_prec = mean_average_precision(pred_boxes,
target_boxes,
iou_threshold=0.5,
box_format="midpoint")
print(f"Train mAP: {mean_avg_prec}")
train_fn(train_loader, model, optimizer, loss_fn)
def main():
model = Yolov1(split_size=7, num_boxes=2, num_classes=20).to(DEVICE)
optimizer = optim.Adam(
model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY
)
loss_fn = YoloLoss()
if LOAD_MODEL:
load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer)
train_dataset = VOCDataset(
# test.csv, 8examples.csv, 100examples.csv
"data/8examples.csv",
transform=transform,
img_dir=IMG_DIR,
label_dir=LABEL_DIR,
)
test_dataset = VOCDataset(
"data/test.csv" , transform = transform, img_dir = IMG_DIR, label_dir = LABEL_DIR
)
train_loader = DataLoader(
dataset=train_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=False,
)
test_loader = DataLoader(
dataset=test_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=False,
)
for epoch in range(EPOCHS):
# for x, y in train_loader:
# x = x.to(DEVICE)
# for idx in range(8):
# bboxes = cellboxes_to_boxes(model(x))
# bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4, box_format="midpoint")
# plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes)
# import sys
# sys.exit()
pred_boxes, target_boxes = get_bboxes(
train_loader, model, iou_threshold=0.5, threshold=0.4 , device= DEVICE,
)
mean_avg_prec = mean_average_precision(
pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint"
)
if mean_avg_prec > 0.9:
checkpoint = {
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
}
save_checkpoint(checkpoint, filename=LOAD_MODEL_FILE)
import time
time.sleep(10)
print(f"Train mAP: {mean_avg_prec}")
train_fn(train_loader, model, optimizer, loss_fn)
def main():
"""
Setup the model, loss function, data loader. Run the train function
for each epoch.
We may want to consider tune the number for non max suppression and mean
average precision in order to remove false positive. BY false positive, they
are a set of wrong output boxes. We can get rid of them by increasing the
parameters.
"""
model = YOLOv3(num_classes=config.NUM_CLASSES).to(config.DEVICE)
optimizer = optim.Adam(model.parameters(),
lr=config.LEARNING_RATE,
weight_decay=config.WEIGHT_DECAY)
loss_fn = YoloLoss()
scaler = torch.cuda.amp.GradScaler()
train_loader, test_loader, train_eval_loader = get_loaders(
train_csv_path=config.DATASET + "/train.csv",
test_csv_path=config.DATASET + "/test.csv")
if config.LOAD_MODEL:
load_checkpoint(config.CHECKPOINT_FILE, model, optimizer,
config.LEARNING_RATE)
scaled_anchors = (torch.tensor(config.ANCHORS) * torch.tensor(
config.S).unsqueeze(1).unsqueeze(1).repeat(1, 3, 2)).to(config.DEVICE)
for epoch in range(config.NUM_EPOCHS):
# plot_couple_examples(model, test_loader, 0.6, 0.5, scaled_anchors)
train_fn(train_loader, model, optimizer, loss_fn, scaler,
scaled_anchors)
if config.SAVE_MODEL:
save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar")
# print(f"Currently epoch {epoch}")
# print("On Train Eval loader:")
# print("On Train loader:")
# check_class_accuracy(model, train_loader, threshold=config.CONF_THRESHOLD)
if epoch > 0 and epoch % 3 == 0:
check_class_accuracy(model,
test_loader,
threshold=config.CONF_THRESHOLD)
pred_boxes, true_boxes = get_evaluation_bboxes(
test_loader,
model,
iou_threshold=config.NMS_IOU_THRESH,
anchors=config.ANCHORS,
threshold=config.CONF_THRESHOLD,
)
mapval = mean_average_precision(
pred_boxes,
true_boxes,
iou_threshold=config.MAP_IOU_THRESH,
box_format="midpoint",
num_classes=config.NUM_CLASSES,
)
print(f"MAP: {mapval.item()}")
model.train()
def _eval_model(self, model: Model, writer: SummaryWriter, step, t,
eval_title, results_dict):
training = model.training
model.eval()
if t in self.config['schedule_simple']:
t_idx = self.config['schedule_simple'].index(t)
else:
t_idx = len(self.config['schedule_simple']) - 1
# for calculating total performance
targets_total = []
probs_total = []
# Accuracy of each subset
for order_i, t_i in enumerate(self.config['schedule_simple'][:t_idx +
1]):
subset_name = t_i
last_id = self.config['schedule_simple'][
-1] # XXX should be -1. -2 for debugging.
subset = self.subsets[t_i]
data = DataLoader(
subset,
batch_size=self.config['eval_batch_size'],
num_workers=self.config['eval_num_workers'],
collate_fn=self.collate_fn,
)
# results is dict. {method: group_averagemeter_object}
results, targets, probs = validate(subset_name, model, data,
self.category_map, results_dict,
last_id, self.split_cats_dict)
targets_total.append(targets)
probs_total.append(probs)
if subset_name in results_dict:
results_dict[subset_name].append(results)
else:
results_dict[subset_name] = [results]
for metric in results.keys():
results[metric].write_to_excel(
os.path.join(writer.logdir,
'results_{}.xlsx'.format(metric)),
sheet_name='task {}'.format(subset_name),
column_name='task {}'.format(
self.config['schedule_simple'][t_idx]),
info='avg')
# =================================================================================================================
# calculate scores for trained tasks.
prefix = 'tally_' # prefix for tensorboard plotting and csv filename
targets_total = torch.cat(targets_total, axis=0)
probs_total = torch.cat(probs_total, axis=0)
predicts_total = probs_total > 0.5 # BCE style predicts
total_metric = ['CP', 'CR', 'CF1', 'OP', 'OR', 'OF1', 'mAP']
results = dict() # reset results
CP, CR, CF1, OP, OR, OF1, mAP = (AverageMeter()
for _ in range(len(total_metric)))
ncats = targets_total.sum(axis=0)
# ignore classes in future tasks
cats_in_task_idx = ncats > 0
cats_in_task_name = self.category_map[cats_in_task_idx].tolist()
targets_total = targets_total
probs_total = probs_total
predicts_total = predicts_total
# calculate score
precision_pc = torch.mean(
precision_score_per_class(targets_total[:, cats_in_task_idx],
predicts_total[:, cats_in_task_idx],
zero_division=0))
recall_pc = torch.mean(
recall_score_per_class(targets_total[:, cats_in_task_idx],
predicts_total[:, cats_in_task_idx],
zero_division=0))
# CF1. note that CF1 is not a mean value of categories' f1_score
f1_pc = ((2 * precision_pc * recall_pc) / (precision_pc + recall_pc)
) if (precision_pc + recall_pc) > 0 else torch.tensor([0.])
precision_oa = precision_score_overall(
targets_total[:, cats_in_task_idx],
predicts_total[:, cats_in_task_idx],
zero_division=0)
recall_oa = recall_score_overall(targets_total[:, cats_in_task_idx],
predicts_total[:, cats_in_task_idx],
zero_division=0)
f1_oa = f1_score_overall(targets_total[:, cats_in_task_idx],
predicts_total[:, cats_in_task_idx],
zero_division=0)
map_ = mean_average_precision(targets_total[:, cats_in_task_idx],
probs_total[:, cats_in_task_idx])
# save to AverageMeter
CP.update(precision_pc.item())
CR.update(recall_pc.item())
CF1.update(f1_pc.item())
OP.update(precision_oa.item())
OR.update(recall_oa.item())
OF1.update(f1_oa.item())
mAP.update(map_.item())
results[prefix + 'CP'] = CP
results[prefix + 'CR'] = CR
results[prefix + 'CF1'] = CF1
results[prefix + 'OP'] = OP
results[prefix + 'OR'] = OR
results[prefix + 'OF1'] = OF1
results[prefix + 'mAP'] = mAP
# for reporting major, moderate, minor cateogory performances
for report_name in self.split_cats_dict.keys():
reporter = Group_AverageMeter()
# get report category idxes
all_cats = self.category_map.tolist()
task_cats = set(cats_in_task_name)
report_cats = task_cats & set(self.split_cats_dict[report_name])
report_cats_idx = torch.tensor(
[all_cats.index(cat) for cat in report_cats], dtype=torch.long)
# CP, CR, CF1 performance of report_categories.
_class_precision = precision_score_per_class(
targets_total[:, report_cats_idx],
predicts_total[:, report_cats_idx],
zero_division=0)
_class_recall = recall_score_per_class(
targets_total[:, report_cats_idx],
predicts_total[:, report_cats_idx],
zero_division=0)
_class_precision = torch.mean(_class_precision)
_class_recall = torch.mean(_class_recall)
# CF1 bias. note that CF1 is not a mean value of categories' f1_score
_class_f1 = ((2*_class_precision*_class_recall)/(_class_precision+_class_recall)) \
if (_class_precision+_class_recall)>0 else torch.tensor([0.])
# OP, OR, OF1 performance of report_categories.
_overall_precision = precision_score_overall(
targets_total[:, report_cats_idx],
predicts_total[:, report_cats_idx],
zero_division=0)
_overall_recall = recall_score_overall(
targets_total[:, report_cats_idx],
predicts_total[:, report_cats_idx],
zero_division=0)
_overall_f1 = f1_score_overall(targets_total[:, report_cats_idx],
predicts_total[:, report_cats_idx],
zero_division=0)
# mAP performance of report_categories.
_mAP = mean_average_precision(targets_total[:, report_cats_idx],
probs_total[:, report_cats_idx])
reporter.update(['CP'], [_class_precision.item()], [1])
reporter.update(['CR'], [_class_recall.item()], [1])
reporter.update(['CF1'], [_class_f1.item()], [1])
reporter.update(['OP'], [_overall_precision.item()], [1])
reporter.update(['OR'], [_overall_recall.item()], [1])
reporter.update(['OF1'], [_overall_f1.item()], [1])
reporter.update(['mAP'], [_mAP.item()], [1])
reporter.total.reset()
results[prefix + report_name] = reporter
# write to tensorboard and csv.
task_len = t_idx + 1
for metric in results.keys():
if not metric in [
prefix + 'CP', prefix + 'CR', prefix + 'OP', prefix + 'OR'
]:
results[metric].write(
writer,
'%s/%s/%s/task_len(%d)' %
(metric, eval_title, self.name, task_len),
step,
info='avg')
results[metric].write_to_excel(
os.path.join(writer.logdir, 'results_{}.xlsx'.format(metric)),
sheet_name=prefix,
column_name='task {}'.format(
self.config['schedule_simple'][t_idx]),
info='avg')
# =================================================================================================================
# print performances at the end
if t_idx == len(self.config['schedule_simple']) - 1:
src = writer.logdir
csv_files = ['major', 'moderate', 'minor', 'OF1', 'CF1', 'mAP', \
prefix+'major', prefix+'moderate', prefix+'minor', prefix+'CF1', prefix+'OF1', prefix+'mAP', \
'forget']
for csv_file in csv_files:
try:
csv = pd.read_csv(os.path.join(
src, 'results_{}.csv'.format(csv_file)),
index_col=0)
# print performance after training last task
pd.set_option('display.max_rows', None)
print(
colorful.bold_green(
'\n{:10} result'.format(csv_file)).styled_string)
print(csv.round(4).iloc[:, -1])
# save as txt
with open(os.path.join(src, 'summary.txt'),
'a') as summary_txt:
summary_txt.write('\n')
summary_txt.write('{:10} result\n'.format(csv_file))
summary_txt.write(csv.round(4).iloc[:, -1].to_string())
summary_txt.write('\n')
except FileNotFoundError:
print("This excperiment doesn't have {} file!! continue.".
format(csv_file))
continue
model.train(training)
return results_dict
def main():
# Load dataset manager
with open('training_set_list.pickle', 'rb') as handle:
training_set = pickle.load(handle)
with open('validation_set_list.pickle', 'rb') as handle:
validation_set = pickle.load(handle)
with open('test_set_list.pickle', 'rb') as handle:
test_set = pickle.load(handle)
with open('genres.json') as json_data:
genres = json.load(json_data)
with open('labels.json') as json_data:
labels = json.load(json_data)
log_file_name = str(datetime.now()) + '-logs.txt'
with open("logs/" + log_file_name, 'w') as log_file:
log_file.write('Training logs \n')
iteration_file_name = str(datetime.now()) + '-iteration.txt'
with open("logs/" + iteration_file_name, 'w') as log_file:
log_file.write('Training iterations \n')
best_iteration_file_name = str(datetime.now()) + '-best_iteration.txt'
with open("logs/" + best_iteration_file_name, 'w') as log_file:
log_file.write('Best Training iterations \n')
dataset_manager = DatasetManager(training_set, validation_set, test_set,
genres, labels)
# Learning params
learning_rate = 0.001
batch_size = 50
# Nombre d'iterations
training_iters = 1000
# display training information (loss, training accuracy, ...) every 10
# iterations
local_train_step = 20
global_validation_step = 25 # test every global_test_step iterations
global_train_step = 100
# Network params
n_classes = 26
keep_rate = 0.5 # for dropout
# Graph input
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, n_classes])
keep_var = tf.placeholder(tf.float32)
# Model
pred = Model.alexnet(x, keep_var) # definition of the network architecture
# Loss and optimizer
# loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=pred, labels=y))
loss = tf.sqrt(tf.reduce_mean(tf.square(tf.subtract(y, pred))))
# optimizer = tf.train.GradientDescentOptimizer(
# learning_rate=learning_rate).minimize(loss)
optimizer = tf.train.RMSPropOptimizer(
learning_rate=learning_rate).minimize(loss)
# Init
init = tf.global_variables_initializer()
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# To do early stopping
max_validation_map = 0
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Load pretrained model
# Skip weights from fc8 (fine-tuning)
load_with_skip('pretrained_alexnet.npy', sess, ['fc8'])
print('Start training.')
step = 1
while step < training_iters:
# print("Iter ", step)
with open("logs/" + iteration_file_name, 'a') as log_file:
log_file.write("Iter {} \n".format(step))
batch_xs, batch_ys = dataset_manager.next_batch(
batch_size, 'train')
sess.run(optimizer,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_var: keep_rate
})
# Display on batch training status
if step % local_train_step == 0:
local_train_output = sess.run(pred,
feed_dict={
x: batch_xs,
keep_var: 1
})
MAP = mean_average_precision(local_train_output, batch_ys)
batch_loss = sess.run(loss,
feed_dict={
x: batch_xs,
y: batch_ys,
keep_var: 1.
})
with open("logs/" + log_file_name, 'a') as log_file:
log_file.write("Iter {} Training Loss = {:.4f}, "
"Mean average precision = {:.4f} \n".format(
step, batch_loss, MAP))
# Display global training error
if step % global_train_step == 0:
train_map_global = 0.
test_count = 0
# test accuracy by group of batch_size images
for _ in range(
int(len(dataset_manager.training_list) / batch_size) +
1):
batch_tx, batch_ty = dataset_manager.next_batch(
batch_size, 'train')
test_output = sess.run(pred,
feed_dict={
x: batch_tx,
keep_var: 1
})
MAP = mean_average_precision(test_output, batch_ty)
train_map_global += MAP
test_count += 1
train_map_global /= test_count
with open("logs/" + log_file_name, 'a') as log_file:
log_file.write(
"Global Training Accuracy = {:.4f} \n".format(
train_map_global))
# Display global testing error
if step % global_validation_step == 0:
validation_map_global = 0.
validation_count = 0
# test accuracy by group of batch_size images
for _ in range(
int(len(dataset_manager.test_list) / batch_size) + 1):
batch_tx, batch_ty = dataset_manager.next_batch(
batch_size, 'val')
test_output = sess.run(pred,
feed_dict={
x: batch_tx,
keep_var: 1
})
MAP = mean_average_precision(test_output, batch_ty)
validation_map_global += MAP
validation_count += 1
validation_map_global /= validation_count
with open("logs/" + log_file_name, 'a') as log_file:
log_file.write(
"Iter {} Global Validation Accuracy = {:.4f} \n".
format(step, validation_map_global))
if validation_map_global >= max_validation_map:
max_validation_map = validation_map_global
with open("logs/" + best_iteration_file_name,
'a') as log_file:
log_file.write("Iter {} \n".format(step))
# Save model
saver.save(
sess,
"saved_models/model_dropout05_mean_square_error.ckpt")
step += 1
# print("Finish!")
with open("logs/finish", 'w') as finish_file:
finish_file.write("Finish")
def train_loop(cfg_path, gpu_n='0'):
# get configs
with open(cfg_path, 'r') as stream:
config = yaml.safe_load(stream)
device = torch.device('cuda:{}'.format(gpu_n) if config['GPU']
and torch.cuda.is_available else 'cpu')
dtype = torch.float32 # TODO: find out how it affects speed and accuracy
MODEL = config['MODEL']
LOAD_MODEL = config['LOAD_MODEL']
LOAD_MODEL_FILE = config['LOAD_MODEL_FILE']
SAVE_MODEL = config['SAVE_MODEL']
SAVE_MODEL_N = config['SAVE_MODEL_N']
SAVE_MODEL_DIR = config['SAVE_MODEL_DIR']
DATASET_DIR = config['DATASET_DIR']
L_RATE = config['LEARNING_RATE']
DECAY_RATE = config['DECAY_RATE']
DECAY_EPOCHS = config['DECAY_EPOCHS']
WEIGHT_DECAY = config['WEIGHT_DECAY']
EPOCHS = config['EPOCHS']
BATCH_SIZE = config['BATCH_SIZE']
NUM_WORKERS = config['NUM_WORKERS']
PIN_MEMORY = config['PIN_MEMORY']
CSV_TRAIN = config['CSV_TRAIN']
CSV_VAL = config['CSV_VAL']
# set up model
if MODEL == 'Darknet':
model = YoloV1(grid_size=7, num_boxes=2, num_classes=20).to(DEVICE)
elif MODEL == 'VGG':
pass # add here VGG backbone
if LOAD_MODEL:
# TODO: load backbone
# cfg_cp, start_epoch = load_checkpoint(LOAD_MODEL_FILE, model)
val = input(
'Do you want to use config from checkpoint? Answer "yes" or "no": '
)
# if 'val' == 'yes':
# L_RATE = cfg_cp['LEARNING_RATE']
# DECAY_RATE = cfg_cp['DECAY_RATE']
# DECAY_EPOCHS = cfg_cp['DECAY_EPOCHS']
# WEIGHT_DECAY = cfg_cp['WEIGHT_DECAY']
# BALANCED = cfg_cp['BALANCED_DATASET']
# BATCH_SIZE = cfg_cp['BATCH_SIZE']
# NUM_WORKERS = cfg_cp['NUM_WORKERS']
# PIN_MEMORY = cfg_cp['PIN_MEMORY']
# MIN_IMAGES = cfg_cp['MIN_IMAGES']
# LOSS = cfg_cp['LOSS']
else:
model = init_weights(model)
start_epoch = 0
optimizer = optim.Adam(model.parameters(),
lr=L_RATE,
weight_decay=WEIGHT_DECAY)
loss_fn = YoloLoss()
loader_params = BATCH_SIZE, NUM_WORKERS, PIN_MEMORY, DATASET_DIR, CSV_TRAIN, CSV_VAL
loader = get_dataloader(loader_params)
# create folder to save models
if SAVE_MODEL:
if not os.path.exists('{}/{}'.format(SAVE_MODEL_DIR, MODEL)):
os.makedirs('{}/{}'.format(SAVE_MODEL_DIR, MODEL))
losses, accuracies = {
'train': [],
'validate': []
}, {
'train': [],
'validate': []
}
for epoch in range(start_epoch, EPOCHS + start_epoch):
t = time()
if (epoch + 1) % DECAY_EPOCHS == 0:
L_RATE *= (1 - DECAY_RATE)
optimizer = optim.Adam(model.parameters(),
lr=L_RATE,
weight_decay=WEIGHT_DECAY)
# print epoch number
print_report(part='start', epoch=epoch)
# train loop
train_epoch(loader['train'], model, optimizer, device, loss_fn)
# print metrics
pred_bb, target_bb = get_bboxes(loader['train'],
model,
iou_threshold=0.5,
threshold=0.4)
train_map = mean_average_precision(pred_bb,
target_bb,
iou_threshold=0.5,
box_format='midpoint')
v_pred_bb, v_target_bb = get_bboxes(loader['val'],
model,
iou_threshold=0.5,
threshold=0.4)
val_map = mean_average_precision(v_pred_bb,
v_target_bb,
iou_threshold=0.5,
box_format='midpoint')
metrics = -1, -1, train_map, val_map
print_report(part='accuracy', metrics=metrics)
# collect metrics
# losses['train'].append(train_loss)
# losses['validate'].append(val_loss)
# accuracies['train'].append(train_acc)
# accuracies['validate'].append(val_acc)
# save models
# if SAVE_MODEL:
# save_checkpoint(model=model, cfg=cfg, epoch=epoch, loss=round(val_loss, 3))
# print time
print_report(part='end', t=int(time() - t))
def main():
# Load dataset manager
with open('training_set_list.pickle', 'rb') as handle:
training_set = pickle.load(handle)
with open('validation_set_list.pickle', 'rb') as handle:
validation_set = pickle.load(handle)
with open('test_set_list.pickle', 'rb') as handle:
test_set = pickle.load(handle)
with open('genres.json') as json_data:
genres = json.load(json_data)
with open('labels.json') as json_data:
labels = json.load(json_data)
dataset_manager = DatasetManager(training_set, validation_set, test_set,
genres, labels)
batch_size = 1
n_classes = 26
# Graph input
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, n_classes])
keep_var = tf.placeholder(tf.float32)
# Model
pred = Model.alexnet(x, keep_var) # definition of the network architecture
# Loss and optimize
# Init
init = tf.global_variables_initializer()
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Load pretrained model
# Skip weights from fc8 (fine-tuning)
# load_with_skip('pretrained_alexnet.npy', sess, ['fc8'])
# saver.restore(sess, "saved_models/MSE_without_data_augmentation_0.75_0.001/film_genre_model.ckpt")
saver.restore(
sess, "saved_models/models/model_dropout05_mean_square_error.ckpt")
print('Model Restored')
test_map_global = 0.
test_count = 0
# test accuracy by group of batch_size images
for _ in range(int(len(dataset_manager.test_list) / batch_size) + 1):
batch_tx, batch_ty = dataset_manager.next_batch(batch_size, 'test')
# print(batch_tx[0], batch_ty[0])
test_output = sess.run(pred, feed_dict={x: batch_tx, keep_var: 1})
# print(test_output[0])
MAP = mean_average_precision(test_output, batch_ty)
test_map_global += MAP
test_count += 1
test_map_global /= test_count
print("Global Test Accuracy = {:.4f}".format(test_map_global))
# Load one image
img = cv2.imread('saved_models/images_tests/yellow.jpg')
img = cv2.resize(img, (227, 227))
img = img.astype(np.float32)
img -= np.array([104., 117., 124.])
print(img)
test_output = sess.run(pred,
feed_dict={
x: np.reshape(img, (1, 227, 227, 3)),
keep_var: 1
})
score_dict = {}
for score, genre in zip(test_output[0], genres):
score_dict[genre] = score
print(list(reversed(sorted(score_dict.items(), key=lambda x: x[1]))))
train_embedings,
test_embedings,
y_train,
y_test,
k=50,
n_cell_types=n_cell_types)
elif model_type == "base_nn":
all_probs = model.predict(x_test)
order = np.argsort(-all_probs, axis=-1)
y_pred = order[:, 0]
y_probs = np.array(
[all_probs[i, order[i, 0]] for i in range(all_probs.shape[0])])
accuracy = accuracy_score(y_test, y_pred)
overall_map = mean_average_precision(y_test, y_pred, y_probs)
cell_type_map = get_cell_type_map(n_cell_types, y_test, y_pred,
y_probs)
np.savetxt("test_label", y_test)
np.savetxt("pred_label", y_pred)
with open(out_file, "w") as out_file:
out_file.writelines('Model type: ' + model_type + "\n")
out_file.writelines('Number of epochs: ' + str(epochs) + "\n")
out_file.writelines('Regularization: l1 ' + str(l1_reg) + " l2 " +
str(l2_reg) + "\n")
out_file.writelines('Test accuracy: ' + str(accuracy) + "\n")
out_file.writelines('Overall MAP: ' + str(overall_map) + '\n')
out_file.writelines('Cell type specific MAP:\n')
out_file.writelines('\t\t\t\t\t' + '\t'.join(label_names) + '\n')
out_file.writelines('\t\t\t\t\t' + '\t'.join(map(str, cell_type_map)) +
'\n')
def main(args):
# scale normalized anchors by their corresponding scales (e.g img_size = 416, feature maps will 32, 16, 8,
# scaled_anchors = normalized_anchors X (8, 16, 32))
"""
scaled anchors
tensor([[ 3.6250, 2.8125],
[ 4.8750, 6.1875],
[11.6562, 10.1875],
[ 1.8750, 3.8125],
[ 3.8750, 2.8125],
[ 3.6875, 7.4375],
[ 1.2500, 1.6250],
[ 2.0000, 3.7500],
[ 4.1250, 2.8750]])
"""
di = torch.tensor([
int(args.img_size / 32),
int(args.img_size / 16),
int(args.img_size / 8)
]).unsqueeze(1)
scaled_anchors = (torch.tensor(anchors) * torch.repeat_interleave(
di, torch.tensor([3, 3, 3]), dim=0).repeat(1, 2)).to(args.device)
train_dataset = YOLODataset(args.train_img_dir,
args.train_label_dir,
args.train_annotation_file,
scaled_anchors,
num_classes,
args.iou_thre,
args.img_size,
transform=train_transforms)
val_dataset = YOLODataset(args.val_img_dir,
args.val_label_dir,
args.val_annotation_file,
scaled_anchors,
num_classes,
args.iou_thre,
args.img_size,
transform=test_transforms)
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True)
val_loader = DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=True)
model = YOLO(num_classes).to(args.device)
# if it is first time to train the model, use darknet53 pretrained weights
# load_weights_darknet53("darknet53.conv.74", model)
optimizer = optim.Adam(model.parameters(), lr=1e-5, weight_decay=1e-4)
load_checkpoint("checkpoint.pth.tar", model, optimizer, 1e-5)
for epoch in range(args.epoches):
train_loop(train_loader, model, YOLOloss, optimizer, args.device,
scaled_anchors)
print(f"Epoch {epoch+1}\n-------------------------------")
save_checkpoint(model, optimizer, filename=f"checkpoint.pth.tar")
model.eval()
results = check_class_accuracy(model, val_loader, args.conf_thre)
mAP = mean_average_precision(model, val_loader, scaled_anchors,
num_classes, args.iou_thre)
results.append(mAP)
with open("result.txt", 'a') as f:
f.write(str(results).strip("[").strip("]") + '\n')
model.train()
print("Done!")
def main():
model = Yolov1(split_size=S, num_boxes=B, num_classes=C).to(DEVICE)
optimizer = optim.Adam(model.parameters(),
lr=LEARNING_RATE,
weight_decay=WEIGHT_DECAY)
loss_fn = YoloLoss(S=S, B=B, C=C)
if LOAD_MODEL:
load_checkpoint(torch.load(LOAD_MODEL_FILE), model, optimizer)
train_dataset = VOCDataset(
training_path=
'/home/mt/Desktop/For_github/computer_vision_projects/face_recognition/data',
S=3,
C=2,
transform=transform)
# test_dataset = VOCDataset(
# "data/test.csv", transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR,
# )
train_loader = DataLoader(
dataset=train_dataset,
batch_size=BATCH_SIZE,
num_workers=NUM_WORKERS,
pin_memory=PIN_MEMORY,
shuffle=True,
drop_last=True,
)
# test_loader = DataLoader(
# dataset=test_dataset,
# batch_size=BATCH_SIZE,
# num_workers=NUM_WORKERS,
# pin_memory=PIN_MEMORY,
# shuffle=True,
# drop_last=True,
# )
for epoch in range(EPOCHS):
# for x, y in train_loader:
# x = x.to(DEVICE)
# for idx in range(8):
# bboxes = cellboxes_to_boxes(model(x))
# bboxes = non_max_suppression(bboxes[idx], iou_threshold=0.5, threshold=0.4, box_format="midpoint")
# plot_image(x[idx].permute(1,2,0).to("cpu"), bboxes)
# import sys
# sys.exit()
pred_boxes, target_boxes = get_bboxes(train_loader,
model,
iou_threshold=0.5,
threshold=0.4)
mean_avg_prec = mean_average_precision(pred_boxes,
target_boxes,
iou_threshold=0.5,
box_format="midpoint")
print(f"Train mAP: {mean_avg_prec}")
#if mean_avg_prec > 0.9:
# checkpoint = {
# "state_dict": model.state_dict(),
# "optimizer": optimizer.state_dict(),
# }
# save_checkpoint(checkpoint, filename=LOAD_MODEL_FILE)
# import time
# time.sleep(10)
train_fn(train_loader, model, optimizer, loss_fn)
def test(data_loader, gallery_loader, distance, cuda):
batch_time = LogMetric.AverageMeter()
acc = LogMetric.AverageMeter()
meanap = LogMetric.AverageMeter()
end = time.time()
distance.eval()
dist_matrix = []
start = time.time()
with torch.no_grad():
g_gallery = []
target_gallery = []
for j, (g, target) in enumerate(gallery_loader):
if cuda:
g.to(torch.device('cuda'))
g.gdata['std'] = g.gdata['std'].cuda()
target_gallery.append(target)
g_gallery.append(g)
target_gallery = np.array(np.concatenate(target_gallery))
gdata = list(map(lambda g: g.gdata['std'], g_gallery))
g_gallery = dgl.batch(g_gallery)
g_gallery.gdata = {'std': torch.cat(gdata)}
target_query = []
for i, (g, target) in enumerate(data_loader):
# Prepare input data
if cuda:
g.to(torch.device('cuda'))
g.gdata['std'] = g.gdata['std'].cuda()
# Output
d = distance(g, g_gallery, mode='retrieval')
dist_matrix.append(d)
target_query.append(target)
dist_matrix = torch.stack(dist_matrix)
target_query = np.array(np.concatenate(target_query))
target_combined_query = np.unique(target_query)
combined_dist_matrix = torch.zeros(target_combined_query.shape[0],
dist_matrix.shape[1])
for i, kw in enumerate(target_combined_query):
ind = kw == target_query
combined_dist_matrix[i] = dist_matrix[ind].min(0).values
# K-NN classifier
acc.update(
knn_accuracy(combined_dist_matrix,
target_gallery,
target_combined_query,
k=5,
dataset=data_loader.dataset.dataset))
# mAP retrieval
meanap.update(
mean_average_precision(combined_dist_matrix, target_gallery,
target_combined_query))
batch_time.update(time.time() - start)
print(
'* Test Acc {acc.avg:.3f}; mAP {meanap.avg: .3f}; Time x Test {b_time.avg:.3f}'
.format(acc=acc, meanap=meanap, b_time=batch_time))
return acc, meanap
def validate(task_id, model, data, cat_map, results_dict, last_id,
additional_report_cats):
"""
:param additional_report_cats: categories list for additional report. \
dict {report_name: [cat1, cat2, cat3, ...] \
ex) major: ['train', 'horse', 'bird', 'clock', ...], \
minor: ['bicycle', 'potted plant', ....], \
moderate: ['kite', 'bench', 'teddy bear', ...]
"""
results = {}
losses = AverageMeter() # loss (per word decoded)
accuracies = AverageMeter()
class_precisions = Group_AverageMeter()
class_recalls = Group_AverageMeter()
class_f1s = Group_AverageMeter()
overall_precisions = AverageMeter()
overall_recalls = AverageMeter()
overall_f1s = AverageMeter()
mAP = AverageMeter()
criterion = model.criterion
print(
colorful.bold_yellow('Validating Task: ' + str(task_id)).styled_string)
cpu_targets = []
cpu_predicts = []
cpu_probs = []
with torch.no_grad():
for i, (imgs, cats) in enumerate(iter(data)):
# batch_size
batch_size = imgs.shape[0]
# Move to device, if available
imgs = imgs.to(model.device)
cats = cats.to(model.device)
# Forward prop.
predict = model.encoder(imgs)
targets = cats
# Calculate loss
loss = criterion(predict, targets)
loss = loss.mean()
predict = torch.sigmoid(predict)
# for mAP score
cpu_probs.append(predict.cpu())
predict = predict > 0.5 # BCE
total_relevant_slots = targets.sum().data
relevant_predict = (predict * targets.float()).sum().data
acc = relevant_predict / total_relevant_slots
losses.update(loss.item(), batch_size)
accuracies.update(acc, batch_size)
cpu_targets.append(targets.cpu())
cpu_predicts.append(predict.cpu())
cpu_targets = torch.cat(cpu_targets, axis=0)
cpu_predicts = torch.cat(cpu_predicts, axis=0)
cpu_probs = torch.cat(cpu_probs, axis=0)
ncats = cpu_targets.sum(axis=0)
# ignore classes in other tasks
cats_in_task_idx = ncats > 0
ncats = ncats[cats_in_task_idx].tolist()
f1_pc = f1_score_per_class(cpu_targets[:, cats_in_task_idx],
cpu_predicts[:, cats_in_task_idx],
zero_division=0)
precision_pc = precision_score_per_class(
cpu_targets[:, cats_in_task_idx],
cpu_predicts[:, cats_in_task_idx],
zero_division=0)
recall_pc = recall_score_per_class(cpu_targets[:, cats_in_task_idx],
cpu_predicts[:, cats_in_task_idx],
zero_division=0)
f1_oa = f1_score_overall(cpu_targets[:, cats_in_task_idx],
cpu_predicts[:, cats_in_task_idx],
zero_division=0)
precision_oa = precision_score_overall(cpu_targets[:,
cats_in_task_idx],
cpu_predicts[:,
cats_in_task_idx],
zero_division=0)
recall_oa = recall_score_overall(cpu_targets[:, cats_in_task_idx],
cpu_predicts[:, cats_in_task_idx],
zero_division=0)
# record performances
cats_in_task_name = cat_map[cats_in_task_idx].tolist()
class_f1s.update(cats_in_task_name, f1_pc.tolist(), ncats)
class_precisions.update(cats_in_task_name, precision_pc.tolist(),
ncats)
class_recalls.update(cats_in_task_name, recall_pc.tolist(), ncats)
overall_f1s.update(f1_oa.item(), len(cpu_targets))
overall_precisions.update(precision_oa.item(), len(cpu_targets))
overall_recalls.update(recall_oa.item(), len(cpu_targets))
# mAP
mAP.update(
mean_average_precision(cpu_targets[:, cats_in_task_idx],
cpu_probs[:, cats_in_task_idx]))
# for reporting major, moderate, minor cateogory performances
for report_name in additional_report_cats.keys():
reporter = Group_AverageMeter()
# get report category idxes
all_cats = cat_map.tolist()
task_cats = set(cats_in_task_name)
report_cats = task_cats & set(additional_report_cats[report_name])
cats_idx = []
for cat in report_cats:
cats_idx.append(all_cats.index(cat))
report_cats_idx = torch.tensor(cats_idx, dtype=torch.long)
# there are tasks where the min/mod/maj are missing.
if len(report_cats_idx) == 0:
reporter.update(['CP'], [float('NaN')], [1])
reporter.update(['CR'], [float('NaN')], [1])
reporter.update(['CF1'], [float('NaN')], [1])
reporter.update(['OP'], [float('NaN')], [1])
reporter.update(['OR'], [float('NaN')], [1])
reporter.update(['OF1'], [float('NaN')], [1])
reporter.update(['mAP'], [float('NaN')], [1])
# for major, moderate and minor report, total is a meaningless metric.
# mean of CP, CR, CF1, ... is meaningless.
reporter.total.reset()
# add to results
results[report_name] = reporter
continue
# CP, CR, CF1 performance of report_categories.
_class_precision = precision_score_per_class(
cpu_targets[:, report_cats_idx],
cpu_predicts[:, report_cats_idx],
zero_division=0)
_class_recall = recall_score_per_class(
cpu_targets[:, report_cats_idx],
cpu_predicts[:, report_cats_idx],
zero_division=0)
_class_precision = torch.mean(_class_precision)
_class_recall = torch.mean(_class_recall)
# CF1 bias. note that CF1 is not a mean value of categories' f1_score
_class_f1 = ((2*_class_precision*_class_recall)/(_class_precision+_class_recall)) \
if (_class_precision+_class_recall)>0 else torch.tensor([0.])
# OP, OR, OF1 performance of report_categories.
_overall_precision = precision_score_overall(
cpu_targets[:, report_cats_idx],
cpu_predicts[:, report_cats_idx],
zero_division=0)
_overall_recall = recall_score_overall(
cpu_targets[:, report_cats_idx],
cpu_predicts[:, report_cats_idx],
zero_division=0)
_overall_f1 = f1_score_overall(cpu_targets[:, report_cats_idx],
cpu_predicts[:, report_cats_idx],
zero_division=0)
# mAP performance of report_categories.
_mAP = mean_average_precision(cpu_targets[:, report_cats_idx],
cpu_probs[:, report_cats_idx])
reporter.update(['CP'], [_class_precision.item()], [1])
reporter.update(['CR'], [_class_recall.item()], [1])
reporter.update(['CF1'], [_class_f1.item()], [1])
reporter.update(['OP'], [_overall_precision.item()], [1])
reporter.update(['OR'], [_overall_recall.item()], [1])
reporter.update(['OF1'], [_overall_f1.item()], [1])
reporter.update(['mAP'], [_mAP.item()], [1])
# for major, moderate and minor report, total is a meaningless metric.
# mean of CP, CR, CF1, ... is meaningless.
reporter.total.reset()
# add to results
results[report_name] = reporter
# CF1 bias. note that CF1 is not a mean value of categories' f1_score
class_f1s.total.reset()
p_pc, r_pc = torch.mean(precision_pc).item(), torch.mean(
recall_pc).item()
class_f1s.total.update(((2 * p_pc * r_pc) /
(p_pc + r_pc)) if (p_pc + r_pc) > 0 else 0)
# save performances
results['OF1'] = overall_f1s
results['OP'] = overall_precisions
results['OR'] = overall_recalls
results['CF1'] = class_f1s
results['CP'] = class_precisions
results['CR'] = class_recalls
results['losses'] = losses
results['accuracies'] = accuracies
results['mAP'] = mAP
# Forgetting Measure
if int(task_id) == int(last_id) and len(results_dict) > 0:
forget_metrics = ['mAP', 'OF1', 'CF1']
forget = Group_AverageMeter()
Cf1_forget = Group_AverageMeter()
forget_results = {}
per_cat_forget_results = {}
for metric in forget_metrics:
per_metric_results = {}
for task_name, per_task_results in results_dict.items():
if metric == 'CF1':
per_total_lst = []
per_cat_dict = {}
# only up to the 2nd last are used to find the max.
for per_task_result in per_task_results[:-1]:
per_total_lst.append(
per_task_result[metric].total.avg)
for cat, cat_avgmtr in per_task_result[
metric].data.items():
if cat in per_cat_dict:
per_cat_dict[cat].append(cat_avgmtr.avg)
else:
per_cat_dict[cat] = [cat_avgmtr.avg]
final_task_result = per_task_results[-1][
metric].total.avg
max_task_result = max(per_total_lst)
# subtract the very last added and max of the tasks before.
metric_forgot = None
if max_task_result == 0 and final_task_result == 0:
forget_results[metric + '_' + str(task_name)] = 1.0
metric_forgot = 1.0
elif max_task_result == 0 and final_task_result != 0:
metric_forgot = (max_task_result -
final_task_result) / 1
forget_results[metric + '_' +
str(task_name)] = metric_forgot
else:
metric_forgot = (
max_task_result -
final_task_result) / abs(max_task_result)
forget_results[metric + '_' +
str(task_name)] = metric_forgot
for cat, catobj in per_task_results[-1][
metric].data.items():
max_cat_result = max(per_cat_dict[cat])
final_cat_result = catobj.avg
if max_cat_result == 0 and final_cat_result == 0:
per_cat_forget_results[cat] = 1.0
elif max_cat_result == 0 and final_cat_result != 0:
per_cat_forget_results[
cat] = max_cat_result - catobj.avg / 1
else:
per_cat_forget_results[cat] = (max_cat_result \
- catobj.avg)/abs(max_cat_result)
else:
per_metric_lst = []
for per_task_result in per_task_results[:-1]:
per_metric_lst.append(per_task_result[metric].avg)
metric_forgot = None
final_task_result = per_task_results[-1][metric].avg
max_task_result = max(per_metric_lst)
if max_task_result == 0 and final_task_result == 0:
metric_forgot = 1.0
forget_results[metric + '_' +
str(task_name)] = metric_forgot
elif max_task_result == 0 and final_task_result != 0:
metric_forgot = (max_task_result -
final_task_result) / 1
forget_results[metric + '_' +
str(task_name)] = metric_forgot
else:
metric_forgot = (
max_task_result -
final_task_result) / abs(max_task_result)
forget_results[metric + '_' +
str(task_name)] = metric_forgot
for split in ['major', 'moderate', 'minor']:
# check if split results in all NaNs
if math.isnan(
per_task_results[0][split].data[metric].avg):
forget_results[split + '_' + metric + '_' +
str(task_name)] = float('NaN')
continue
per_metric_lst = []
for per_task_result in per_task_results[:-1]:
per_metric_lst.append(
per_task_result[split].data[metric].avg)
final_task_result = per_task_results[-1][split].data[
metric].avg
max_task_result = max(per_metric_lst)
split_forgot = None
if max_task_result == 0 and final_task_result == 0:
split_forgot = 1.0 # forgotten within the first task by majority dominance.
forget_results[split + '_' + metric + '_' +
str(task_name)] = split_forgot
elif max_task_result == 0 and final_task_result != 0:
split_forgot = (max_task_result -
final_task_result) / 1
forget_results[split + '_' + metric + '_' +
str(task_name)] = split_forgot
else:
split_forgot = (max_task_result - final_task_result
) / abs(max_task_result)
forget_results[split + '_' + metric + '_' +
str(task_name)] = split_forgot
if metric + split + 'Overall' in per_metric_results.keys(
):
per_metric_results[metric + split +
'Overall'].append(split_forgot)
else:
per_metric_results[metric + split +
'Overall'] = [split_forgot]
if metric + 'Overall' in per_metric_results.keys():
per_metric_results[metric +
'Overall'].append(metric_forgot)
else:
per_metric_results[metric +
'Overall'] = [metric_forgot]
forget_results[metric + 'Overall'] = average_lst(
per_metric_results[metric + 'Overall'])
forget_results[metric + 'majorOverall'] = average_lst(
per_metric_results[metric + 'majorOverall'])
forget_results[metric + 'moderateOverall'] = average_lst(
per_metric_results[metric + 'moderateOverall'])
forget_results[metric + 'minorOverall'] = average_lst(
per_metric_results[metric + 'minorOverall'])
keys = []
values = []
n = []
for k, v in forget_results.items():
keys.append(k)
values.append(v)
n.append(1)
forget.update(keys, values, n)
keys = []
values = []
n = []
for k, v in per_cat_forget_results.items():
keys.append(k)
values.append(v)
n.append(1)
Cf1_forget.update(keys, values, n)
results['forget'] = forget
results['class_forget'] = Cf1_forget
print(
colorful.bold_cyan(
'LOSS - {loss.avg:.3f}, ACCURACY - {acc.avg:.3f}, RECALL - {rcl.total.avg:.4f},\
PRECISION - {prc.total.avg:.4f}, F1 - {f1.total.avg:.4f}'.format(
loss=losses,
acc=accuracies,
rcl=class_recalls,
prc=class_precisions,
f1=class_f1s)).styled_string)
return results, cpu_targets, cpu_probs
def main():
IMG_DIR = "../datasets/voc/images"
LABEL_DIR = "../datasets/voc/labels"
MAPPING_FILE = "../datasets/voc/100examples.csv"
BATCH_SIZE = 8
NUM_WORKERS = 2
PIN_MEMORY = True
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
LEARNING_RATE = 4e-4
WEIGHT_DECAY = 0.0
EPOCHS = 100
LOAD_MODEL_FILE = "retrained_grad_true.pth.tar"
S = 7
C = 20
B = 2
PRETRAINED = True
print("Using Device:", DEVICE)
class Compose(object):
def __init__(self, transforms):
self.transforms = transforms
def __call__(self, image, bboxes):
for t in self.transforms:
image = t(image)
return image, bboxes
# Types of preprocessing transforms we want to apply
convert_transform = transforms.ToTensor()
resize_transform = transforms.Resize((448, 448))
# normalize_transform = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
transform = Compose([convert_transform, resize_transform])
train_dataset = VOCDataset(MAPPING_FILE, S=S, C=C, B=B, transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR)
train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True)
val_dataset = VOCDataset(MAPPING_FILE, S=S, C=C, B=B, transform=transform, img_dir=IMG_DIR, label_dir=LABEL_DIR)
val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, num_workers=NUM_WORKERS, pin_memory=PIN_MEMORY, shuffle=True, drop_last=True)
model = YoloV1(S=S, B=B, C=C, pretrained=PRETRAINED).to(DEVICE)
for counter, param in enumerate(model.resnet.parameters()):
param.requires_grad = True
model.resnet.fc.requires_grad = True
for counter, param in enumerate(model.parameters()):
print(counter, param.requires_grad)
parameters = [param.numel() for param in model.parameters()]
print("Number of model parameters", sum(parameters))
parameters = [param.numel() for param in model.resnet.parameters()]
print("Number of resnet parameters", sum(parameters))
parameter_list = filter(lambda p: p.requires_grad, model.parameters())
optimizer = optim.Adam(parameter_list, lr=LEARNING_RATE, weight_decay=WEIGHT_DECAY)
yolo_loss = YoloLoss(S=S, C=C, B=B)
writer = SummaryWriter()
current_time = time.time()
mean_loss_list = []
for epoch in range(EPOCHS):
mean_loss = []
pred_boxes, target_boxes = get_bboxes(train_loader, model, iou_threshold=0.5, threshold=0.4, device=DEVICE)
mean_avg_prec = mean_average_precision(pred_boxes, target_boxes, iou_threshold=0.5, box_format="midpoint")
# Save the model
if mean_avg_prec > 0.99 or epoch == (EPOCHS - 1):
checkpoint = {
"state_dict": model.state_dict(),
"optimizer": optimizer.state_dict(),
}
print("=> Saving checkpoint")
torch.save(checkpoint, LOAD_MODEL_FILE)
time.sleep(10)
for batch_idx, (x, y) in enumerate(train_loader):
x, y = x.to(DEVICE), y.to(DEVICE)
y_p = model(x)
# y = torch.flatten(y, start_dim=1, end_dim=3)
y_p = torch.reshape(y_p, (BATCH_SIZE, S, S, C + 5 * B))
loss = yolo_loss(y_p, y)
mean_loss.append(loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
# Calculate average loss
mean_loss = sum(mean_loss)/len(mean_loss)
delta_time = time.time() - current_time
current_time = time.time()
writer.add_scalar("Average Loss: ", mean_loss, epoch)
writer.add_scalar("Mean Average Precision: ", mean_avg_prec, epoch)
writer.add_scalar("Epoch Duration [s]", delta_time, epoch)
print("Epoch:", epoch)
mean_loss_list.append(mean_loss)
writer.close()
