def init_losses_and_metrics(self):
"""Initializes loss and metrics for the model based ``losses`` key
and ``metrics`` keys. Automatically called by Pythia internally after
building the model.
"""
self.loss = Losses(self.config.losses)
self.metrics = Metrics(self.config.metrics)
def init_losses_and_metrics(self):
"""Initializes loss and metrics for the model based ``losses`` key
and ``metrics`` keys. Automatically called by Pythia internally after
building the model.
"""
losses = self.config.get("losses", [])
metrics = self.config.get("metrics", [])
if len(losses) == 0:
warnings.warn(
"No losses are defined in model configuration. You are expected "
"to return loss in your return dict from forward.")
if len(metrics) == 0:
warnings.warn(
"No metrics are defined in model configuration. You are expected "
"to return metrics in your return dict from forward.")
self.losses = Losses(losses)
self.metrics = Metrics(metrics)
def evaluate_full(self, loader, use_tqdm=False):
meter = Meter()
# metrics = ['vqamb_map', 'vqamb_f1'] # hardcode metrics for now
metrics = ['accuracy']
# metrics = ['vqamb_f1pt']
print(len(loader))
with torch.no_grad():
self.model.eval()
tot_preds = []
tot_targets = []
tot_ids = []
tot_att_pt = []
tot_att_img = []
tot_bbox_gt = []
tot_bbox_pt = []
tot_bbox_img = []
tot_part = []
# tot_qa_ids = []
for batch in tqdm(loader, disable=not use_tqdm):
report = self._forward_pass(batch)
tot_preds.append(report.scores)
tot_targets.append(report.targets)
# tot_ids.extend(report.qa_id)
# tot_att_pt.append(report.att)
# tot_att_img.append(report.att_img)
# tot_bbox_gt.append(report.gt_bbox)
# tot_bbox_img.append(report.img_bbox)
# tot_bbox_pt.append(report.pt_bbox)
# tot_part.append(report.part)
# tot_bbox_gt.append(report.gt_bbox)
# tot_ptpath.append(report.ptpath)
# tot_bbox_pt.append(report.bboxes)
# tot_bbox_gt.append(report.gt_bbox)
# tot_qa_ids.extend(report.qa_id)
tot_preds = torch.cat(tot_preds, dim=0)
tot_targets = torch.cat(tot_targets, dim=0)
# tot_att_pt = torch.cat(tot_att_pt, dim=0)
# tot_att_img = torch.cat(tot_att_img, dim=0)
# tot_att_pt = torch.cat(tot_att_pt, dim=0)
# tot_bbox_pt = torch.cat(tot_bbox_pt, dim=0)
# tot_bbox_gt = torch.cat(tot_bbox_gt, dim=0)
# tot_bbox_img = torch.cat(tot_bbox_img, dim=0)
# Find bounding box with max attention
# max_att_pt = tot_att_pt.argmax(dim=1)
# max_bbox_pt = tot_bbox_pt[torch.arange(tot_bbox_pt.size(0)), max_att_pt]
'''
torch.save(tot_att_pt, 'tot_pt_att_objpartdev.pt')
torch.save(tot_bbox_pt, 'tot_ptbboxes_objpartdev.pt')
tot_part = sum(tot_part, [])
torch.save(torch.Tensor(tot_part), 'tot_part_objpartdev.pt')
'''
# torch.save(tot_att_pt, 'tot_att_pt_localqafinal.pt')
# torch.save(tot_att_img, 'tot_att_img_pythiaptfinal.pt')
# torch.save(tot_bbox_pt, 'tot_bbox_pt_localqafinal.pt')
# torch.save(tot_bbox_img, 'tot_bbox_img_pythia_ptfinal.pt')
# torch.save(tot_bbox_gt, 'tot_bboxgt_localqafinal.pt')
# torch.save(tot_preds, 'tot_preds_localqafinal.pt')
# torch.save(tot_targets, 'tot_targets_localqafinal.pt')
# torch.save(max_bbox_pt, 'max_pt_bbox_pythiaptfinal.pt')
# torch.save(tot_bbox_gt, 'gt_bbox_pythiaptfinal.pt')
# torch.save(tot_preds, 'tot_preds_localqa.pt')
# torch.save(tot_targets, 'tot_targets_localqa.pt')
# torch.save(tot_ptpath, 'tot_ptpath_vqambnew.pt')
# torch.save(tot_att, 'tot_att_vqambnew.pt')
# tot_qa_ids = torch.Tensor(tot_qa_ids)
# torch.save(tot_qa_ids, 'tot_qa_ids.pt')
model_output = {"scores": tot_preds}
sample = Sample({"targets": tot_targets}) # "qa_index": tot_qa_index}) # "dataset_type": report.dataset_type, "dataset_name": report.dataset_name})
sample_list = SampleList([sample])
sample_list.add_field('dataset_type', report.dataset_type)
sample_list.add_field('dataset_name', report.dataset_name)
metric_fn = Metrics(metrics)
full_met = metric_fn(sample_list, model_output)
self.writer.write(full_met)
if report.dataset_type == 'test':
return
meter.update(full_met)
stop = self.early_stopping(self.current_iteration, meter)
should_break = False
if stop is True:
self.writer.write("Early stopping activated")
should_break = True
self.model.train()
return should_break