def make_meters(): meters_dict = { 'loss': logger.AvgMeter(), 'acc1': logger.AvgMeter(), 'acc5': logger.AvgMeter(), 'batch_time': logger.AvgMeter(), 'data_time': logger.AvgMeter(), 'epoch_time': logger.SumMeter() } return meters_dict
def construct_logger(name: str) -> logger.Experiment: exp_logger = logger.Experiment(name) exp_logger.add_meters( "test", { "acc1": logger.AvgMeter(), "acc5": logger.AvgMeter(), "acc10": logger.AvgMeter(), "bleu_score": logger.AvgMeter(), "batch_time": logger.AvgMeter(), }, ) return exp_logger
def make_meters(): meters_dict = { 'loss': logger.AvgMeter(), 'loss_a': logger.AvgMeter(), 'loss_q': logger.AvgMeter(), 'batch_time': logger.AvgMeter(), 'data_time': logger.AvgMeter(), 'epoch_time': logger.SumMeter(), 'bleu_score': logger.AvgMeter(), 'acc1': logger.AvgMeter(), 'acc5': logger.AvgMeter(), 'acc10': logger.AvgMeter(), 'dual_loss': logger.AvgMeter(), } return meters_dict
def make_meters(): meters_dict = { "loss": logger.AvgMeter(), "loss_a": logger.AvgMeter(), "loss_q": logger.AvgMeter(), "batch_time": logger.AvgMeter(), "data_time": logger.AvgMeter(), "epoch_time": logger.SumMeter(), "bleu_score": logger.AvgMeter(), "acc1": logger.AvgMeter(), "acc5": logger.AvgMeter(), "acc10": logger.AvgMeter(), "dual_loss": logger.AvgMeter(), } return meters_dict
def evaluate(loader, model, logger, print_freq=10, sampling_num=5): model.eval() model.module.set_testing(True, sample_num=sampling_num) meters = logger.reset_meters('test') meters['Bleu_1'] = logger2.AvgMeter() meters['Bleu_2'] = logger2.AvgMeter() meters['Bleu_3'] = logger2.AvgMeter() meters['Bleu_4'] = logger2.AvgMeter() meters['METEOR'] = logger2.AvgMeter() meters['ROUGE_L'] = logger2.AvgMeter() meters['CIDEr'] = logger2.AvgMeter() results = [] end = time.time() blue_score_all = 0 for i, sample in enumerate(loader): batch_size = sample['visual'].size(0) # measure data loading time input_visual = Variable(sample['visual'].cuda(async=True), volatile=True) input_answer = Variable(sample['answer'].cuda(async=True), volatile=True) target_answer = sample['answer'] input_question = Variable(sample['question'].cuda(async=True), volatile=True) nlg_metrics = calculate_nlg_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word) output_answer, g_answers, g_answers_score, generated_q = model( input_visual, input_question, input_answer) bleu_score = calculate_bleu_score(generated_q.cpu().data, sample['question'], loader.dataset.wid_to_word) acc1, acc5, acc10 = utils.accuracy(output_answer.cpu().data, target_answer, topk=(1, 5, 10)) meters['acc1'].update(acc1[0], n=batch_size) meters['acc5'].update(acc5[0], n=batch_size) meters['acc10'].update(acc10[0], n=batch_size) meters['bleu_score'].update(bleu_score, n=batch_size) meters['Bleu_1'].update(nlg_metrics['Bleu_1'], n=batch_size) meters['Bleu_2'].update(nlg_metrics['Bleu_2'], n=batch_size) meters['Bleu_3'].update(nlg_metrics['Bleu_3'], n=batch_size) meters['Bleu_4'].update(nlg_metrics['Bleu_4'], n=batch_size) meters['METEOR'].update(nlg_metrics['METEOR'], n=batch_size) meters['ROUGE_L'].update(nlg_metrics['ROUGE_L'], n=batch_size) meters['CIDEr'].update(nlg_metrics['CIDEr'], n=batch_size) g_answers = g_answers.cpu().data g_answers_score = g_answers_score.cpu().data for j in range(batch_size): new_question = generated_q.cpu().data[j].tolist() new_answer = g_answers[j] new_answer_score = g_answers_score[j] sampled_aqa = [ [new_question, new_answer, new_answer_score], ] num_result = { 'gt_question': sample['question'][j] [1:].tolist(), #sample['question'][j].numpy(), 'gt_answer': sample['answer'][j], 'augmented_qa': sampled_aqa, } readable_result = { 'gt_question': translate_tokens(sample['question'][j][1:], loader.dataset.wid_to_word), 'gt_answer': loader.dataset.aid_to_ans[sample['answer'][j]], 'augmented_qa': [ [ translate_tokens( item[0], loader.dataset.wid_to_word), # translate question loader.dataset.aid_to_ans[item[1]], # translate answer ] for item in sampled_aqa ], } results.append( { 'image': sample['image'][j], 'numeric_result': num_result, 'readable_result': readable_result }, ) # measure elapsed time meters['batch_time'].update(time.time() - end, n=batch_size) end = time.time() print('* [Evaluation] Result: Acc@1:{acc1.avg:.3f}\t' 'Acc@5:{acc5.avg:.3f}\tAcc@10:{acc10.avg:.3f}\t' 'Time: {batch_time.avg:.3f}\t' 'Bleu_1: {Bleu_1.avg:.5f}\t' 'Bleu_2: {Bleu_2.avg:.5f}\t' 'Bleu_3: {Bleu_3.avg:.5f}\t' 'Bleu_4: {Bleu_4.avg:.5f}\t' 'METEOR: {METEOR.avg:.5f}\t' 'ROUGE_L: {ROUGE_L.avg:.5f}\t ' 'CIDEr: {CIDEr.avg:.5f}\t' 'BLEU: {bleu_score.avg:.5f}'.format(acc1=meters['acc1'], acc5=meters['acc5'], acc10=meters['acc10'], batch_time=meters['batch_time'], Bleu_1=meters['Bleu_1'], Bleu_2=meters['Bleu_2'], Bleu_3=meters['Bleu_3'], Bleu_4=meters['Bleu_4'], METEOR=meters['METEOR'], ROUGE_L=meters['ROUGE_L'], CIDEr=meters['CIDEr'], bleu_score=meters['bleu_score'])) model.module.set_testing(False) return results