def eval_model(dataset, dataset_loader, standard_encoding, model_encoding, model, trustedEncoder=False, image_group={}): model.eval() print "evaluating model..." if trustedEncoder == False: print "not using trusted encoder. This may take signficantly longer as predictions are converted to other encoding." mx = len(dataset_loader) batches = [] top1 = imSituTensorEvaluation(1, 3, image_group) top5 = imSituTensorEvaluation(5, 3, image_group) for i, (indexes, input, target) in enumerate(dataset_loader): if True or i % 10 == 0: print "batch {} out of {}\r".format(i + 1, mx), input_var = torch.autograd.Variable(input.cuda(), volatile=True) #target_var = torch.autograd.Variable(target.cuda(), volatile = True) (scores, predictions) = model.forward_max(input_var) (s_sorted, idx) = torch.sort(scores, 1, True) if not trustedEncoder: predictions = standard_encoding.to_tensor( model_encoding.to_situation(predictions), False, False) predictions = predictions.view(target.size()[0], standard_encoding.n_verbs(), -1) else: predictions = predictions.data #(s_sorted, idx) = torch.sort(scores, 1, True) top1.add_point(target, predictions, idx.data, dataset.index_image(indexes)) top5.add_point(target, predictions, idx.data, dataset.index_image(indexes)) return (top1, top5)
def eval_file(output, dataset, standard_encoding, image_group): f = open(output) top1 = imSituTensorEvaluation(1, 3, image_group) top5 = imSituTensorEvaluation(5, 3, image_group) curr_image = "" predictions = [] #indexes = torch.LongTensor(range(0,standard_encoding.n_verbs())).view(1,-1) order = [] j = 0 for line in f: tabs = line.split("\t") if curr_image == "": curr_image = tabs[0] elif curr_image != tabs[0]: #print dataset[curr_image] predictions = sorted( predictions, key=lambda x: standard_encoding.v_id[x["verb"]]) #print len(predictions) encoded_predictions = standard_encoding.to_tensor( predictions, False, False).view(1, standard_encoding.n_verbs(), -1) #print encoded_predictions #== -2 indexes = torch.LongTensor(order).view(1, -1) encoded_reference = standard_encoding.to_tensor( [dataset[curr_image]]).view(1, -1) top1.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) top5.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) curr_image = tabs[0] predictions = [] order = [] print("batch {} out of {}\r".format(j, len(dataset))), #if j == 1000: return (top1,top5) j += 1 image_id = tabs[0] verb = tabs[1].strip() roles = {} #print tabs for i in range(2, len(tabs), 2): if tabs[i + 1].strip() == "null": n = "" else: n = tabs[i + 1].strip() roles[tabs[i].strip()] = n #tabs[i+1] predictions.append({"verb": verb, "frames": [roles]}) #print predictions order.append(standard_encoding.v_id[verb]) #last one predictions = sorted(predictions, key=lambda x: standard_encoding.v_id[x["verb"]]) encoded_predictions = standard_encoding.to_tensor( predictions, False, False).view(1, standard_encoding.n_verbs(), -1) indexes = torch.LongTensor(order).view(1, -1) encoded_reference = standard_encoding.to_tensor([dataset[curr_image] ]).view(1, -1) top1.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) top5.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) return (top1, top5)
def eval_file(output, dataset, standard_encoding, image_group): f = open(output) top1 = imSituTensorEvaluation(1, 3, image_group) top5 = imSituTensorEvaluation(5, 3, image_group) curr_image = "" predictions = [] #indexes = torch.LongTensor(range(0,standard_encoding.n_verbs())).view(1,-1) order = [] j = 0 for line in f: tabs = line.split("\t") if curr_image == "": curr_image = tabs[0] elif curr_image != tabs[0]: #print dataset[curr_image] predictions = sorted(predictions, key=lambda x: standard_encoding.v_id[x["verb"]]) #print len(predictions) encoded_predictions = standard_encoding.to_tensor(predictions, False, False).view(1,standard_encoding.n_verbs(), -1) #print encoded_predictions #== -2 indexes = torch.LongTensor(order).view(1, -1) encoded_reference = standard_encoding.to_tensor([dataset[curr_image]]).view(1,-1) top1.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) top5.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) curr_image =tabs[0] predictions = [] order = [] print "batch {} out of {}\r".format(j,len(dataset)), #if j == 1000: return (top1,top5) j+=1 image_id = tabs[0] verb = tabs[1].strip() roles = {} #print tabs for i in range(2, len(tabs), 2): if tabs[i+1].strip() == "null": n = "" else: n = tabs[i+1].strip() roles[tabs[i].strip()] = n#tabs[i+1] predictions.append({"verb":verb,"frames":[roles]}) #print predictions order.append(standard_encoding.v_id[verb]) #last one predictions = sorted(predictions, key=lambda x: standard_encoding.v_id[x["verb"]]) encoded_predictions = standard_encoding.to_tensor(predictions, False, False).view(1,standard_encoding.n_verbs(), -1) indexes = torch.LongTensor(order).view(1, -1) encoded_reference = standard_encoding.to_tensor([dataset[curr_image]]).view(1,-1) top1.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) top5.add_point(encoded_reference, encoded_predictions, indexes, [curr_image]) return (top1,top5)
def eval_model(dataset_loader, encoding, model): model.eval() print "evaluating model..." top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) mx = len(dataset_loader) for i, (index, input, target) in enumerate(dataset_loader): print "{}/{} batches\r".format(i + 1, mx), input_var = torch.autograd.Variable(input.cuda(), volatile=True) target_var = torch.autograd.Variable(target.cuda(), volatile=True) (scores, predictions) = model.forward_max(input_var) (s_sorted, idx) = torch.sort(scores, 1, True) top1.add_point(target, predictions.data, idx.data) top5.add_point(target, predictions.data, idx.data) print "\ndone." return (top1, top5)
def eval_model(dataset_loader, encoding, model): model.eval() print("evaluating model...") top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) mx = len(dataset_loader) for i, (index, input, target) in enumerate(dataset_loader): if i % 50 == 0: print("{}/{} batches\r".format(i + 1, mx)) with torch.no_grad(): input = input.to(device) (scores, predictions) = model.forward_max(input) (s_sorted, idx) = torch.sort(scores, 1, True) top1.add_point(target, predictions.data, idx.data) top5.add_point(target, predictions.data, idx.data) print("\ndone.") return (top1, top5)
def eval_model(dataset, dataset_loader, standard_encoding, model_encoding, model, trustedEncoder = False, image_group = {}): model.eval() print "evaluating model..." if trustedEncoder == False: print "not using trusted encoder. This may take signficantly longer as predictions are converted to other encoding." mx = len(dataset_loader) batches = [] top1 = imSituTensorEvaluation(1, 3, image_group) top5 = imSituTensorEvaluation(5, 3, image_group) for i, (indexes, input, target) in enumerate(dataset_loader): if True or i % 10 == 0: print "batch {} out of {}\r".format(i+1,mx), input_var = torch.autograd.Variable(input.cuda(), volatile = True) #target_var = torch.autograd.Variable(target.cuda(), volatile = True) (scores,predictions) = model.forward_max(input_var) (s_sorted, idx) = torch.sort(scores, 1, True) if not trustedEncoder: predictions = standard_encoding.to_tensor(model_encoding.to_situation(predictions), False, False) predictions = predictions.view(target.size()[0], standard_encoding.n_verbs(), -1) else: predictions = predictions.data #(s_sorted, idx) = torch.sort(scores, 1, True) top1.add_point(target, predictions, idx.data, dataset.index_image(indexes)) top5.add_point(target, predictions, idx.data, dataset.index_image(indexes)) return (top1, top5)
def train_model(max_epoch, eval_frequency, train_loader, dev_loader, model, encoding, optimizer, output_dir, timing=False): model.train() print("Training model...") time_all = time.time() top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0 total_steps = 0 avg_scores = [] for k in range(0, max_epoch): for i, (index, input, target) in enumerate(train_loader): total_steps += 1 t0 = time.time() t1 = time.time() input = input.to(device) # target_var = torch.autograd.Variable(target.to(device)) (_, v, vrn, norm, scores, predictions) = model(input) (s_sorted, idx) = torch.sort(scores, 1, True) # print norm if timing: print("forward time = {}".format(time.time() - t1)) optimizer.zero_grad() t1 = time.time() loss = model.mil_loss(v, vrn, norm, target, 3) if timing: print("loss time = {}".format(time.time() - t1)) t1 = time.time() loss.backward() # print loss if timing: print("backward time = {}".format(time.time() - t1)) optimizer.step() loss_total += loss.item() # score situation t2 = time.time() top1.add_point(target, predictions.data, idx.data) top5.add_point(target, predictions.data, idx.data) if timing: print("eval time = {}".format(time.time() - t2)) if timing: print("batch time = {}".format(time.time() - t0)) if total_steps % print_freq == 0: top1_a = top1.get_average_results() verb_1 = top1_a["verb"] value_1 = top1_a["value"] print( "Epoch {}/{}, batch {}/{} | verb_1: {:.2f}, value_1: {:.2f}, loss={:.5f}, avg loss={:.5f}, batch time = {:.2f}" .format( k + 1, max_epoch, i + 1, len(train_loader), verb_1, value_1, loss.item(), loss_total / ((total_steps) % (eval_frequency * len(train_loader) + 1)), (time.time() - time_all))) time_all = time.time() if (k + 1) % eval_frequency == 0: print("eval...") etime = time.time() (top1, top5) = eval_model(dev_loader, encoding, model) model.train() print("... done after {:.2f} s".format(time.time() - etime)) top1_a = top1.get_average_results() top5_a = top5.get_average_results() avg_score = top1_a["verb"] + top1_a["value"] + top1_a["value-all"] + top5_a["verb"] + \ top5_a["value"] + top5_a["value-all"] + \ top5_a["value*"] + top5_a["value-all*"] avg_score /= 8 print("Epoch {} average :{:.2f} {} {}".format( k + 1, avg_score * 100, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"))) avg_scores.append(avg_score) maxv = max(avg_scores) if maxv == avg_scores[-1]: torch.save(model.state_dict(), output_dir + "/{0}.model".format(maxv)) print("new best model saved! {0}".format(maxv)) top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0
def train_model(max_epoch, eval_frequency, train_loader, dev_loader, traindev_loader, model, encoding, optimizer, save_dir, timing=False): model.train() time_all = time.time() pmodel = torch.nn.DataParallel(model, device_ids=device_array) top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0 print_freq = 50 total_steps = 0 avg_scores = [] for k in range(0, max_epoch): for i, (index, input, target) in enumerate(train_loader): total_steps += 1 t0 = time.time() t1 = time.time() input_var = torch.autograd.Variable(input.cuda()) target_var = torch.autograd.Variable(target.cuda()) (_, v, vrn, norm, scores, predictions) = pmodel(input_var) (s_sorted, idx) = torch.sort(scores, 1, True) #print norm if timing: print("forward time = {}".format(time.time() - t1)) optimizer.zero_grad() t1 = time.time() loss = model.mil_loss(v, vrn, norm, target, 3) if timing: print("loss time = {}".format(time.time() - t1)) t1 = time.time() loss.backward() #print loss if timing: print("backward time = {}".format(time.time() - t1)) optimizer.step() loss_total += loss.data[0] #score situation t2 = time.time() top1.add_point(target, predictions.data, idx.data) top5.add_point(target, predictions.data, idx.data) if timing: print("eval time = {}".format(time.time() - t2)) if timing: print("batch time = {}".format(time.time() - t0)) if total_steps % print_freq == 0: top1_a = top1.get_average_results() top5_a = top5.get_average_results() print( "{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}, batch time = {:.2f}" .format(total_steps - 1, k, i, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"), loss.data[0], loss_total / ((total_steps - 1) % eval_frequency), (time.time() - time_all) / ((total_steps - 1) % eval_frequency))) if total_steps % eval_frequency == 0: print("eval...") etime = time.time() (top1, top5) = eval_model(dev_loader, encoding, model) model.train() print("... done after {:.2f} s".format(time.time() - etime)) top1_a = top1.get_average_results() top5_a = top5.get_average_results() avg_score = top1_a["verb"] + top1_a["value"] + top1_a[ "value-all"] + top5_a["verb"] + top5_a["value"] + top5_a[ "value-all"] + top5_a["value*"] + top5_a["value-all*"] avg_score /= 8 print("Dev {} average :{:.2f} {} {}".format( total_steps - 1, avg_score * 100, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"))) avg_scores.append(avg_score) maxv = max(avg_scores) if maxv == avg_scores[-1]: torch.save(model.state_dict(), save_dir + "/{0}.model".format(maxv)) print("new best model saved! {0}".format(maxv)) #traindevset (top1, top5) = eval_model(traindev_loader, encoding, model) model.train() print("... done after {:.2f} s".format(time.time() - etime)) top1_a = top1.get_average_results() top5_a = top5.get_average_results() avg_score = top1_a["verb"] + top1_a["value"] + top1_a[ "value-all"] + top5_a["verb"] + top5_a["value"] + top5_a[ "value-all"] + top5_a["value*"] + top5_a["value-all*"] avg_score /= 8 print("TRAINDev {} average :{:.2f} {} {}".format( total_steps - 1, avg_score * 100, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"))) top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0 time_all = time.time()
def train_model(max_epoch, eval_frequency, train_loader, dev_loader, model, encoding, optimizer, save_dir, device_array, args, timing=False): if args.use_wandb: wandb.init(project='imSitu_YYS_V1', name='CRF', config=args) model.train() time_all = time.time() pmodel = torch.nn.DataParallel(model, device_ids=device_array) top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0 print_freq = 10 total_steps = 0 avg_scores = [] for k in range(0, max_epoch): for i, (index, input_var, target) in enumerate(train_loader): total_steps += 1 t0 = time.time() t1 = time.time() (_, v, vrn, norm, scores, predictions) = pmodel(input_var) (s_sorted, idx) = torch.sort(scores, 1, True) # print norm if timing: print("forward time = {}".format(time.time() - t1)) optimizer.zero_grad() t1 = time.time() loss = model.mil_loss(v, vrn, norm, target, 3) if timing: print("loss time = {}".format(time.time() - t1)) t1 = time.time() loss.backward() # print loss if timing: print("backward time = {}".format(time.time() - t1)) optimizer.step() loss_total += loss.item() # score situation t2 = time.time() top1.add_point(target, predictions.data, idx.data) top5.add_point(target, predictions.data, idx.data) if timing: print("eval time = {}".format(time.time() - t2)) if timing: print("batch time = {}".format(time.time() - t0)) if total_steps % print_freq == 0: top1_a = top1.get_average_results() top5_a = top5.get_average_results() print( "{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}, batch time = {:.2f}" .format(total_steps - 1, k, i, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"), loss.item(), loss_total / ((total_steps - 1) % eval_frequency), (time.time() - time_all) / ((total_steps - 1) % eval_frequency))) if args.use_wandb: wandb.log( { 'train/loss': loss.item(), 'train/avg_loss': loss_total / ((total_steps - 1) % eval_frequency), }, step=total_steps) wandb.log( { 'train/top1-{}'.format(k): v for k, v in top1_a.items() }, step=total_steps) wandb.log( { 'train/top5-{}'.format(k): v for k, v in top5_a.items() }, step=total_steps) if total_steps % eval_frequency == 0: print("eval...") etime = time.time() (top1, top5) = eval_model(dev_loader, encoding, model, device_array[0]) model.train() print("... done after {:.2f} s".format(time.time() - etime)) top1_a = top1.get_average_results() top5_a = top5.get_average_results() avg_score = top1_a["verb"] + top1_a["value"] + top1_a["value-all"] + top5_a["verb"] + \ top5_a["value"] + top5_a["value-all"] + \ top5_a["value*"] + top5_a["value-all*"] avg_score /= 8 print("Dev {} average :{:.2f} {} {}".format( total_steps - 1, avg_score * 100, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-"))) if args.use_wandb: wandb.log({ 'eval/avg_score': avg_score, }, step=total_steps) wandb.log( { 'eval/top1-{}'.format(k): v for k, v in top1_a.items() }, step=total_steps) wandb.log( { 'eval/top5-{}'.format(k): v for k, v in top5_a.items() }, step=total_steps) avg_scores.append(avg_score) maxv = max(avg_scores) if maxv == avg_scores[-1]: torch.save(model.state_dict(), save_dir + "/best.model".format(maxv)) print("new best model saved! {0}".format(maxv)) top1 = imSituTensorEvaluation(1, 3, encoding) top5 = imSituTensorEvaluation(5, 3, encoding) loss_total = 0 time_all = time.time()
def lagrange_with_margins(margin, constraints, encoder, model, dataset, loader, arg_to_v, all_man_idx, all_woman_idx, wordmap2, arg_idx_map, vrn_map, cons_verbs): eta = 0.1 max_epoch = 2 lambdas = {item: [0, 0] for item in constraints} training_ratio, train_collect_agents = myutils.get_training_gender_ratio( training_file, words_file, num_gender) training_ratio_id = {} for verb in training_ratio: v_id = encoder.v_id[verb] training_ratio_id[v_id] = training_ratio[verb] mx = len(loader) top1 = [] pred_agents = {} print("Initial inferencing.") for batch_idx, (index, input, target) in enumerate(loader): if batch_idx % 100 == 0: print("Batch: {}/{}".format(batch_idx, mx)) vrn_potential_tmp = torch.load(vrn_potential_dir + "%d" % (batch_idx + 1)) v_potential_tmp = torch.load(v_potential_dir + "%d" % (batch_idx + 1)).to(device) for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.to(device) _top1, _pred_agents, _, _ = myutils.inference_step(encoder, model, wordmap2, vrn_potential_tmp, v_potential_tmp, vrn_map, cons_verbs, isPR=0) top1.append(_top1) for v_id in _pred_agents: if v_id in pred_agents: pred_agents[v_id][0] += _pred_agents[v_id][0] pred_agents[v_id][1] += _pred_agents[v_id][1] else: pred_agents[v_id] = _pred_agents[v_id] print("Starting Lagrangian part") for epoch in range(max_epoch): print("Epoch: {}/{}".format(epoch, max_epoch)) count = 0 error = {item: [0, 0] for item in constraints} results = [] t0 = time.time() # update lambdas and error for k in constraints: if k in pred_agents: lambdas[k][0] += eta * constraints[k][0][0] * pred_agents[k][0] lambdas[k][0] += eta * constraints[k][0][1] * pred_agents[k][1] error[k][0] += constraints[k][0][0] * pred_agents[k][0] error[k][0] += constraints[k][0][1] * pred_agents[k][1] lambdas[k][1] += eta * constraints[k][1][0] * pred_agents[k][0] lambdas[k][1] += eta * constraints[k][1][1] * pred_agents[k][1] error[k][1] += constraints[k][1][0] * pred_agents[k][0] error[k][1] += constraints[k][1][1] * pred_agents[k][1] for k in lambdas: for i in range(2): if lambdas[k][i] <= 0: lambdas[k][i] = 0 for k in error: for i in range(2): if error[k][i] > 0: count += 1 # update potential scores mx = len(loader) top1_before = copy.deepcopy(top1) top1 = [] pred_agents_before = copy.deepcopy(pred_agents) pred_agents = {} idx_sorted = 0 predictions = 0 target = 0 print("Start new inference.") for batch_idx, (index, input, _target) in enumerate(loader): if batch_idx % 100 == 0: print("Batch: {}/{}".format(batch_idx, mx)) vrn_potential_tmp = torch.load(vrn_potential_dir + "%d" % (batch_idx + 1)) v_potential_tmp = torch.load(v_potential_dir + "%d" % (batch_idx + 1)).to(device) for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.to(device) n_ins = vrn_potential_tmp[0].size()[0] for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.transpose(0, 2) # vrn_potential: [(50, 1125, n_ins), (100, 518, n_ins), (283, 145, n_ins)] for arg_id in all_man_idx: v_id, n_localid, noun = arg_to_v[arg_id] if v_id not in constraints: continue if lambdas[v_id][0] == 0 and lambdas[v_id][1] == 0: continue r_id = encoder.r_id["agent"] vr_id = encoder.vr_id[(v_id, r_id)] vr_localid = model.vr_id_to_local[vr_id] if vr_localid == 0: print("Wrong! v_id is", v_id) sys.exit() if vr_localid > splits_offset[2]: split_id = 2 vr_split_localid = vr_localid - splits_offset[2] - 1 elif vr_localid > splits_offset[1]: split_id = 1 vr_split_localid = vr_localid - splits_offset[1] - 1 else: split_id = 0 vr_split_localid = vr_localid - 1 vrn_potential_tmp[split_id][n_localid][ vr_split_localid] -= lambdas[v_id][0] * constraints[v_id][ 0][0] vrn_potential_tmp[split_id][n_localid][ vr_split_localid] -= lambdas[v_id][1] * constraints[v_id][ 1][0] for arg_id in all_woman_idx: v_id, n_localid, noun = arg_to_v[arg_id] if v_id not in constraints: continue if lambdas[v_id][0] == 0 and lambdas[v_id][1] == 0: continue r_id = encoder.r_id["agent"] vr_id = encoder.vr_id[(v_id, r_id)] vr_localid = model.vr_id_to_local[vr_id] if vr_localid > splits_offset[2]: split_id = 2 vr_split_localid = vr_localid - splits_offset[2] - 1 elif vr_localid > splits_offset[1]: split_id = 1 vr_split_localid = vr_localid - splits_offset[1] - 1 else: split_id = 0 vr_split_localid = vr_localid - 1 vrn_potential_tmp[split_id][n_localid][ vr_split_localid] -= lambdas[v_id][0] * constraints[v_id][ 0][1] vrn_potential_tmp[split_id][n_localid][ vr_split_localid] -= lambdas[v_id][1] * constraints[v_id][ 1][1] for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.transpose(0, 2) _top1, _pred_agents, _idx_sorted, _predictions = myutils.inference_step( encoder, model, wordmap2, vrn_potential_tmp, v_potential_tmp, vrn_map, cons_verbs, isPR=0) top1.append(_top1) for v_id in _pred_agents: if v_id in pred_agents: pred_agents[v_id][0] += _pred_agents[v_id][0] pred_agents[v_id][1] += _pred_agents[v_id][1] else: pred_agents[v_id] = _pred_agents[v_id] if batch_idx == 0: idx_sorted = _idx_sorted predictions = _predictions target = _target else: idx_sorted = torch.cat((idx_sorted, _idx_sorted)) predictions = torch.cat((predictions, _predictions)) target = torch.cat((target, _target)) for k in pred_agents: if k in pred_agents_before: if pred_agents[k] != pred_agents_before[k] and k == 243: print("v_id:{}, before:{}, after:{}, tr:{}, constraint:{}". format(k, pred_agents_before[k], pred_agents[k], training_ratio_id[k], constraints[k])) if epoch % 10 == 0 or epoch == max_epoch - 1: top1_eval = imSituTensorEvaluation(1, 3, encoder) top1_eval.add_point(target, predictions.data, idx_sorted.data) acc1 = top1_eval.get_average_results()["value"] print("%s-epoch, acc is: " % (epoch), acc1) results.append([epoch, count, acc1]) print("Time for one epoch:", time.time() - t0) print( "%s-th epoch, number of times that constrints are not satisfied:" % (epoch), count) if count == 0: break myutils.save_iterations(save_iteration + "_margin_" + str(margin), results) myutils.save_lambdas(save_lambda + "_margin_" + str(margin), lambdas) return lambdas
def posterior_regularization(constraints, dataset, loader, encoding, cons_verbs, agent_verbs, model, word_map, wordmap2, output_index, all_man_idx, all_woman_idx, arg_to_v, is_dev): training_ratio, train_collect_agents = myutils.get_training_gender_ratio( training_file, words_file, num_gender) acc, pred_agents_ori = myutils.get_ori_gender_ratio( loader, encoding, model, wordmap2, output_index, cons_verbs) # inference results before using PR print("TEST acc is:{}.".format(acc)) # apply gradient descent Adam to find the lambda max_epoch = 1 learning_rate = 0.01 lr_decay = 0.998 alpha = 0.9 beta = 0.999 epsilon = 1e-6 inference_freq = 1 # frequency to inference print_freq = 1 # frequency to print ratio and lambdas update_freq = 1 # frequency to update lambdas # other initialization results = [] mx = len(loader) batch_num = 0 # number of batches that have been used in total min_error = 300 batch_bst = 0 lambdas_bst = {item: [0.0, 0.0] for item in constraints} lambdas = {item: [0.0, 0.0] for item in constraints } # lambdas: {v_id: [lambda_left, lambda_right]} first_order_sum = {item: [0.0, 0.0] for item in constraints} second_order_sum = {item: [0.0, 0.0] for item in constraints} M = len(constraints) N = len(dataset) # Get results of bias amplification before PR. res, ori_ratio_PR = myutils.get_gender_ratio_res_PR( cons_verbs, num_gender, words_file, training_file, encoding, model, loader, all_man_idx, all_woman_idx, output_index, wordmap2, arg_to_v, lambdas_bst, constraints) print("=== Before calibrating ===") mystat.get_violated_verbs(res, margin) mystat.get_bias_score(res, margin) # Start PR process print("Start PR process.") gradientZ = {item: [0.0, 0.0] for item in constraints} for epoch in range(max_epoch): # max_epoch can't be too large for batch_idx, (index, input, _target) in enumerate(loader): batch_num += 1 print("Epoch:{}, batch_num:{}".format(epoch, batch_num)) vrn_grouped = torch.load(vrn_grouped_dir + "%d" % (batch_idx + 1)).cpu().detach().numpy() v_prob = torch.load(v_logProb_dir + "%d" % (batch_idx + 1)).cpu().detach().numpy() # compute gradientZ sum0_tot = 0.0 sum1_tot = {item: [0.0, 0.0] for item in constraints} n_batch = len(v_prob) for ins_id in range(n_batch): # print ("------") # print ("ins_id:", ins_id) # if ins_id % 10 == 0: # print ("batch:{}, ins_id:{}".format(batch_num, ins_id)) sum0_tot = 0.0 sum1_tot = {item: [0.0, 0.0] for item in constraints} for v_id in cons_verbs: sum0 = 0.0 sum1 = {item: [0.0, 0.0] for item in constraints} r_id = encoding.r_id["agent"] vr_id = encoding.vr_id[(v_id, r_id)] vr_localid = model.vr_id_to_local[vr_id] r_localid = model.vr_v[vr_localid][1] for n_localid in range(len(encoding.vr_id_n[vr_id])): # an enumerate of v_g n_id = encoding.vr_id_n[vr_id][n_localid] n = encoding.id_n[n_id] temp = 0 # lambda dot feature function # constraints: {v_id: ((m_c1, f_c1), (m_c2, f_c2), val)} if n != "": if wordmap2[n] == "m": temp -= constraints[v_id][0][0] * lambdas[ v_id][0] temp -= constraints[v_id][1][0] * lambdas[ v_id][1] elif wordmap2[n] == "f": temp -= constraints[v_id][0][1] * lambdas[ v_id][0] temp -= constraints[v_id][1][1] * lambdas[ v_id][1] sum0 += ( math.exp(temp) * (vrn_grouped[ins_id][v_id][r_localid][n_localid])) if n != "": if wordmap2[n] == "m": sum1[v_id][0] -= constraints[v_id][0][ 0] * math.exp(temp) * (vrn_grouped[ins_id][ v_id][r_localid][n_localid]) sum1[v_id][1] -= constraints[v_id][1][ 0] * math.exp(temp) * (vrn_grouped[ins_id][ v_id][r_localid][n_localid]) elif wordmap2[n] == "f": sum1[v_id][0] -= constraints[v_id][0][ 1] * math.exp(temp) * (vrn_grouped[ins_id][ v_id][r_localid][n_localid]) sum1[v_id][1] -= constraints[v_id][1][ 1] * math.exp(temp) * (vrn_grouped[ins_id][ v_id][r_localid][n_localid]) sum0 *= math.exp(v_prob[ins_id][v_id]) sum1[v_id][0] *= math.exp(v_prob[ins_id][v_id]) sum1[v_id][1] *= math.exp(v_prob[ins_id][v_id]) sum0_tot += sum0 sum1_tot[v_id][0] += sum1[v_id][0] sum1_tot[v_id][1] += sum1[v_id][1] for v_id_ordi in range(num_verb): if v_id_ordi not in cons_verbs: sum0_tot += math.exp(v_prob[ins_id][v_id_ordi]) for v_id in cons_verbs: gradientZ[v_id][0] += (sum1_tot[v_id][0] / sum0_tot) gradientZ[v_id][1] += (sum1_tot[v_id][1] / sum0_tot) # update lambdas if batch_num % update_freq != 0: continue count = 0 # print ("update lambdas!") for cons_id in constraints: for direc in range(0, 2): first_order_sum[cons_id][ direc] = first_order_sum[cons_id][direc] * alpha + ( 1.0 - alpha) * gradientZ[cons_id][direc] second_order_sum[cons_id][direc] = second_order_sum[ cons_id][direc] * beta + (1.0 - beta) * gradientZ[ cons_id][direc] * gradientZ[cons_id][direc] first_order_mean = first_order_sum[cons_id][direc] / ( 1.0 - pow(alpha, batch_num)) second_order_mean = second_order_sum[cons_id][direc] / ( 1.0 - pow(beta, batch_num)) lambdas[cons_id][direc] -= first_order_mean / ( math.sqrt(second_order_mean) + epsilon) * learning_rate if lambdas[cons_id][direc] < 0: lambdas[cons_id][direc] = 0 if lambdas[cons_id][direc] > 0: count += 1 gradientZ = {item: [0.0, 0.0] for item in constraints} learning_rate *= lr_decay # Inference once, using lambdas. if batch_num % inference_freq == 0: print("=== inference begin ===") pred_agents = {} predictions = [] target = [] idx_sorted = [] for batch_idx_, (index_, input, _target_) in enumerate(loader): if batch_idx_ % 100 == 0: print("Test batch: {}/{}".format(batch_idx_, mx)) vrn_potential_tmp = torch.load(vrn_logProb_dir + "%d" % (batch_idx_ + 1)) v_potential_tmp = torch.load(v_logProb_dir + "%d" % (batch_idx_ + 1)).to(device) for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.to(device) n_ins = vrn_potential_tmp[0].size()[0] for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.transpose(0, 2) # vrn_potential: [(50, 1125, n_ins), (100, 518, n_ins), (283, 145, n_ins)] # update vrn_after using lambdas for arg_id in all_man_idx: v_id, n_localid, noun = arg_to_v[arg_id] if v_id not in constraints: continue if lambdas[v_id][0] == 0 and lambdas[v_id][1] == 0: continue split_id, n_localid, vr_split_localid = myutils.arg_id_to_more( arg_id, arg_to_v, constraints, lambdas, encoding, model) temp = -lambdas[v_id][0] * constraints[v_id][0][ 0] - lambdas[v_id][1] * constraints[v_id][1][0] vrn_potential_tmp[split_id][n_localid][ vr_split_localid] += temp for arg_id in all_woman_idx: v_id, n_localid, noun = arg_to_v[arg_id] if v_id not in constraints: continue if lambdas[v_id][0] == 0 and lambdas[v_id][1] == 0: continue split_id, n_localid, vr_split_localid = myutils.arg_id_to_more( arg_id, arg_to_v, constraints, lambdas, encoding, model) temp = -lambdas[v_id][0] * constraints[v_id][0][ 1] - lambdas[v_id][1] * constraints[v_id][1][1] vrn_potential_tmp[split_id][n_localid][ vr_split_localid] += temp for vrn_idx, _vrn in enumerate(vrn_potential_tmp): vrn_potential_tmp[vrn_idx] = _vrn.transpose(0, 2) _top1, _pred_agents, _idx_sorted, _predictions = myutils.inference_step( encoding, model, wordmap2, vrn_potential_tmp, v_potential_tmp, output_index, cons_verbs) if len(_pred_agents) != 0: for v_id in _pred_agents: if v_id in pred_agents: pred_agents[v_id][0] += _pred_agents[v_id][0] pred_agents[v_id][1] += _pred_agents[v_id][1] else: pred_agents[v_id] = [0, 0] pred_agents[v_id][0] = _pred_agents[v_id][0] pred_agents[v_id][1] = _pred_agents[v_id][1] if batch_idx_ == 0: idx_sorted = _idx_sorted predictions = _predictions target = _target_ else: idx_sorted = torch.cat((idx_sorted, _idx_sorted)) predictions = torch.cat((predictions, _predictions)) target = torch.cat((target, _target_)) top1_eval = imSituTensorEvaluation(1, 3, encoding) top1_eval.add_point(target, predictions.data, idx_sorted.data) acc1 = top1_eval.get_average_results()["value"] print("{}-batch, acc is:{}, {} constraints are not satisfied". format(batch_num, acc1, count)) results.append([epoch, count, acc1]) # Ratio here is got from inference results, not probability. In fact, we want both for analysis. after_ratio = myutils.get_pred_gender_ratio(pred_agents, 1) ori_ratio = myutils.get_pred_gender_ratio(pred_agents_ori, 1) _, after_ratio_PR = myutils.get_gender_ratio_res_PR( cons_verbs, num_gender, words_file, training_file, encoding, model, loader, all_man_idx, all_woman_idx, output_index, wordmap2, arg_to_v, lambdas, constraints) if batch_num % print_freq == 0: for v_id in after_ratio_PR: if not ori_ratio.__contains__(v_id): ori_ratio[v_id] = 0 if not after_ratio.__contains__(v_id): after_ratio[v_id] = 0 print( "{} {}:\ttrain:{:f}\tbefore_prob:{:f}\tafter_prob:{:f}\tbefore_inf:{:f}\tafter_inf:{:f}\tlambdas:{}" .format(v_id, encoding.id_v[v_id], training_ratio[encoding.id_v[v_id]], ori_ratio_PR[v_id], after_ratio_PR[v_id], ori_ratio[v_id], after_ratio[v_id], lambdas[v_id])) golden_verbs_df = pd.DataFrame([verb for verb in cons_verbs], columns=['verb_id']) train_df = pd.DataFrame(training_ratio.items(), columns=["verb", "training_ratio"]) ori_df = pd.DataFrame(ori_ratio.items(), columns=['verb_id', 'bef_ratio']) after_df = pd.DataFrame(after_ratio.items(), columns=['verb_id', 'after_ratio']) ori_df_PR = pd.DataFrame(ori_ratio_PR.items(), columns=['verb_id', 'bef_ratio_PR']) after_df_PR = pd.DataFrame( after_ratio_PR.items(), columns=['verb_id', 'after_ratio_PR']) tmp = ori_df.merge(after_df).merge(ori_df_PR).merge( after_df_PR) train_df["verb_id"] = train_df["verb"].apply( lambda x: encoding.v_id[x]) com_df = train_df.merge(tmp) res0 = com_df.sort_values(by=['training_ratio'], ascending=1) res0['bef_diff'] = res0['training_ratio'] - res0[ 'bef_ratio'] # training_ratio - bef_ratio res0['after_diff'] = res0['training_ratio'] - res0[ 'after_ratio'] # training_ratio - after_ratio res0['bef_diff_PR'] = res0['training_ratio'] - res0[ 'bef_ratio_PR'] # training_ratio - bef_ratio res0['after_diff_PR'] = res0['training_ratio'] - res0[ 'after_ratio_PR'] # training_ratio - after_ratio resx = res0.merge(golden_verbs_df) resx.sort_values(by=['training_ratio'], ascending=1, inplace=True) del resx['verb_id'] resx.reset_index(inplace=True, drop=True) before_error, after_error, before_error_PR, after_error_PR = mystat.get_violated_verbs( resx, margin) # To find in which batch we get the best performance on reducing bias if after_error_PR < min_error: min_error = after_error_PR res_bst = resx lambdas_bst = lambdas batch_bst = batch_num print( "**Epoch:{}, Batch_num:{}, Best batch:{}, min_error:{}**". format(epoch, batch_num, batch_bst, min_error)) print("|____inference end____|\n") print("**Stop batch:{}. Best batch:{}, min_error:{}**".format( batch_num, batch_bst, min_error)) # show results and calculate bias score. print("=== After calibrating ===") myplt.plot_tr_ax_gender(res_bst, margin, filename="Gender ratio predicted") mystat.get_violated_verbs(res_bst, margin) mystat.get_bias_score(res_bst, margin)
def train_model(max_epoch, eval_frequency, train_loader, dev_loader, model, encoding, optimizer, save_dir, timing = False): model.train() time_all = time.time() #pmodel = torch.nn.DataParallel(model, device_ids=device_array) top1 = imSituTensorEvaluation(1, 1, encoding) top5 = imSituTensorEvaluation(5, 1, encoding) loss_total = 0 print_freq = 10 total_steps = 0 avg_scores = [] for k in range(0,max_epoch): for i, (index, input,im_info, target) in enumerate(train_loader): no_valid_vrole = True for vector in target[0]: if vector[0] != -1: no_valid_vrole = False if no_valid_vrole: continue total_steps += 1 t0 = time.time() t1 = time.time() #input_var = torch.autograd.Variable(input.cuda()) #target_var = torch.autograd.Variable(target.cuda()) #todo : fix data parallelism (_,v,vrn,norm,scores,predictions) = model.forward(input, im_info) (s_sorted, idx) = torch.sort(scores, 1, True) #print norm if timing : print "forward time = {}".format(time.time() - t1) optimizer.zero_grad() t1 = time.time() #print('all' , target[0].size()) loss = model.mil_loss(v,vrn,norm, target[0], 1) if timing: print "loss time = {}".format(time.time() - t1) t1 = time.time() loss.backward() #print loss if timing: print "backward time = {}".format(time.time() - t1) optimizer.step() loss_total += loss.data[0] #score situation t2 = time.time() top1.add_point(target[0], predictions.data, idx.data) top5.add_point(target[0], predictions.data, idx.data) #print('top results', top1.score_cards, top5.score_cards) if timing: print "eval time = {}".format(time.time() - t2) if timing: print "batch time = {}".format(time.time() - t0) if total_steps % print_freq == 0: top1_a = top1.get_average_results() top5_a = top5.get_average_results() print "{},{},{}, {} , {}, loss = {:.2f}, avg loss = {:.2f}, batch time = {:.2f}".format(total_steps-1,k,i, format_dict(top1_a, "{:.2f}", "1-"), format_dict(top5_a,"{:.2f}","5-"), loss.data[0], loss_total / ((total_steps-1)%eval_frequency) , (time.time() - time_all)/ ((total_steps-1)%eval_frequency)) if total_steps % eval_frequency == 0: print "eval..." etime = time.time() (top1, top5) = eval_model(dev_loader, encoding, model) model.train() print "... done after {:.2f} s".format(time.time() - etime) top1_a = top1.get_average_results() top5_a = top5.get_average_results() avg_score = top1_a["verb"] + top1_a["value"] + top1_a["value-all"] + top5_a["verb"] + top5_a["value"] + top5_a["value-all"] + top5_a["value*"] + top5_a["value-all*"] avg_score /= 8 print "Dev {} average :{:.2f} {} {}".format(total_steps-1, avg_score*100, format_dict(top1_a,"{:.2f}", "1-"), format_dict(top5_a, "{:.2f}", "5-")) avg_scores.append(avg_score) maxv = max(avg_scores) if maxv == avg_scores[-1]: torch.save(model.state_dict(), save_dir + "/{0}.model".format(maxv)) print "new best model saved! {0}".format(maxv) top1 = imSituTensorEvaluation(1, 1, encoding) top5 = imSituTensorEvaluation(5, 1, encoding) loss_total = 0 time_all = time.time()