def calc_expression(start=0, end=5, k=3, target_set='test'):
matched_label = np.load(f'{target_set}_imgs_label_matching_with_true_target.npy', allow_pickle=True)
rel_load = np.load(f'./{target_set}_relation_extraction.npy', allow_pickle=True)
exps = []
references = []
low_confidence_track = [] # keep track if rsa ever encounter a low confident situation (don't have anything to say or no preference-0probabilty everywhere)
for i in range(start, end):
df = pd.read_csv(os.path.join(data_path,f'refCOCO/{target_set}/attr_tables_with_target_box/attr_{i}.tsv'), encoding='utf-8',sep='\t')
# UNCOMMENT TO SAVE THE REFERENCES OF THE SAME RANGE AS THE PROCESSED IMAGES
with open(os.path.join(data_path,f'refCOCO/{target_set}/labels/lab_{i}.json')) as json_file:
label = json.load(json_file)
refs = [[r] for r in label['ref_sents']]
references.append(refs)
generated_relations = rel_load[i]
### adding LSTM to adjust utterance prior ###
#lstm = tf.keras.models.load_model('three_gram_lstm_loss_2.8469_accuracy_0.4227.h5')
# f = open('word_to_idx_vice_versa.json')
# tokenizer = json.load(f)
# word_to_idx = tokenizer['word_to_idx']
# idx_to_word = tokenizer['idx_to_word']
rsa_agent = RSA(df, generated_relations=generated_relations)#,\
#model=lstm, word_to_idx=word_to_idx, idx_to_word=idx_to_word)
targets = [matched_label[i][j][1] for j in range(min(k, len(matched_label[i])))]
word_lists = []
is_low_confidence = []
for target in targets:
list_utterances, low_confidence = rsa_agent.full_speaker(target)
word_lists.append(list_utterances)
is_low_confidence.append(low_confidence)
expression = [' '.join(word_lists[j][::-1]) for j in range(len(word_lists))]
exps.append(expression)
low_confidence_track.append(is_low_confidence)
if i % 50 == 0:
print(f'finished file {i}')
np.save(f'./data/{target_set}/detectron2_with_target/top{k}_exps_from_{start}_to_{end}.npy',exps)
np.save(f'./data/{target_set}/detectron2_with_target/top{k}_exps_confidence_record_from_{start}_to_{end}.npy',low_confidence_track)
# UNCOMMENT TO SAVE THE REFERENCES OF THE SAME RANGE AS THE PROCESSED IMAGES
np.save(f'./data/{target_set}/detectron2_with_target/references_from_{start}_to_{end}.npy',references)
box_data = df[['box_alias', 'x1','y1','w','h']]
fig,ax = plt.subplots(1)
img = image
# ax.imshow(img)
rng = [i for i in range(len(box_data))]
for i in [4]:#rng[:]:
name, x,y,w,h = list(box_data.iloc[i,:])
ax = draw_box_obj(name,x,y,w,h,img,ax)
print(label['ref_sents'])
bbox = label['bbox'][0]
sentence = label['ref_sents'][0]
fig,ax_true_label = plt.subplots(1)
ax_true_label.imshow(img)
draw_box_obj(sentence,bbox[0],bbox[1],bbox[2],bbox[3],img,ax_true_label)
rsa_agent = RSA(df, generated_relations=generated_relations)
rsa_agent.objects_by_type
# output = rsa_agent.full_speaker('woman-2')
matched_boxes = np.load('train_imgs_label_matching.npy', allow_pickle=True)[21540]
# print(output)
print("######")
for matched in matched_boxes:
_, target,_ = matched
print(target, rsa_agent.full_speaker(target))
print("$$$$$$")
print(rsa_agent.full_speaker('woman-2'))
import pandas as pd
import json
import numpy as np
from helper import *
import argparse
from rsa import RSA
import matplotlib.pyplot as plt
import os
with open('config.json') as config_file:
config = json.load(config_file)
data_path = config['data_path']
file_id = 21540#3278#182
df = pd.read_csv(os.path.join(data_path,f'refCOCO/train/attr_tables/attr_{file_id}.tsv'), encoding='utf-8',sep='\t')
with open(os.path.join(data_path,f'refCOCO/train/labels/lab_{file_id}.json')) as json_file:
label = json.load(json_file)
refs = [[r] for r in label['ref_sents']]
print(refs)
img_id = df['image_id'][0]
filename = os.path.join(data_path, f'refCOCO/train/imgs_by_id/{img_id}.jpg')
print(filename)
image = plt.imread(filename)
rsa_agent = RSA(df)
speech = rsa_agent.full_speaker('woman-1')
print(speech)
