def prepare(self):
""" this is a RequestHandler class method; called on packet"""
if self.request.headers.get("Content-Type",
"").startswith("application/json"):
self.json_args = json.loads(self.request.body)
self.json_args['target'] = get_img_path(self.json_args['target'])
# Sort because apparently the updating depends on context order?
self.json_args['context'] = sorted(
[get_img_path(img) for img in self.json_args['context']])
else:
self.json_args = None
def sample_img_from_tag(self, img_tag):
"""
Samples img (and meta-data) from provided coco category tag
"""
cat_id = coco.getCatIds(catNms=img_tag)
img_id = np.random.choice(coco.getImgIds(catIds=cat_id), 1)
img_path = utils.get_img_path(
coco.loadImgs(int(img_id))[0]['file_name'])
return utils.load_image(img_path), img_path, img_tag
def main_memory(args):
path = '../data/model_output/listener_lesions.csv'
writer = EfficiencyWriter(args, path)
# init listener model
listener = AdaptiveAgent(args)
listener.reduction_history_window = 'complete'
grid = construct_context_grid(args)
for ctx in grid:
print("\n------gameid: {}, sample_num: {}, loss: {}, handle-oov: {}"
.format(ctx['gameid'], ctx['sample_num'], ctx['loss'], ctx['handleOOV']))
# reset speaker & listener to pretrained settings
listener.reset_to_initialization(ctx['dirs']) # setting context
listener.loss = ctx['loss']
listener.dataset_type = ctx['ds_type']
listener.history = []
for datum in ctx['speaker_data'] :
rep_num = datum['repNum']
trial_num = datum['trialNum']
target = utils.get_img_path(datum['targetImg'])
raw_cap = datum['msg']
listener.trial_num = trial_num
listener.sample_num = ctx['sample_num']
# Set up for new round
print('\nround {}, target {}, msg {}'.format(
rep_num, utils.get_id_from_path(target), raw_cap
))
listener.set_image(target)
id_cap = (listener.process_human_caption(raw_cap) if ctx['handleOOV']
else utils.words_to_ids(raw_cap, listener.vocab))
scores = listener.L0_score(np.expand_dims(id_cap, axis=0), ctx['dirs'])
cap_score = listener.S0_score(utils.load_image(target).to(device),
torch.tensor([id_cap]).to(device),
len(id_cap))
# Write out
scores = scores.data.cpu().numpy()[0]
target_idx = listener.context.index(listener.raw_image)
target_score = scores[target_idx]
best = list(scores).index(max(scores))
correct = best == target_idx
if args.debug: print(list(map(lambda x: np.exp(x), scores)))
if args.debug: print('{}, model says: {}, target actually: {}'.format(correct, best, target_idx))
if args.debug: print('accuracy in real game: {}'.format(datum['correct']))
writer.writerow(ctx, datum, raw_cap, scores, len(raw_cap),
target_score, cap_score, correct)
# Update models as relevant
if ctx['loss'] != 'fixed' :
listener.update_model(trial_num, raw_cap)
def get_ctx_from_tag(self, ctx_tag):
"""
Retrieves requested context from coco_contexts.json or coco_contexts_easy.json
"""
if self.ctx_type == "challenge":
coco_contexts = coco_contexts_hard
if self.ctx_type == "easy":
coco_contexts = coco_contexts_easy
ctx = next(filter(lambda x: x['cluster_ids'] == ctx_tag,
coco_contexts))
filenames = ctx['neighbor_names'][:self.ctx_size]
paths = [utils.get_img_path(name) for name in filenames]
tags = ['custom' + str(ctx_tag) for i in range(self.ctx_size)]
imgs = [utils.load_image(path) for path in paths]
return imgs, paths, tags
def main_memory(args):
path = '../data/model_output/cap_comparison.csv'
writer = CapCompWriter(args, path)
categories = coco.loadCats(coco.getCatIds())
listener = AdaptiveAgent(args)
grid = construct_context_grid(args)
for ctx in grid:
print("\nsample_num: {}, loss: {}, getting captions..."
.format(ctx['sample_num'], ctx['loss']))
target_caps, control_cap, speaker_decoders = get_eval_captions(args, ctx)
print(target_caps)
print("no. speaker decoders: ", len(speaker_decoders))
for trial in ctx['speaker_data'] :
print("\n evaluating for trial : ", trial)
# retrieve model state and vars for this round
speaker_decoder = speaker_decoders[trial['trialNum']]
# update decoder weights for this trial
current_vocab = copy.deepcopy(speaker_decoder.vocab)
listener.decoder = DecoderRNN(256, 512, current_vocab).to(device)
listener.decoder.load_state_dict(speaker_decoder.state_dict())
target_img = utils.get_img_path(trial['targetImg'])
control_img = ctx['control_dir']
# we have all the captions but let's just score first and last
first_cap, last_cap = retrieve_eval_caps(target_caps, target_img)
curr_cap = retrieve_curr_cap(target_caps, target_img, trial['repNum'])
control_score = get_score(listener, control_img, control_cap)
first_score = get_score(listener, target_img, first_cap)
last_score = get_score(listener, target_img, last_cap)
human_cap = trial['msg']
if args.debug: print("\ncontrol: {}\n, target: {}\n, reduced target: {}"
.format(control_string, first_string, last_string))
if args.debug: print("control {}, target {}, reduced target {}"
.format(control_score, first_score, last_score))
# save scores
writer.writerow(ctx, trial['trialNum'], target_img,
control_cap, curr_cap, human_cap,
control_score, first_score, last_score)
def main(args):
path = '../data/model_output/speaker_lesions.csv'
writer = EfficiencyWriter(args, path)
speaker = AdaptiveAgent(args)
grid = construct_context_grid(args)
for ctx in grid:
print("\n------gameid: {}, sample_num: {}, loss: {}, ds_type: {}, speaker_model: {}, cost_weight: {}"
.format(ctx['gameid'], ctx['sample_num'], ctx['loss'], ctx['ds_type'],
ctx['speaker_model'], ctx['cost_weight']))
speaker.loss = ctx['loss']
speaker.reset_to_initialization(ctx['dirs'])
speaker.dataset_type = ctx['ds_type']
speaker.context_type = ctx['context_type']
speaker.cost_weight = ctx['cost_weight']
# simulate round-robin style by looping through targets in random order
for datum in ctx['speaker_data'] :
rep_num = datum['repNum']
trial_num = datum['trialNum']
target = utils.get_img_path(datum['targetImg'])
print(target)
speaker.trial_num = trial_num
speaker.sample_num = ctx['sample_num']
speaker.set_image(target)
cap = speaker.generate_utterance(ctx['speaker_model'], as_string = True)
cap = utils.ids_to_words(utils.words_to_ids(cap, speaker.vocab), speaker.vocab)
if cap[:7] == '<start>' :
cap = cap[8:-6]
print('\nround {}, target {}, msg {}'.format(
rep_num, utils.get_id_from_path(target), cap
))
if datum['correct'] == True :
print('training')
speaker.update_model(trial_num, cap)
writer.writerow(ctx, datum, trial_num, target, cap, len(cap))
def get_eval_captions(args, ctx):
speaker = AdaptiveAgent(args)
speaker.loss = ctx['loss']
speaker_decoders = []
target_caps = []
# generate control caption & save round 0 speaker model
print(ctx['control_dir'])
speaker_decoders.append(copy.deepcopy(speaker.decoder))
speaker.reset_to_initialization(ctx['target_dirs'])
speaker.set_context([ctx['control_dir']])
speaker.set_image(ctx['control_dir'])
orig_control_cap = speaker.generate_utterance('S0')
# generate initial target captions
speaker.set_context(ctx['target_dirs'])
for t, target_img_dir in enumerate(ctx['target_dirs']):
speaker.set_image(target_img_dir)
target_caps.append({'gen_cap' : speaker.generate_utterance('S0'),
'target' : target_img_dir,
'rep_num' : 'pre'})
# adapt model and get generated cap at each point
for trial in ctx['speaker_data'] :
target = utils.get_img_path(trial['targetImg'])
target_idx = speaker.context.index(target)
speaker.set_image(target)
speaker.update_model(trial['trialNum'], trial['msg'])
target_caps.append({'gen_cap' : speaker.generate_utterance('S0'),
'target' : target,
'rep_num' : trial['repNum']})
speaker_decoders.append(copy.deepcopy(speaker.decoder))
for t, target_img_dir in enumerate(ctx['target_dirs']):
speaker.set_image(target_img_dir)
target_caps.append({'gen_cap' : speaker.generate_utterance('S0'),
'target' : target_img_dir,
'rep_num' : 'post'})
return target_caps, orig_control_cap, speaker_decoders
def _read_data(self):
if self.flags.is_test:
# real test images and vessels in the memory
self.test_imgs, self.test_vessels, self.test_masks, self.test_mean_std = utils.get_test_imgs(
target_dir=self.test_dir,
img_size=self.image_size,
dataset=self.dataset)
self.test_img_files = utils.all_files_under(
os.path.join(self.test_dir, 'images'))
self.num_test = self.test_imgs.shape[0]
elif not self.flags.is_test:
random.seed(datetime.now()) # set random seed
self.train_img_files, self.train_vessel_files, mask_files = utils.get_img_path(
self.train_dir, self.dataset)
self.num_train = int(len(self.train_img_files))
self.num_val = int(
np.floor(self.val_ratio * int(len(self.train_img_files))))
self.num_train -= self.num_val
self.val_img_files = self.train_img_files[-self.num_val:]
self.val_vessel_files = self.train_vessel_files[-self.num_val:]
val_mask_files = mask_files[-self.num_val:]
self.train_img_files = self.train_img_files[:-self.num_val]
self.train_vessel_files = self.train_vessel_files[:-self.num_val]
# read val images and vessels in the memory
self.val_imgs, self.val_vessels, self.val_masks, self.val_mean_std = utils.get_val_imgs(
self.val_img_files,
self.val_vessel_files,
val_mask_files,
img_size=self.image_size)
self.num_val = self.val_imgs.shape[0]
# optimizer
t_vars = tf.trainable_variables()
d_vars = [var for var in t_vars if 'discriminator' in var.name]
g_vars = [var for var in t_vars if 'generator' in var.name]
for var in t_vars:
print(var.name)
# d_optim = tf.train.AdamOptimizer().minimize(d_loss, var_list=d_vars)
# g_optim = tf.train.AdamOptimizer().minimize(g_loss, var_list=g_vars)
d_optim = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(
d_loss, var_list=d_vars)
g_optim = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(
g_loss, var_list=g_vars)
# read images
train_img, train_labels = utils.get_img_path(data_path)
num_train = int(len(train_img))
# initialize all valuables
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
init = tf.global_variables_initializer()
resume_training = True
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
sess.run(init)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=100)
sess.graph.finalize()
dir_save = "./training_weights/"
utils.mkdir(dir_save)
def main(args):
path = '../data/model_output/listener_cat_forgetting.csv'
writer = EfficiencyWriter(args, path)
# init listener model
listener = AdaptiveAgent(args)
listener.reduction_history_window = 'complete'
grid = construct_context_grid(args)
for ctx in grid:
print("\n------train gameid: {}, sample_num: {}, loss: {}".format(
ctx['train_context_info']['gameid'], ctx['sample_num'],
ctx['loss']))
# train: reduce with human
# reset speaker and listener to pretrained setting
listener.reset_to_initialization(ctx['train_dirs']) # setting context
listener.loss = ctx['loss']
listener.dataset_type = ctx['ds_type']
listener.history = [] # TODO: redundant ?
train_target = None
for datum in ctx['train_context_info']['speaker_data']:
rep_num = datum['repNum']
trial_num = datum['trialNum']
target = utils.get_img_path(datum['targetImg'])
raw_cap = datum['msg']
listener.trial_num = trial_num
listener.sample_num = ctx['sample_num']
# Set up for new round
print('\nround {}, target {}, msg {}'.format(
rep_num, utils.get_id_from_path(target), raw_cap))
listener.set_image(target)
id_cap = (listener.process_human_caption(raw_cap)
if ctx['handleOOV'] else utils.words_to_ids(
raw_cap, listener.vocab))
scores = listener.L0_score(np.expand_dims(id_cap, axis=0),
ctx['train_dirs'])
cap_score = listener.S0_score(
utils.load_image(target).to(device),
torch.tensor([id_cap]).to(device), len(id_cap))
# Write out
scores = scores.data.cpu().numpy()[0]
target_idx = listener.context.index(listener.raw_image)
target_score = scores[target_idx]
best = list(scores).index(max(scores))
correct = best == target_idx
if args.debug: print(list(map(lambda x: np.exp(x), scores)))
if args.debug:
print('{}, model says: {}, target actually: {}'.format(
correct, best, target_idx))
if args.debug:
print('accuracy in real game: {}'.format(datum['correct']))
# Update models as relevant
if ctx['loss'] != 'fixed':
listener.update_model(trial_num, raw_cap)
if rep_num == 5:
writer.writerow(ctx, datum, raw_cap, scores, len(raw_cap),
target_score, cap_score, correct, {
'gameid': None,
'context_id': None
})
# test on new human
if args.debug: print("\nTESTING!")
for j, test_context_info in enumerate(ctx['test_context_infos']):
print("\ntest context: {}".format(j))
listener.set_context(
ctx['test_dirs']
[j]) # set context to test dirs, BUT don't reset weights
listener.history = []
# TODO: should we reset vocab?
#for datum in ctx['test_context_info']['speaker_data']:
for datum in test_context_info['speaker_data']:
rep_num = datum['repNum']
if rep_num > 0:
break
trial_num = datum['trialNum']
target = utils.get_img_path(datum['targetImg'])
raw_cap = datum['msg']
listener.trial_num = trial_num
listener.sample_num = ctx['sample_num']
# Set up for new round
print('\nround {}, target {}, msg {}'.format(
rep_num, utils.get_id_from_path(target), raw_cap))
listener.set_image(target)
id_cap = (listener.process_human_caption(raw_cap)
if ctx['handleOOV'] else utils.words_to_ids(
raw_cap, listener.vocab))
scores = listener.L0_score(np.expand_dims(id_cap, axis=0),
ctx['train_dirs'])
cap_score = listener.S0_score(
utils.load_image(target).to(device),
torch.tensor([id_cap]).to(device), len(id_cap))
# Write out
scores = scores.data.cpu().numpy()[0]
target_idx = listener.context.index(listener.raw_image)
target_score = scores[target_idx]
best = list(scores).index(max(scores))
correct = best == target_idx
if args.debug: print(list(map(lambda x: np.exp(x), scores)))
if args.debug:
print('{}, model says: {}, target actually: {}'.format(
correct, best, target_idx))
if args.debug:
print('accuracy in real game: {}'.format(datum['correct']))
writer.writerow(ctx, datum, raw_cap, scores, len(raw_cap),
target_score, cap_score, correct,
test_context_info)