def interactive_rank(opt, print_parser=None):
# Create model and assign it to the specified task
human = create_agent(opt)
task = create_task_agent_from_taskname(opt)[0]
metrics = Metrics(opt)
episodes = 0
def print_metrics():
report = metrics.report()
report['episodes'] = episodes
print(report)
# Show some example dialogs:
try:
while not task.epoch_done():
msg = task.act()
print('[{id}]: {text}'.format(id=task.getID(), text=msg.get('text', '')))
cands = list(msg.get('label_candidates', []))
random.shuffle(cands)
for i, c in enumerate(cands):
print(' [{i}]: {c}'.format(i=i, c=c))
print('[ Please choose a response from the list. ]')
choice = None
while choice is None:
choice = human.act().get('text')
try:
choice = int(choice)
if choice >= 0 and choice < len(cands):
choice = cands[choice]
else:
print(
'[ Try again: you selected {i} but the '
'candidates are indexed from 0 to {j}. ]'
''.format(i=choice, j=len(cands) - 1)
)
choice = None
except (TypeError, ValueError):
print('[ Try again: you did not enter a valid index. ]')
choice = None
print('[ You chose ]: {}'.format(choice))
reply = {'text_candidates': [choice]}
labels = msg.get('eval_labels', msg.get('labels'))
metrics.update(reply, labels)
if msg.get('episode_done'):
episodes += 1
print_metrics()
print('------------------------------')
print('[ True reply ]: {}'.format(labels[0]))
if msg.get('episode_done'):
print('******************************')
except KeyboardInterrupt:
pass
print()
print_metrics()
class SplitTeacher(Teacher):
"""FVQA Teacher, which loads the json VQA data and implements its own
`act` method for interacting with student agent.
Use "fvqa:split:X" to choose between splits 0-4 (inclusive), or just
"fvqa" to use the default split (0).
"""
def __init__(self, opt, shared=None):
super().__init__(opt)
dt = opt['datatype'].split(':')[0]
if dt not in ('train', 'test'):
raise RuntimeError('Not valid datatype (only train/test).')
task = opt.get('task', 'fvqa:split:0')
task_num = 0 # default to train/split 0
split = task.split(':')
if len(split) > 2:
task_num = split[2]
if task_num not in [str(i) for i in range(5)]:
raise RuntimeError(
'Invalid train/test split ID (0-4 inclusive)')
if not hasattr(self, 'factmetrics'):
if shared and shared.get('factmetrics'):
self.factmetrics = shared['factmetrics']
else:
self.factmetrics = Metrics(opt)
self.datatype = opt['datatype']
questions_path, trainset_path, self.image_path = _path(opt)
if shared and 'ques' in shared:
self.ques = shared['ques']
else:
self._setup_data(questions_path, trainset_path, dt, task_num)
self.len = len(self.ques)
self.asked_question = False
# for ordered data in batch mode (especially, for validation and
# testing), each teacher in the batch gets a start index and a step
# size so they all process disparate sets of the data
self.step_size = opt.get('batchsize', 1)
self.data_offset = opt.get('batchindex', 0)
self.image_loader = ImageLoader(opt)
self.reset()
def num_examples(self):
return self.len
def num_episodes(self):
return self.len
def report(self):
r = super().report()
r['factmetrics'] = self.factmetrics.report()
return r
def reset(self):
# Reset the dialog so that it is at the start of the epoch,
# and all metrics are reset.
super().reset()
self.lastY = None
self.episode_idx = self.data_offset - self.step_size
self.epochDone = False
def reset_metrics(self):
super().reset_metrics()
self.factmetrics.clear()
def observe(self, observation):
"""Process observation for metrics."""
if self.lastY is not None:
if self.asked_question:
self.metrics.update(observation, self.lastY[0])
else:
self.factmetrics.update(observation, self.lastY[1])
self.lastY = None
return observation
def act(self):
if self.asked_question:
self.asked_question = False
action = {
'text': 'Which fact supports this answer?',
'episode_done': True
}
if self.datatype.startswith('train'):
action['labels'] = self.lastY[1]
if self.datatype != 'train' and self.episode_idx + self.step_size >= self.num_episodes(
):
self.epochDone = True
return action
if self.datatype == 'train':
self.episode_idx = random.randrange(self.len)
else:
self.episode_idx = (self.episode_idx +
self.step_size) % self.num_episodes()
self.asked_question = True
qa = self.ques[self.episode_idx]
question = qa['question']
img_path = self.image_path + qa['img_file']
action = {
'image': self.image_loader.load(img_path),
'text': question,
'episode_done': False
}
human_readable = qa['fact_surface'].replace('[', '').replace(']', '')
self.lastY = [[qa['answer']], [human_readable]]
if self.datatype.startswith('train'):
action['labels'] = self.lastY[0]
return action
def share(self):
shared = super().share()
shared['factmetrics'] = self.factmetrics
shared['ques'] = self.ques
if hasattr(self, 'facts'):
shared['facts'] = self.facts
return shared
def _setup_data(self, questions_path, trainset_path, datatype, task_num):
print('loading: ' + questions_path)
with open(questions_path) as questions_file:
questions = json.load(questions_file)
train_test_images = set()
with open(
os.path.join(trainset_path,
'{}_list_{}.txt'.format(datatype,
task_num))) as imageset:
for line in imageset:
train_test_images.add(line.strip())
self.ques = [
questions[k] for k in sorted(questions.keys())
if questions[k]['img_file'] in train_test_images
]
class SplitTeacher(Teacher):
"""FVQA Teacher, which loads the json VQA data and implements its own
`act` method for interacting with student agent.
Use "fvqa:split:X" to choose between splits 0-4 (inclusive), or just
"fvqa" to use the default split (0).
"""
def __init__(self, opt, shared=None):
super().__init__(opt)
dt = opt['datatype'].split(':')[0]
if dt not in ('train', 'test'):
raise RuntimeError('Not valid datatype (only train/test).')
task = opt.get('task', 'fvqa:split:0')
task_num = 0 # default to train/split 0
split = task.split(':')
if len(split) > 2:
task_num = split[2]
if task_num not in [str(i) for i in range(5)]:
raise RuntimeError('Invalid train/test split ID (0-4 inclusive)')
if not hasattr(self, 'factmetrics'):
if shared and shared.get('factmetrics'):
self.factmetrics = shared['factmetrics']
else:
self.factmetrics = Metrics(opt)
self.datatype = opt['datatype']
questions_path, trainset_path, self.image_path = _path(opt)
if shared and 'ques' in shared:
self.ques = shared['ques']
else:
self._setup_data(questions_path, trainset_path, dt, task_num)
self.len = len(self.ques)
self.asked_question = False
# for ordered data in batch mode (especially, for validation and
# testing), each teacher in the batch gets a start index and a step
# size so they all process disparate sets of the data
self.step_size = opt.get('batchsize', 1)
self.data_offset = opt.get('batchindex', 0)
self.image_loader = ImageLoader(opt)
self.reset()
def __len__(self):
return self.len
def report(self):
r = super().report()
r['factmetrics'] = self.factmetrics.report()
return r
def reset(self):
# Reset the dialog so that it is at the start of the epoch,
# and all metrics are reset.
super().reset()
self.lastY = None
self.episode_idx = self.data_offset - self.step_size
self.epochDone = False
def reset_metrics(self):
super().reset_metrics()
self.factmetrics.clear()
def observe(self, observation):
"""Process observation for metrics."""
if self.lastY is not None:
if self.asked_question:
self.metrics.update(observation, self.lastY[0])
else:
self.factmetrics.update(observation, self.lastY[1])
self.lastY = None
return observation
def act(self):
if self.asked_question:
self.asked_question = False
action = {'text': 'Which fact supports this answer?', 'episode_done': True}
if self.datatype.startswith('train'):
action['labels'] = self.lastY[1]
if self.datatype != 'train' and self.episode_idx + self.step_size >= len(self):
self.epochDone = True
return action
if self.datatype == 'train':
self.episode_idx = random.randrange(self.len)
else:
self.episode_idx = (self.episode_idx + self.step_size) % len(self)
self.asked_question = True
qa = self.ques[self.episode_idx]
question = qa['question']
img_path = self.image_path + qa['img_file']
action = {
'image': self.image_loader.load(img_path),
'text': question,
'episode_done': False
}
human_readable = qa['fact_surface'].replace('[', '').replace(']', '')
self.lastY = [[qa['answer']], [human_readable]]
if self.datatype.startswith('train'):
action['labels'] = self.lastY[0]
return action
def share(self):
shared = super().share()
shared['factmetrics'] = self.factmetrics
shared['ques'] = self.ques
if hasattr(self, 'facts'):
shared['facts'] = self.facts
return shared
def _setup_data(self, questions_path, trainset_path, datatype, task_num):
print('loading: ' + questions_path)
with open(questions_path) as questions_file:
questions = json.load(questions_file)
train_test_images = set()
with open(os.path.join(trainset_path, '{}_list_{}.txt'.format(datatype, task_num))) as imageset:
for line in imageset:
train_test_images.add(line.strip())
self.ques = [questions[k] for k in sorted(questions.keys()) if questions[k]['img_file'] in train_test_images]