def __init__(self, opt):
self.second_resp = opt.get('second_response')
self.examples_idx_stack_path = os.path.join(
os.getcwd(),
'./{}_examples_stack{}.pkl'.format(
'second_response' if self.second_resp else 'first_response',
'_sandbox' if opt['is_sandbox'] else '',
),
)
self.OLD = OffensiveStringMatcher()
self.opt = opt
build_pc(opt)
build_ic(opt)
df = 'personality_captions' if not self.second_resp else 'image_chat'
data_path = os.path.join(self.opt['datapath'], '{}/{}.json')
self.data = []
for dt in ['train', 'val', 'test']:
if self.second_resp and dt == 'val':
dt = 'valid'
with open(data_path.format(df, dt)) as f:
self.data += json.load(f)
if self.second_resp:
self.data = [d for d in self.data if len(d['dialog']) > 1]
if os.path.exists(self.examples_idx_stack_path):
with open(self.examples_idx_stack_path, 'rb') as handle:
self.idx_stack = pickle.load(handle)
else:
self.idx_stack = []
self.add_idx_stack()
self.save_idx_stack()
def main():
"""
Human Evaluation of various responses to comments on images.
A turker is shown an image and some dialog history. Then, the
turker is asked to choose which response they think is more engaging.
If no `--eval-data-path` is given, the data from the original
Image-Chat dataset is used.
To use your own data, please specify `--eval-data-path`, a path to an
appropriate json file with a list of examples, where each example
has the following structure:
{
'image_hash': <hash of image>,
'dialog': [(personality, text), ...] - list of personality, text tuples
'personality': <personality of responses to compare>
'<compare_key_1>': <first response to compare>,
'<compare_key_2>': <second option to compare>,
.
.
.
}
Note that compare_key_1 and compare_key_2 can be any field, as long as they
map to a string response.
Example Scenario:
Suppose you have the original Image-Chat dataset, and
you would like to compare the outputs of your model called `model`.
Your data may look like the following:
[{
'image_hash': hashforimageofcat,
'dialog': [
('Sweet', 'What a cute cat!'),
('Neutral', 'Just looks like a plain cat to me')
]
'personality': 'Sweet',
'comment': 'It really is adorable if you look!', # Human Comment
'model_comment': 'You'll love it if you pet it!' # Model Comment
}, ...]
Thus, you would specify `-ck1 comment -ck2 model_comment` to evaluate
the outputs of the model vs. the human comments from Personality-Captions
"""
argparser = ParlaiParser(False, False)
argparser.add_parlai_data_path()
argparser.add_mturk_args()
argparser.add_argument('-min_t',
'--min_turns',
default=3,
type=int,
help='minimum number of turns')
argparser.add_argument('-mt',
'--max_turns',
default=5,
type=int,
help='maximal number of chat turns')
argparser.add_argument(
'-mx_rsp_time',
'--max_resp_time',
default=1800,
type=int,
help='time limit for entering a dialog message',
)
argparser.add_argument(
'-mx_onb_time',
'--max_onboard_time',
type=int,
default=300,
help='time limit for turker'
'in onboarding',
)
argparser.add_argument(
'-ni',
'--num_images',
type=int,
default=10,
help='number of images to show \
to turker',
)
argparser.add_argument(
'--auto-approve-delay',
type=int,
default=3600 * 24,
help='how long to wait for \
auto approval',
)
argparser.add_argument('--data-path',
type=str,
default='',
help='where to save data')
argparser.add_argument(
'--eval-data-path',
type=str,
default='',
help='where to load data to rank from. Leave '
'blank to use Image-Chat data',
)
argparser.add_argument(
'-ck1',
'--compare-key-1',
type=str,
default='comment',
help='key of first comparable',
)
argparser.add_argument(
'-ck2',
'--compare-key-2',
type=str,
default='comment',
help='key of first comparable',
)
argparser.add_argument(
'-rnd',
'--dialog-round',
type=str,
default='first_response',
choices=round_choices,
help='which dialog round to show',
)
argparser.add_argument(
'--show-personality',
default=True,
type='bool',
help='whether to show the personality',
)
ImageChatTeacher.add_cmdline_args(argparser)
opt = argparser.parse_args()
build_ic(opt)
directory_path = os.path.dirname(os.path.abspath(__file__))
opt['task'] = os.path.basename(directory_path)
if 'data_path' not in opt or opt['data_path'] == '':
opt['data_path'] = os.getcwd() + '/data/' + opt['task']
if opt.get('eval_data_path') == '':
opt['eval_data_path'] = os.path.join(opt['datapath'],
'image_chat/test.json')
config = config_first if opt[
'dialog_round'] == 'first_response' else config_second
opt.update(config)
mturk_agent_ids = [CHOOSER]
mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids)
example_generator = ExampleGenerator(opt)
mturk_manager.setup_server(task_directory_path=directory_path)
try:
mturk_manager.start_new_run()
def run_onboard(worker):
worker.example_generator = example_generator
world = RoleOnboardWorld(opt, worker)
world.parley()
world.shutdown()
mturk_manager.set_onboard_function(onboard_function=run_onboard)
mturk_manager.ready_to_accept_workers()
mturk_manager.create_hits()
def check_worker_eligibility(worker):
return True
def assign_worker_roles(workers):
for w in workers:
w.id = mturk_agent_ids[0]
def run_conversation(mturk_manager, opt, workers):
agents = workers[:]
conv_idx = mturk_manager.conversation_index
world = MTurkImageChatStackRankWorld(
opt,
agents=agents,
world_tag='conversation t_{}'.format(conv_idx))
while not world.episode_done():
world.parley()
world.save_data()
world.shutdown()
world.review_work()
mturk_manager.start_task(
eligibility_function=check_worker_eligibility,
assign_role_function=assign_worker_roles,
task_function=run_conversation,
)
except BaseException:
raise
finally:
mturk_manager.expire_all_unassigned_hits()
mturk_manager.shutdown()