def interactive(opt, print_parser=None):
if print_parser is not None:
if print_parser is True and isinstance(opt, ParlaiParser):
print_parser = opt
elif print_parser is False:
print_parser = None
if isinstance(opt, ParlaiParser):
logging.error('interactive should be passed opt not Parser')
opt = opt.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
if print_parser:
# Show arguments after loading model
print_parser.opt = agent.opt
print_parser.print_args()
human_agent = LocalHumanAgent(opt)
# set up world logger
world_logger = WorldLogger(opt) if opt['save_world_logs'] else None
world = create_task(opt, [human_agent, agent])
# Show some example dialogs:
while not world.epoch_done():
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt.get('display_examples'):
print("---")
print(world.display())
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
base_outfile = opt['report_filename'].split('.')[0]
outfile = f'{base_outfile}_{opt["task"]}_replies.jsonl'
world_logger.write(outfile, world, file_format=opt['save_format'])
def interactive(opt: 'zoo:tutorial_transformer_generator/model'):
if isinstance(opt, ParlaiParser):
logging.error('interactive should be passed opt not Parser')
opt = opt.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=False)
agent.opt.log()
human_agent = LocalHumanAgent(opt)
# set up world logger
world_logger = WorldLogger(opt) if opt.get('outfile') else None
world = create_task(opt, [human_agent, agent])
# Show some example dialogs:
while not world.epoch_done():
world.parley()
print("done by me!")
print(world.display())
if world.epoch_done() or world.get_total_parleys() <= 0:
# chat was reset with [DONE], [EXIT] or EOF
if world_logger is not None:
world_logger.reset()
continue
if world_logger is not None:
world_logger.log(world)
if opt.get('display_examples'):
print("---")
print(world.display())
if world_logger is not None:
# dump world acts to file
world_logger.write(opt['outfile'],
world,
file_format=opt['save_format'])
def _eval_single_world(opt, agent, task):
print('[ Evaluating task {} using datatype {}. ] '.format(
task, opt.get('datatype', 'N/A')))
# set up world logger
world_logger = WorldLogger(opt) if opt['save_world_logs'] else None
task_opt = opt.copy() # copy opt since we're editing the task
# task_opt['task'] = task
world = create_task(task_opt, agent) # create worlds for tasks
# set up logging
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# max number of examples to evaluate
max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
cnt = 0
while not world.epoch_done() and cnt < max_cnt:
cnt += opt.get('batchsize', 1)
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt['display_examples']:
# display examples
print(world.display() + '\n~~')
# for a in world.acts:
# print (a)
# print (world.get_acts())
# print (world.acts)
if log_time.time() > log_every_n_secs:
report = world.report()
text, report = log_time.log(report.get('exs', 0),
world.num_examples(), report)
print(text)
report = world.report()
print("Printing Report")
print(report)
world.reset()
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
base_outfile = opt['report_filename'].split('.')[0]
print("filename: ", base_outfile)
outfile = base_outfile + f'_{task}_replies.jsonl'
# world_logger.write_jsonl_format(outfile)
world_logger.write_parlai_format(outfile)
return report
def _eval_single_world(opt, agent, task):
logging.info(
f'Evaluating task {task} using datatype {opt.get("datatype")}.')
# set up world logger
world_logger = WorldLogger(opt) if opt['world_logs'] else None
task_opt = opt.copy() # copy opt since we're editing the task
task_opt['task'] = task
world = create_task(task_opt, agent) # create worlds for tasks
# set up logging
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# max number of examples to evaluate
max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
cnt = 0
total_cnt = world.num_examples()
if is_distributed():
logging.warning('Progress bar is approximate in distributed mode.')
while not world.epoch_done() and cnt < max_cnt:
cnt += opt.get('batchsize', 1)
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt['display_examples']:
# display examples
print(world.display() + '\n~~')
if log_time.time() > log_every_n_secs:
report = world.report()
text, report = log_time.log(report.get('exs', 0),
min(max_cnt, total_cnt), report)
logging.info(text)
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = opt['world_logs']
world_logger.write(outfile, world, file_format=opt['save_format'])
report = aggregate_unnamed_reports(all_gather_list(world.report()))
world.reset()
return report
def interactive(opt, print_parser=None):
if print_parser is not None:
if print_parser is True and isinstance(opt, ParlaiParser):
print_parser = opt
elif print_parser is False:
print_parser = None
if isinstance(opt, ParlaiParser):
print(
'[ Deprecated Warning: interactive should be passed opt not Parser ]'
)
opt = opt.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
human_agent = LocalHumanAgent(opt)
world = create_task(opt, [human_agent, agent])
# set up world logger
world_logger = WorldLogger(opt) if opt['save_world_logs'] else None
if print_parser:
# Show arguments after loading model
print_parser.opt = agent.opt
print_parser.print_args()
# Show some example dialogs:
while True:
try:
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt.get('display_examples'):
print("---")
print(world.display())
if world.epoch_done():
print("EPOCH DONE")
break
except KeyboardInterrupt:
if world_logger is not None:
print(f"\nWriting out world log.")
# Save report
report = world.report()
world.reset()
# dump world acts to file
world_logger.reset() # add final acts to logs
base_outfile = opt['report_filename'].split('.')[0]
outfile = base_outfile + f'_interactive_replies.json'
world_logger.write(outfile,
file_format=opt['world_logs_format'])
quit()
def interactive(opt, print_parser=None):
if print_parser is not None:
if print_parser is True and isinstance(opt, ParlaiParser):
print_parser = opt
elif print_parser is False:
print_parser = None
if isinstance(opt, ParlaiParser):
print(
'[ Deprecated Warning: interactive should be passed opt not Parser ]'
)
opt = opt.parse_args()
# Create model and assign it to the specified task
agent = create_agent(opt, requireModelExists=True)
if print_parser:
# Show arguments after loading model
print_parser.opt = agent.opt
print_parser.print_args()
# human_agent = LocalHumanAgent(opt)
human_agent = AutoQueryAgent(opt,
query_txt=opt['report_filename'] + ".txt")
world_logger = WorldLogger(opt)
world = create_task(opt, [human_agent, agent])
# Show some example dialogs:
#while not world.epoch_done():
with open(opt['report_filename'] + ".txt", "r") as f:
length = len(f.read().split("\n")) - 1
#for _ in range(5) :
for _ in range(length):
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt.get('display_examples'):
print("---")
print(world.display())
if world_logger is not None:
world_logger.reset()
base_outfile = opt['report_filename'].split('.')[0]
outfile = f'{base_outfile}_{opt["task"]}_replies.jsonl'
# world_logger.write(outfile, world, file_format=opt['save_format'])
# world_logger.write(outfile, world, file_format='conversations')
world_logger.write(outfile, world, file_format='text')
def _run_single_eval(self, opt, valid_world, max_exs, datatype, is_multitask, task):
# run evaluation on a single world
valid_world.reset()
world_logger = None
task_opt = opt.copy()
# set up world logger for the "test" fold
if opt['world_logs'] and datatype == 'test':
task_opt['world_logs'] = get_task_world_logs(
task, opt['world_logs'], is_multitask
)
world_logger = WorldLogger(task_opt)
cnt = 0
max_cnt = max_exs if max_exs > 0 else float('inf')
while not valid_world.epoch_done() and cnt < max_cnt:
valid_world.parley()
if world_logger is not None:
world_logger.log(valid_world)
if cnt == 0 and opt['display_examples']:
print(valid_world.display() + '\n~~')
print(valid_world.report())
cnt = valid_world.report().get('exs') or 0
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(task_opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = task_opt['world_logs']
world_logger.write(outfile, valid_world, file_format=opt['save_format'])
valid_report = valid_world.report()
if opt.get('validation_share_agent', False):
valid_world.reset() # make sure world doesn't remember valid data
return valid_report
def _eval_single_world(opt, agent, task):
logging.info(
f'Evaluating task {task} using datatype {opt.get("datatype")}.')
# set up world logger
task_opt = opt.copy() # copy opt since we're editing the task
task_opt['task'] = task
# add task suffix in case of multi-tasking
if opt['world_logs']:
task_opt['world_logs'] = get_task_world_logs(
task,
task_opt['world_logs'],
is_multitask=len(opt['task'].split(',')) > 1)
world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None
world = create_task(task_opt, agent) # create worlds for tasks
# set up logging
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# max number of examples to evaluate
max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
cnt = 0
total_cnt = world.num_examples()
if is_distributed():
logging.warning('Progress bar is approximate in distributed mode.')
while not world.epoch_done() and cnt < max_cnt:
cnt += opt.get('batchsize', 1)
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt['display_examples']:
# display examples
print(world.display() + '\n~~')
if log_time.time() > log_every_n_secs:
report = world.report()
text, report = log_time.log(report.get('exs', 0),
min(max_cnt, total_cnt), report)
logging.info(text)
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(task_opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = task_opt['world_logs']
world_logger.write(outfile, world, file_format=opt['save_format'])
report = aggregate_unnamed_reports(all_gather_list(world.report()))
if isinstance(world.agents, list) and len(world.agents) > 1:
classifier_agent = world.agents[CLASSIFIER_AGENT]
if hasattr(classifier_agent, 'calc_auc') and classifier_agent.calc_auc:
for class_indices, curr_auc in zip(
classifier_agent.auc_class_indices, classifier_agent.aucs):
report[
f'AUC_{classifier_agent.class_list[class_indices]}'] = curr_auc
classifier_agent.reset_auc()
# for safety measures
agent.reset_auc()
world.reset()
return report
