def _eval_single_world(opt, agent, task):
logging.info(
f'Evaluating task {task} using datatype {opt.get("datatype")}.')
# set up world logger
world_logger = WorldLogger(opt) if opt['world_logs'] else None
task_opt = opt.copy() # copy opt since we're editing the task
task_opt['task'] = task
world = create_task(task_opt, agent) # create worlds for tasks
# set up logging
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# max number of examples to evaluate
max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
cnt = 0
total_cnt = world.num_examples()
if is_distributed():
logging.warning('Progress bar is approximate in distributed mode.')
while not world.epoch_done() and cnt < max_cnt:
cnt += opt.get('batchsize', 1)
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt['display_examples']:
# display examples
print(world.display() + '\n~~')
if log_time.time() > log_every_n_secs:
report = world.report()
text, report = log_time.log(report.get('exs', 0),
min(max_cnt, total_cnt), report)
logging.info(text)
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = opt['world_logs']
world_logger.write(outfile, world, file_format=opt['save_format'])
report = aggregate_unnamed_reports(all_gather_list(world.report()))
world.reset()
return report
def __init__(self, opt: Opt, shared=None):
if not hasattr(self, "fold"):
self.fold = DatatypeHelper.fold(opt["datatype"])
super().__init__(opt, shared)
self.epochDone = False
self.batchsize = opt.get("batchsize", 1)
self.max_episodes = len(self.episodes)
if opt.get("num_episodes", 0) > 0:
self.max_episodes = min(self.max_episodes, opt.get("num_episodes"))
self.episode_idx = opt.get("batchindex", 0)
self._setup_next_episode()
self.round_idx = 0 # for some downstream utt + sysUttAndApiCallAgents.
if is_distributed(): # cause gotta manually handle
rank = get_rank()
chunk_size = ceil(self.max_episodes / num_workers())
self.episode_idx += rank * chunk_size
self.max_episodes = min(self.max_episodes, (rank + 1) * chunk_size)
def __init__(self, opt: Opt, shared=None):
super().__init__(opt, shared)
self.epochDone = False
if shared is None:
self.episodes = self._setup_single_goal_episodes()
else:
# Handled fine in _TodDataDumpAgent
pass
self.max_episodes = len(self.episodes)
if opt.get("num_episodes", 0) > 0:
self.max_episodes = min(self.max_episodes, opt.get("num_episodes"))
if is_distributed(): # cause gotta manually handle
rank = get_rank()
chunk_size = ceil(self.max_episodes / num_workers())
self.max_episodes = min(self.max_episodes, (rank + 1) * chunk_size)
self._setup_next_episode()
def _run_single_eval(self, opt, valid_world, max_exs, datatype, is_multitask, task):
# run evaluation on a single world
valid_world.reset()
world_logger = None
task_opt = opt.copy()
# set up world logger for the "test" fold
if opt['world_logs'] and datatype == 'test':
task_opt['world_logs'] = get_task_world_logs(
task, opt['world_logs'], is_multitask
)
world_logger = WorldLogger(task_opt)
cnt = 0
max_cnt = max_exs if max_exs > 0 else float('inf')
while not valid_world.epoch_done() and cnt < max_cnt:
valid_world.parley()
if world_logger is not None:
world_logger.log(valid_world)
if cnt == 0 and opt['display_examples']:
print(valid_world.display() + '\n~~')
print(valid_world.report())
cnt = valid_world.report().get('exs') or 0
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(task_opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = task_opt['world_logs']
world_logger.write(outfile, valid_world, file_format=opt['save_format'])
valid_report = valid_world.report()
if opt.get('validation_share_agent', False):
valid_world.reset() # make sure world doesn't remember valid data
return valid_report
def _eval_single_world(opt, agent, task):
logging.info(
f'Evaluating task {task} using datatype {opt.get("datatype")}.')
# set up world logger
task_opt = opt.copy() # copy opt since we're editing the task
task_opt['task'] = task
# add task suffix in case of multi-tasking
if opt['world_logs']:
task_opt['world_logs'] = get_task_world_logs(
task,
task_opt['world_logs'],
is_multitask=len(opt['task'].split(',')) > 1)
world_logger = WorldLogger(task_opt) if task_opt['world_logs'] else None
world = create_task(task_opt, agent) # create worlds for tasks
# set up logging
log_every_n_secs = opt.get('log_every_n_secs', -1)
if log_every_n_secs <= 0:
log_every_n_secs = float('inf')
log_time = TimeLogger()
# max number of examples to evaluate
max_cnt = opt['num_examples'] if opt['num_examples'] > 0 else float('inf')
cnt = 0
total_cnt = world.num_examples()
if is_distributed():
logging.warning('Progress bar is approximate in distributed mode.')
while not world.epoch_done() and cnt < max_cnt:
cnt += opt.get('batchsize', 1)
world.parley()
if world_logger is not None:
world_logger.log(world)
if opt['display_examples']:
# display examples
print(world.display() + '\n~~')
if log_time.time() > log_every_n_secs:
report = world.report()
text, report = log_time.log(report.get('exs', 0),
min(max_cnt, total_cnt), report)
logging.info(text)
if world_logger is not None:
# dump world acts to file
world_logger.reset() # add final acts to logs
if is_distributed():
rank = get_rank()
base_outfile, extension = os.path.splitext(task_opt['world_logs'])
outfile = base_outfile + f'_{rank}' + extension
else:
outfile = task_opt['world_logs']
world_logger.write(outfile, world, file_format=opt['save_format'])
report = aggregate_unnamed_reports(all_gather_list(world.report()))
if isinstance(world.agents, list) and len(world.agents) > 1:
classifier_agent = world.agents[CLASSIFIER_AGENT]
if hasattr(classifier_agent, 'calc_auc') and classifier_agent.calc_auc:
for class_indices, curr_auc in zip(
classifier_agent.auc_class_indices, classifier_agent.aucs):
report[
f'AUC_{classifier_agent.class_list[class_indices]}'] = curr_auc
classifier_agent.reset_auc()
# for safety measures
agent.reset_auc()
world.reset()
return report
def _save_outputs(self, opt, world, logger, episode_metrics):
if is_distributed(): # flatten everything intelligently if need be
world_report = aggregate_unnamed_reports(
all_gather_list(world.report()))
episode_metrics_unflattened = all_gather_list(episode_metrics)
flattened = []
for rank_elem in episode_metrics_unflattened:
for elem in rank_elem:
flattened.append(elem)
episode_metrics = flattened
else:
world_report = world.report()
logging.report("Final report:\n" + nice_report(world_report))
report = dict_report(world_report)
def get_episode_report(goal, episode_metric):
metrics_dict = dict_report(episode_metric.report())
metrics_dict["goal"] = goal
return metrics_dict
report["tod_metrics"] = [
get_episode_report(g, e) for g, e in episode_metrics
]
if "report_filename" in opt and opt["report_filename"] is not None:
if len(world_report) == 0:
logging.warning("Report is empty; not saving report")
report_fname = f"{opt['report_filename']}.json"
# Save report
if not is_distributed() or is_primary_worker():
with PathManager.open(report_fname, "w") as f:
logging.info(f"Saving model report to {report_fname}")
json.dump({"opt": opt, "report": report}, f, indent=4)
f.write("\n") # for jq
if "world_logs" in opt and opt["world_logs"] is not None:
if is_distributed(): # Save separately, then aggregate together
rank = get_rank()
log_outfile_part = (
f"{opt['world_logs']}_{opt['save_format']}_{rank}.jsonl")
logger.write(log_outfile_part,
world,
file_format=opt["save_format"])
sync_object(None)
if is_primary_worker():
log_outfile = f"{opt['world_logs']}_{opt['save_format']}.jsonl"
log_outfile_metadata = (
f"{opt['world_logs']}_{opt['save_format']}.metadata")
with open(log_outfile, "w+") as outfile:
for rank in range(num_workers()):
log_outfile_part = (
f"{opt['world_logs']}_{opt['save_format']}_{rank}.jsonl"
)
with open(log_outfile_part) as infile:
for line in infile:
json_blob = json.loads(line.strip())
if (
len(json_blob["dialog"]) < 2
): # skip when we don't have generation
continue
json_blob[
"metadata_path"] = log_outfile_metadata
outfile.write(json.dumps(json_blob))
outfile.write("\n")
log_output_part_metadata = f"{opt['world_logs']}_{opt['save_format']}_{rank}.metadata"
if rank == 0:
copyfile(log_output_part_metadata,
log_outfile_metadata),
os.remove(log_outfile_part)
os.remove(log_output_part_metadata)
else:
log_outfile = f"{opt['world_logs']}_{opt['save_format']}.jsonl"
logger.write(log_outfile,
world,
file_format=opt["save_format"])
return report
