def __init__(self, opt, agents=None, shared=None): self.id = opt['task'] self.opt = copy.deepcopy(opt) if shared: # Create agents based on shared data. self.agents = create_agents_from_shared(shared['agents']) else: # Add passed in agents to world directly. self.agents = agents self.max_exs = None self.total_exs = 0 self.total_epochs = 0 self.total_parleys = 0 self.time = Timer()
def eval_model(parser, printargs=True): random.seed(42) opt = parser.parse_args(print_args=False) nomodel = False # check to make sure the model file exists if opt.get('model_file') is None: nomodel = True elif not os.path.isfile(opt['model_file']): raise RuntimeError('WARNING: Model file does not exist, check to make ' 'sure it is correct: {}'.format(opt['model_file'])) # Create model and assign it to the specified task agent = create_agent(opt) if nomodel and hasattr(agent, 'load'): # double check that we didn't forget to set model_file on loadable model raise RuntimeError('Stopping evaluation because model_file unset but ' 'model has a `load` function.') world = create_task(opt, agent) # Show arguments after loading model parser.opt = agent.opt if (printargs): parser.print_args() log_every_n_secs = opt.get('log_every_n_secs', -1) if log_every_n_secs <= 0: log_every_n_secs = float('inf') log_time = Timer() tot_time = 0 # Show some example dialogs: cnt = 0 while not world.epoch_done(): cnt += 1 world.parley() if opt['display_examples']: print("---") print(world.display() + "\n~~") if log_time.time() > log_every_n_secs: tot_time += log_time.time() print(str(int(tot_time)) + "s elapsed: " + str(world.report())) log_time.reset() if opt['num_examples'] > 0 and cnt >= opt['num_examples']: break if world.epoch_done(): print("EPOCH DONE") report = world.report() print(report) return report
def run_eval(valid_worlds, opt, datatype, max_exs=-1, write_log=False): """ Eval on validation/test data. :param valid_world: list of the pre-created validation worlds. :param opt: the options that specific the task, eval_task, etc :param datatype: the datatype to use, such as "valid" or "test" :param bool write_log: specifies to write metrics to file if the model_file is set :param int max_exs: limits the number of examples if max_exs > 0 """ if valid_worlds is None: # This isn't the primary worker, so we can just skip evaluation return None print('[ running eval: ' + datatype + ' ]') timer = Timer() reports = [] for v_world in valid_worlds: task_report = _run_single_eval(opt, v_world, max_exs / len(valid_worlds)) reports.append(task_report) tasks = [world.opt['task'] for world in valid_worlds] report = aggregate_task_reports(reports, tasks, micro=opt.get('aggregate_micro', True)) metrics = f'{datatype}:{report}' print(f'[ eval completed in {timer.time():.2f}s ]') print(metrics) # write to file if write_log and opt.get('model_file'): # Write out metrics f = open(opt['model_file'] + '.' + datatype, 'a+') f.write(metrics + '\n') f.close() return report
def eval_ppl(opt, build_dict): """Evaluates the the perplexity of a model. See the documentation for this file for more info. :param opt: option dict :param build_dict: function for building official dictionary. note that this function does not use the opt passed into eval_ppl, but rather should have hardcoded settings for its dictionary. """ dict_agent = build_dict() # create agents agent = create_agent(opt) world = create_task(opt, [agent, dict_agent], default_world=PerplexityWorld) # set up logging log_time = Timer() tot_time = 0 while not world.epoch_done(): world.parley() # process an example if log_time.time() > 1: # log every 1 sec tot_time += log_time.time() report = world.report() print('{}s elapsed, {}%% complete, {}'.format( int(tot_time), round_sigfigs(report['total'] / world.num_examples() * 100, 3), report)) log_time.reset() print('EPOCH DONE') tot_time += log_time.time() final_report = world.report() print('{}s elapsed: {}'.format(int(tot_time), final_report)) print("============================") print("FINAL PPL: " +str(final_report['ppl'])) if final_report.get('ppl', 0) == float('inf'): print('Note: you got inf perplexity. Consider adding (or raising) the ' 'minimum probability you assign to each possible word. If you ' 'assign zero probability to the correct token in the evaluation ' 'vocabulary, you get inf probability immediately.')
def eval_ppl(opt): """Evaluates the the perplexity and f1 of a model (and hits@1 if model has ranking enabled. """ dict_agent = build_dict() # create agents agent = create_agent(opt) world = create_task(opt, [agent, dict_agent], default_world=PerplexityWorld) world.dict = dict_agent # set up logging log_time = Timer() tot_time = 0 while not world.epoch_done(): world.parley() # process an example if log_time.time() > 1: # log every 1 sec tot_time += log_time.time() report = world.report() print('{}s elapsed, {}%% complete, {}'.format( int(tot_time), round_sigfigs(report['total'] / world.num_examples() * 100, 3), report)) log_time.reset() if world.epoch_done(): print('EPOCH DONE') tot_time += log_time.time() final_report = world.report() print('{}s elapsed: {}'.format(int(tot_time), final_report)) print("============================") print("FINAL PPL: " + str(final_report['ppl'])) if final_report.get('ppl', 0) == float('inf'): print('Note: you got inf perplexity. Consider adding (or raising) the ' 'minimum probability you assign to each possible word. If you ' 'assign zero probability to the correct token in the evaluation ' 'vocabulary, you get inf probability immediately.')
def __init__(self, opt, shared=None): super().__init__(opt, shared) if not shared: # don't enter this loop for shared instantiations self.path = opt['model_file'] # loss self.loss_time = Timer() self.losses = [] self.save_loss_every_n_secs = opt['save_loss_every_n_secs'] # attention if opt['task'] == "babi:All1k": self.tasks = {'babi:Task1k:' + str(i): 0 for i in range(1, 21)} elif opt['task'] == "babi:All10k": self.tasks = { 'babi:Task10k:' + str(i): 0 for i in range(1, 21) } else: self.tasks = {task: 0 for task in opt['task'].split(',')} self.attention_weights = {task: [] for task in self.tasks.keys()} self.save_attention_exs = opt['save_attention_exs']
def main(): # Get command line arguments parser = ParlaiParser(True, True) train = parser.add_argument_group('Training Loop Arguments') train.add_argument('-et', '--evaltask', help=('task to use for valid/test (defaults to the ' + 'one used for training if not set)')) train.add_argument('-d', '--display-examples', type='bool', default=False) train.add_argument('-e', '--num-epochs', type=float, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) train.add_argument('-lparl', '--log-every-n-parleys', type=int, default=100) train.add_argument('-vtim', '--validation-every-n-secs', type=float, default=-1) train.add_argument('-vparl', '--validation-every-n-parleys', type=int, default=-1) train.add_argument('-vme', '--validation-max-exs', type=int, default=-1, help='max examples to use during validation (default ' + '-1 uses all)') train.add_argument( '-vp', '--validation-patience', type=int, default=5, help=('number of iterations of validation where result ' + 'does not improve before we stop training')) train.add_argument( '-vmt', '--validation-metric', default='accuracy', help='key into report table for selecting best validation') train.add_argument('-dbf', '--dict-build-first', type='bool', default=True, help='build dictionary first before training agent') opt = parser.parse_args() # Set logging logger = logging.getLogger('DrQA') logger.setLevel(logging.INFO) fmt = logging.Formatter('%(asctime)s: %(message)s', '%m/%d/%Y %I:%M:%S %p') console = logging.StreamHandler() console.setFormatter(fmt) logger.addHandler(console) if 'log_file' in opt: logfile = logging.FileHandler(opt['log_file'], 'w') logfile.setFormatter(fmt) logger.addHandler(logfile) logger.info('[ COMMAND: %s ]' % ' '.join(sys.argv)) # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' logger.info("[ building dictionary first... ]") build_dict.build_dict(opt) # Create model and assign it to the specified task agent = create_agent(opt) world = create_task(opt, agent) train_time = Timer() validate_time = Timer() log_time = Timer() logger.info('[ training... ]') parleys = 0 total_exs = 0 max_exs = opt['num_epochs'] * len(world) max_parleys = math.ceil(max_exs / opt['batchsize']) best_valid = 0 impatience = 0 saved = False valid_world = None best_accuracy = 0 while True: world.parley() parleys += 1 if opt['num_epochs'] > 0 and parleys >= max_parleys: logger.info('[ num_epochs completed: {} ]'.format( opt['num_epochs'])) break if opt['max_train_time'] > 0 and train_time.time( ) > opt['max_train_time']: logger.info('[ max_train_time elapsed: {} ]'.format( train_time.time())) break # instead of every_n_secs, use n_parleys # if opt['log_every_n_secs'] > 0 and log_time.time() > opt['log_every_n_secs']: if opt['log_every_n_parleys'] > 0 and parleys % opt[ 'log_every_n_parleys'] == 0: if opt['display_examples']: logger.info(world.display() + '\n~~') logs = [] # time elapsed logs.append('time:{}s'.format(math.floor(train_time.time()))) logs.append('parleys:{}'.format(parleys)) # get report and update total examples seen so far if hasattr(agent, 'report'): train_report = agent.report() agent.reset_metrics() else: train_report = world.report() world.reset_metrics() if hasattr(train_report, 'get') and train_report.get('total'): total_exs += train_report['total'] logs.append('total_exs:{}'.format(total_exs)) # check if we should log amount of time remaining time_left = None if opt['num_epochs'] > 0: exs_per_sec = train_time.time() / total_exs time_left = (max_exs - total_exs) * exs_per_sec if opt['max_train_time'] > 0: other_time_left = opt['max_train_time'] - train_time.time() if time_left is not None: time_left = min(time_left, other_time_left) else: time_left = other_time_left if time_left is not None: logs.append('time_left:{}s'.format(math.floor(time_left))) # join log string and add full metrics report to end of log log = '[ {} ] {}'.format(' '.join(logs), train_report) logger.info(log) log_time.reset() # instead of every_n_secs, use n_parleys # if (opt['validation_every_n_secs'] > 0 and # validate_time.time() > opt['validation_every_n_secs']): if (opt['validation_every_n_parleys'] > 0 and parleys % opt['validation_every_n_parleys'] == 0): #if True : valid_report, valid_world = run_eval(agent, opt, 'valid', opt['validation_max_exs'], logger=logger) #if False : if valid_report[opt['validation_metric']] > best_accuracy: best_accuracy = valid_report[opt['validation_metric']] impatience = 0 logger.info('[ new best accuracy: ' + str(best_accuracy) + ' ]') world.save_agents() saved = True if best_accuracy == 1: logger.info('[ task solved! stopping. ]') break #if True: else: opt['learning_rate'] *= 0.5 agent.model.set_lrate(opt['learning_rate']) logger.info('[ Decrease learning_rate %.2e]' % opt['learning_rate']) impatience += 1 logger.info( '[ did not beat best accuracy: {} impatience: {} ]'.format( round(best_accuracy, 4), impatience)) validate_time.reset() if opt['validation_patience'] > 0 and impatience >= opt[ 'validation_patience']: logger.info('[ ran out of patience! stopping training. ]') break if opt['learning_rate'] < pow(10, -6): logger.info( '[ learning_rate < pow(10,-6) ! stopping training. ]') break world.shutdown() if not saved: world.save_agents() else: # reload best validation model agent = create_agent(opt) run_eval(agent, opt, 'valid', write_log=True, logger=logger) run_eval(agent, opt, 'test', write_log=True, logger=logger)
def __init__(self, opt, shared=None): opt = copy.deepcopy(opt) super().__init__(opt, shared) self.use_cuda = not opt['no_cuda'] and torch.cuda.is_available() self.is_combine_attr = (hasattr(self, 'other_task_datafiles') and self.other_task_datafiles) self.random_policy = opt.get('random_policy', False) self.count_sample = opt.get('count_sample', False) self.anti = opt.get('anti', False) if self.random_policy: random.seed(17) if not shared: if not self.stream and opt.get('pace_by', 'sample') == 'bucket': score_list = [episode[0][2] for episode in self.data.data] assert score_list == sorted(score_list) num_buckets = opt.get('num_buckets', int(self.num_episodes() / 10)) lb_indices = [ int(len(score_list) * i / num_buckets) for i in range(num_buckets) ] lbs = [score_list[idx] for idx in lb_indices] bucket_ids = [ self.sort_into_bucket(ctrl_val, lbs) for ctrl_val in score_list ] bucket_cnt = [0 for _ in range(num_buckets)] for i in range(num_buckets): bucket_cnt[i] = bucket_ids.count(i) self.bucket_cnt = bucket_cnt self.lastYs = [None] * self.bsz # build multiple task data self.tasks = [self.data] if self.is_combine_attr: print('[ build multiple task data ... ]') for datafile in self.other_task_datafiles: task_opt = copy.deepcopy(opt) task_opt['datafile'] = datafile self.tasks.append( DialogData(task_opt, data_loader=self.setup_data, cands=self.label_candidates())) print('[ build multiple task data done! ]') # record the selections of each subtasks self.subtasks = opt['subtasks'].split(':') self.subtask_counter = OrderedDict() self.p_selections = OrderedDict() self.c_selections = OrderedDict() for t in self.subtasks: self.subtask_counter[t] = 0 self.p_selections[t] = [] self.c_selections[t] = [] if self.count_sample and not self.stream: self.sample_counter = OrderedDict() for idx, t in enumerate(self.subtasks): self.sample_counter[t] = [ 0 for _ in self.tasks[idx].data ] # setup the tensorboard log if opt['tensorboard_log_teacher'] is True: opt['tensorboard_tag'] = 'task' teacher_metrics = 'reward,policy_loss,critic_loss,mean_advantage_reward,action_ent'.split( ',') opt['tensorboard_metrics'] = ','.join( opt['tensorboard_metrics'].split(',') + teacher_metrics) self.writer = TensorboardLogger(opt) else: self.lastYs = shared['lastYs'] self.tasks = shared['tasks'] if not self.stream and opt.get('pace_by', 'sample') == 'bucket': self.bucket_cnt = shared['bucket_cnt'] if 'writer' in shared: self.writer = shared['writer'] if 'subtask_counter' in shared: self.subtask_counter = shared['subtask_counter'] if 'p_selections' in shared: self.p_selections = shared['p_selections'] if 'c_selections' in shared: self.c_selections = shared['c_selections'] # build the policy net, criterion and optimizer here self.state_dim = 32 + len(self.tasks) # hand-craft features self.action_dim = len(self.tasks) if not shared: self.policy = PolicyNet(self.state_dim, self.action_dim) self.critic = CriticNet(self.state_dim, self.action_dim) init_teacher = get_init_teacher(opt, shared) if init_teacher is not None: # load teacher parameters if available print('[ Loading existing teacher params from {} ]' ''.format(init_teacher)) states = self.load(init_teacher) else: states = {} else: self.policy = shared['policy'] self.critic = shared['critic'] states = shared['states'] if ( # only build an optimizer if we're training 'train' in opt.get('datatype', '') and # and this is the main model shared is None): # for policy net self.optimizer = self.init_optim( [p for p in self.policy.parameters() if p.requires_grad], lr=opt['learningrate_teacher'], optim_states=states.get('optimizer'), saved_optim_type=states.get('optimizer_type')) self.scheduler = optim.lr_scheduler.ReduceLROnPlateau( self.optimizer, 'min', factor=0.8, # 0.5 --> 0.8 patience=5, # 3 -- > 5 verbose=True) if 'lr_scheduler' in states: self.scheduler.load_state_dict(states['lr_scheduler']) # for critic net self.optimizer_critic = self.init_optim( [p for p in self.critic.parameters() if p.requires_grad], lr=opt['learningrate_teacher_critic'], optim_states=states.get('optimizer_critic'), saved_optim_type=states.get('optimizer_type')) self.scheduler_critic = optim.lr_scheduler.ReduceLROnPlateau( self.optimizer_critic, 'min', factor=0.8, # 0.5 --> 0.8 patience=5, # 3 -- > 5 verbose=True) if 'lr_scheduler_critic' in states: self.scheduler_critic.load_state_dict( states['lr_scheduler_critic']) self.critic_criterion = torch.nn.SmoothL1Loss() self.reward_metric = opt.get('reward_metric', 'total_metric') self.reward_metric_mode = opt.get('reward_metric_mode', 'max') self.prev_prev_valid_report = states[ 'prev_prev_valid_report'] if 'prev_prev_valid_report' in states else None self.prev_valid_report = states[ 'prev_valid_report'] if 'prev_valid_report' in states else None self.current_valid_report = states[ 'current_valid_report'] if 'current_valid_report' in states else None self.saved_actions = states[ 'saved_actions'] if 'saved_actions' in states else OrderedDict() self.saved_state_actions = states[ 'saved_state_actions'] if 'saved_state_actions' in states else OrderedDict( ) if self.use_cuda: for k, v in self.saved_actions.items(): self.saved_actions[k] = v.cuda() for k, v in self.saved_state_actions.items(): self.saved_state_actions[k] = v.cuda() self._number_teacher_updates = states[ '_number_teacher_updates'] if '_number_teacher_updates' in states else 0 # enable the batch_act self.use_batch_act = self.bsz > 1 self.T = self.opt.get('T', 1000) self.c0 = self.opt.get('c0', 0.01) self.p = self.opt.get('p', 2) # setup the timer self.log_every_n_secs = opt['log_every_n_secs'] if opt['log_every_n_secs'] > 0 \ else float('inf') self.action_log_time = Timer() self.move_to_cuda()
def main(): # Get command line arguments parser = ParlaiParser(True, True) train = parser.add_argument_group('Training Loop Arguments') train.add_argument('-et', '--evaltask', help=('task to use for valid/test (defaults to the ' + 'one used for training if not set)')) train.add_argument('-d', '--display-examples', type='bool', default=False) train.add_argument('-e', '--num-epochs', type=int, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) train.add_argument('-vtim', '--validation-every-n-secs', type=float, default=-1) train.add_argument('-vme', '--validation-max-exs', type=int, default=-1, help='max examples to use during validation (default ' + '-1 uses all)') train.add_argument('-vp', '--validation-patience', type=int, default=5, help=('number of iterations of validation where result ' + 'does not improve before we stop training')) train.add_argument('-dbf', '--dict-build-first', type='bool', default=True, help='build dictionary first before training agent') opt = parser.parse_args() # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' build_dict.build_dict(opt) # Create model and assign it to the specified task agent = create_agent(opt) world = create_task(opt, agent) train_time = Timer() validate_time = Timer() log_time = Timer() print('[ training... ]') total_exs = 0 max_exs = opt['num_epochs'] * len(world) best_accuracy = 0 impatience = 0 saved = False while True: world.parley() if opt['num_epochs'] > 0 and total_exs >= max_exs: print('[ num_epochs completed: {} ]'.format(opt['num_epochs'])) break if opt['max_train_time'] > 0 and train_time.time() > opt['max_train_time']: print('[ max_train_time elapsed: {} ]'.format(train_time.time())) break if opt['log_every_n_secs'] > 0 and log_time.time() > opt['log_every_n_secs']: if opt['display_examples']: print(world.display() + '\n~~') logs = [] # time elapsed logs.append('time:{}s'.format(math.floor(train_time.time()))) # get report and update total examples seen so far if hasattr(agent, 'report'): train_report = agent.report() agent.reset_metrics() else: train_report = world.report() world.reset_metrics() total_exs += train_report['total'] logs.append('total_exs:{}'.format(total_exs)) # check if we should log amount of time remaining time_left = None if opt['num_epochs'] > 0: exs_per_sec = train_time.time() / total_exs time_left = (max_exs - total_exs) * exs_per_sec if opt['max_train_time'] > 0: other_time_left = opt['max_train_time'] - train_time.time() if time_left is not None: time_left = min(time_left, other_time_left) else: time_left = other_time_left if time_left is not None: logs.append('time_left:{}s'.format(math.floor(time_left))) # join log string and add full metrics report to end of log log = '[ {} ] {}'.format(' '.join(logs), train_report) print(log) log_time.reset() if (opt['validation_every_n_secs'] > 0 and validate_time.time() > opt['validation_every_n_secs']): valid_report = run_eval(agent, opt, 'valid', True, opt['validation_max_exs']) if valid_report['accuracy'] > best_accuracy: best_accuracy = valid_report['accuracy'] impatience = 0 print('[ new best accuracy: ' + str(best_accuracy) + ' ]') if opt['model_file']: agent.save(opt['model_file']) saved = True if best_accuracy == 1: print('[ task solved! stopping. ]') break else: impatience += 1 print('[ did not beat best accuracy: {} impatience: {} ]'.format( round(best_accuracy, 4), impatience)) validate_time.reset() if opt['validation_patience'] > 0 and impatience >= opt['validation_patience']: print('[ ran out of patience! stopping training. ]') break world.shutdown() if not saved: if opt['model_file']: agent.save(opt['model_file']) else: # reload best validation model agent = create_agent(opt) run_eval(agent, opt, 'valid') run_eval(agent, opt, 'test')
def __init__(self, opt): if isinstance(opt, ParlaiParser): opt = opt.parse_args() # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get( 'model_file_transmitter') and opt.get( 'model_file_receiver'): opt['dict_file'] = opt['model_file_transmitter'] + '_' + opt[ 'model_file_receiver'] + '.dict' print("[ building dictionary first... ]") build_dict(opt, skip_if_built=False) # Create model and assign it to the specified task print("[ create meta-agent ... ]") self.agent = create_agent(opt) print("[ create agent A ... ]") shared = self.agent.share() self.agent_a = create_agent_from_shared(shared) self.agent_a.set_id(suffix=' A') print("[ create agent B ... ]") self.agent_b = create_agent_from_shared(shared) # self.agent_b = create_agent(opt) self.agent_b.set_id(suffix=' B') # self.agent_a.copy(self.agent, 'transmitter') # self.agent_b.copy(self.agent, 'transmitter') self.world = create_selfplay_world(opt, [self.agent_a, self.agent_b]) # TODO: if batch, it is also not parallel # self.world = BatchSelfPlayWorld(opt, self_play_world) self.train_time = Timer() self.train_dis_time = Timer() self.validate_time = Timer() self.log_time = Timer() self.save_time = Timer() print('[ training... ]') self.parleys_episode = 0 self.max_num_epochs = opt[ 'num_epochs'] if opt['num_epochs'] > 0 else float('inf') self.max_train_time = opt[ 'max_train_time'] if opt['max_train_time'] > 0 else float('inf') self.log_every_n_secs = opt['log_every_n_secs'] if opt[ 'log_every_n_secs'] > 0 else float('inf') self.train_dis_every_n_secs = opt['train_display_every_n_secs'] if opt[ 'train_display_every_n_secs'] > 0 else float('inf') self.val_every_n_secs = opt['validation_every_n_secs'] if opt[ 'validation_every_n_secs'] > 0 else float('inf') self.save_every_n_secs = opt['save_every_n_secs'] if opt[ 'save_every_n_secs'] > 0 else float('inf') self.valid_optim = 1 if opt['validation_metric_mode'] == 'max' else -1 self.best_valid = None if opt.get('model_file_transmitter') and os.path.isfile( opt['model_file_transmitter'] + '.best_valid'): with open(opt['model_file_transmitter'] + ".best_valid", 'r') as f: x = f.readline() self.best_valid = float(x) f.close() self.impatience = 0 self.saved = False self.valid_world = None self.opt = opt if opt['tensorboard_log'] is True: self.writer = TensorboardLogger(opt)
def main(): # Get command line arguments parser = ParlaiParser(True, True) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.add_argument('-e', '--num-epochs', type=int, default=1) parser.add_argument('-ttim', '--max-train-time', type=float, default=float('inf')) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=1) parser.add_argument('-vtim', '--validation-every-n-secs', type=float, default=False) parser.add_argument('-vp', '--validation-patience', type=int, default=5, help=('number of iterations of validation where result ' + 'does not improve before we stop training')) parser.add_argument('-dbf', '--dict_build_first', type='bool', default=True, help='build dictionary first before training agent') opt = parser.parse_args() # Possibly build a dictionary (not all models do this). if opt['dict_build_first']: if 'dict_file' not in opt and 'model_file' in opt: opt['dict_file'] = opt['model_file'] + '.dict' build_dict.build_dict(opt) # Create model and assign it to the specified task agent = create_agent(opt) world = create_task(opt, agent) train_time = Timer() validate_time = Timer() log_time = Timer() print("[ training... ]") parleys = 0 num_parleys = opt['num_epochs'] * int(len(world) / opt['batchsize']) best_accuracy = 0 impatience = 0 saved = False for i in range(num_parleys): world.parley() parleys = parleys + 1 if train_time.time() > opt['max_train_time']: print("[ max_train_time elapsed: " + str(train_time.time()) + " ]") break if log_time.time() > opt['log_every_n_secs']: if opt['display_examples']: print(world.display() + "\n~~") parleys_per_sec = train_time.time() / parleys time_left = (num_parleys - parleys) * parleys_per_sec log = ("[ time:" + str(math.floor(train_time.time())) + "s parleys:" + str(parleys) + " time_left:" + str(math.floor(time_left)) + "s ]") if hasattr(agent, 'report'): log = log + str(agent.report()) else: log = log + str(world.report()) # TODO: world.reset_metrics() print(log) log_time.reset() if (opt['validation_every_n_secs'] and validate_time.time() > opt['validation_every_n_secs']): valid_report = run_eval(agent, opt, 'valid', True) if valid_report['accuracy'] > best_accuracy: best_accuracy = valid_report['accuracy'] impatience = 0 print("[ new best accuracy: " + str(best_accuracy) + " ]") if opt['model_file']: agent.save(opt['model_file']) saved = True if best_accuracy == 1: print('[ task solved! stopping. ]') break else: impatience += 1 print("[ did not beat best accuracy: " + str(best_accuracy) + " impatience: " + str(impatience) + " ]") validate_time.reset() if impatience >= opt['validation_patience']: print('[ ran out of patience! stopping. ]') break world.shutdown() if not saved: if opt['model_file']: agent.save(opt['model_file']) else: # reload best validation model agent = create_agent(opt) run_eval(agent, opt, 'valid') run_eval(agent, opt, 'test')
def main(parser): opt = parser.parse_args() # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' print("[ building dictionary first... ]") build_dict.build_dict(opt) # Create model and assign it to the specified task agent = create_agent(opt) world = create_task(opt, agent) train_time = Timer() validate_time = Timer() log_time = Timer() print('[ training... ]') parleys = 0 total_exs = 0 max_exs = opt['num_epochs'] * len(world) max_parleys = math.ceil(max_exs / opt['batchsize']) best_valid = 0 impatience = 0 saved = False valid_world = None with world: while True: world.parley() parleys += 1 if opt['num_epochs'] > 0 and parleys >= max_parleys: print('[ num_epochs completed: {} ]'.format(opt['num_epochs'])) break if opt['max_train_time'] > 0 and train_time.time( ) > opt['max_train_time']: print('[ max_train_time elapsed: {} ]'.format( train_time.time())) break if opt['log_every_n_secs'] > 0 and log_time.time( ) > opt['log_every_n_secs']: if opt['display_examples']: print(world.display() + '\n~~') logs = [] # time elapsed logs.append('time:{}s'.format(math.floor(train_time.time()))) logs.append('parleys:{}'.format(parleys)) # get report and update total examples seen so far if hasattr(agent, 'report'): train_report = agent.report() agent.reset_metrics() else: train_report = world.report() world.reset_metrics() if hasattr(train_report, 'get') and train_report.get('total'): total_exs += train_report['total'] logs.append('total_exs:{}'.format(total_exs)) # check if we should log amount of time remaining time_left = None if opt['num_epochs'] > 0: exs_per_sec = train_time.time() / total_exs time_left = (max_exs - total_exs) * exs_per_sec if opt['max_train_time'] > 0: other_time_left = opt['max_train_time'] - train_time.time() if time_left is not None: time_left = min(time_left, other_time_left) else: time_left = other_time_left if time_left is not None: logs.append('time_left:{}s'.format(math.floor(time_left))) # join log string and add full metrics report to end of log log = '[ {} ] {}'.format(' '.join(logs), train_report) print(log) log_time.reset() if (opt['validation_every_n_secs'] > 0 and validate_time.time() > opt['validation_every_n_secs']): valid_report, valid_world = run_eval(agent, opt, 'valid', opt['validation_max_exs'], valid_world=valid_world) if valid_report[opt['validation_metric']] > best_valid: best_valid = valid_report[opt['validation_metric']] impatience = 0 print('[ new best {}: {} ]'.format( opt['validation_metric'], best_valid)) world.save_agents() saved = True if opt['validation_metric'] == 'accuracy' and best_valid > 99.5: print('[ task solved! stopping. ]') break else: impatience += 1 print('[ did not beat best {}: {} impatience: {} ]'.format( opt['validation_metric'], round(best_valid, 4), impatience)) validate_time.reset() if opt['validation_patience'] > 0 and impatience >= opt[ 'validation_patience']: print('[ ran out of patience! stopping training. ]') break if not saved: # save agent world.save_agents() elif opt.get('model_file'): # reload best validation model agent = create_agent(opt) run_eval(agent, opt, 'valid', write_log=True) run_eval(agent, opt, 'test', write_log=True)
def eval_model(opt, printargs=None, print_parser=None): """Evaluates a model. Arguments: opt -- tells the evaluation function how to run print_parser -- if provided, prints the options that are set within the model after loading the model """ if printargs is not None: print('[ Deprecated Warning: eval_model no longer uses `printargs` ]') print_parser = printargs if print_parser is not None: if print_parser is True and isinstance(opt, ParlaiParser): print_parser = opt elif print_parser is False: print_parser = None if isinstance(opt, ParlaiParser): print( '[ Deprecated Warning: eval_model should be passed opt not Parser ]' ) opt = opt.parse_args() random.seed(42) nomodel = False # check to make sure the model file exists if opt.get('model_file') is None: nomodel = True elif not os.path.isfile(opt['model_file']): raise RuntimeError('WARNING: Model file does not exist, check to make ' 'sure it is correct: {}'.format(opt['model_file'])) # Create model and assign it to the specified task agent = create_agent(opt) if nomodel and hasattr(agent, 'load'): # double check that we didn't forget to set model_file on loadable model print('WARNING: model_file unset but model has a `load` function.') world = create_task(opt, agent) if print_parser: # Show arguments after loading model print_parser.opt = agent.opt print_parser.print_args() log_every_n_secs = opt.get('log_every_n_secs', -1) if log_every_n_secs <= 0: log_every_n_secs = float('inf') log_time = Timer() tot_time = 0 # Show some example dialogs: cnt = 0 while not world.epoch_done(): cnt += 1 world.parley() if opt['display_examples']: print("---") print(world.display() + "\n~~") if log_time.time() > log_every_n_secs: tot_time += log_time.time() print(str(int(tot_time)) + "s elapsed: " + str(world.report())) log_time.reset() if opt['num_examples'] > 0 and cnt >= opt['num_examples']: break if world.epoch_done(): print("EPOCH DONE") report = world.report() print(report) return report
def __train_single_model(opt): """Train single model. opt is a dictionary returned by arg_parse """ # Create model and assign it to the specified task agent = create_agent(opt) world = create_task(opt, agent) print('[ training... ]') train_dict = { 'train_time': Timer(), 'validate_time': Timer(), 'log_time': Timer(), 'new_epoch': None, 'epochs_done': 0, 'max_exs': opt['num_epochs'] * len(world), 'total_exs': 0, 'parleys': 0, 'max_parleys': math.ceil(opt['num_epochs'] * len(world) / opt['batchsize']), 'best_metrics': opt['chosen_metrics'], 'best_metrics_value': 0, 'impatience': 0, 'lr_drop_impatience': 0, 'saved': False, 'train_report': None, 'train_report_agent': None, 'train_report_world': None, 'break': None } try: while True: world.parley() train_dict['parleys'] += 1 train_dict['new_epoch'] = world.epoch_done() if train_dict['new_epoch']: world.reset() train_dict['epochs_done'] += 1 if opt['num_epochs'] > 0 and train_dict['parleys'] >= train_dict[ 'max_parleys']: print('[ num_epochs completed: {} ]'.format(opt['num_epochs'])) break if 0 < opt['max_train_time'] < train_dict['train_time'].time(): print('[ max_train_time elapsed: {} ]'.format( train_dict['train_time'].time())) break world, agent, train_dict = __train_log(opt, world, agent, train_dict) _, agent, train_dict = __intermediate_validation( opt, world, agent, train_dict) if train_dict['break']: break except KeyboardInterrupt: print('Stopped training, starting testing') if not train_dict['saved']: world.save_agents() world.shutdown() agent.shutdown() # reload best validation model vopt = copy.deepcopy(opt) if vopt.get('evaltask'): vopt['task'] = vopt['evaltask'] vopt['datatype'] = 'valid' vopt['pretrained_model'] = vopt['model_file'] agent = create_agent(vopt) valid_world = create_task(vopt, agent) metrics, _ = __evaluate_model(valid_world, vopt['batchsize'], 'valid', vopt['display_examples'], vopt['validation_max_exs']) valid_world.shutdown() agent.shutdown() return metrics
def eval_ppl(opt, build_dict=None, dict_file=None): """Evaluates the the perplexity of a model. This uses a dictionary which implements the following functions: - tokenize(text): splits string up into list of tokens - __in__(text): checks whether dictionary contains a token - keys(): returns an iterator over all tokens in the dictionary :param opt: option dict :param build_dict: function which returns a dictionary class implementing the functions above. :param dict_file: file used when loading the dictionary class set via the "dictionary_class" argument (defaults to parlai.core.dict:DictionaryAgent). Either build_dict or dict_file must be set (both default to None) to determine the dictionary used for the evaluation. """ if not build_dict and not dict_file: raise RuntimeError('eval_ppl script either needs a dictionary build ' 'function or a dictionary file.') if build_dict: dict_agent = build_dict() else: dict_opt = copy.deepcopy(opt) dict_opt['model'] = dict_opt.get( 'dictionary_class', 'parlai.core.dict:DictionaryAgent' ) dict_opt['model_file'] = dict_file if 'override' in dict_opt: del dict_opt['override'] dict_agent = create_agent(dict_opt, requireModelExists=True) # create agents agent = create_agent(opt) world = create_task(opt, [agent, dict_agent], default_world=PerplexityWorld) # set up logging log_time = Timer() tot_time = 0 while not world.epoch_done(): world.parley() # process an example if log_time.time() > 1: # log every 1 sec tot_time += log_time.time() report = world.report() print('{}s elapsed, {}%% complete, {}'.format( int(tot_time), round_sigfigs(report['exs'] / world.num_examples() * 100, 3), report)) log_time.reset() print('EPOCH DONE') tot_time += log_time.time() final_report = world.report() print('{}s elapsed: {}'.format(int(tot_time), final_report)) print("============================") print("FINAL PPL: " + str(final_report['ppl'])) if final_report.get('ppl', 0) == float('inf'): print('Note: you got inf perplexity. Consider adding (or raising) the ' 'minimum probability you assign to each possible word. If you ' 'assign zero probability to the correct token in the evaluation ' 'vocabulary, you get inf probability immediately.')
def __init__(self, opt): signal.signal(signal.SIGINT, signal.default_int_handler) if isinstance(opt, ParlaiParser): opt = opt.parse_args() # Possibly load from checkpoint trainstats_suffix = '.trainstats' if (opt.get('model_file') and isfile(opt['model_file'] + '.checkpoint')): opt['init_model'] = opt['model_file'] + '.checkpoint' trainstats_suffix = '.checkpoint.trainstats' else: pass # TODO for testing only # raise RuntimeError('WARNING: Reinforcement learning' # ' must be initialized by a model.checkpoint ' # 'file and {} does not exist.'.format( # opt['model_file'] + '.checkpoint')) # Possibly build a dictionary (not all models do this). if (opt['dict_build_first'] and not (opt.get('dict_file') or opt.get('model_file'))): raise RuntimeError('WARNING: For train_model, ' 'please specify either a ' 'model_file or dict_file.') if opt['dict_build_first'] and 'dict_file' in opt: if opt.get('pytorch_teacher_task'): opt['dict_file'] = get_pyt_dict_file(opt) elif opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' print("[ building dictionary first... ]") build_dict(opt, skip_if_built=True) # Create model and assign it to the specified task self.agent = create_agent(opt) # Freeze the model for the static dialogue partner static_agent = copy.deepcopy(self.agent) self.agent.id = ACTIVE static_agent.id = STATIC freeze_agent(static_agent) self.world = create_task(opt, self.agent, static_agent) # set up timers self.train_time = Timer() self.validate_time = Timer() self.log_time = Timer() self.save_time = Timer() print('[ training... ]') self.parleys = 0 self.max_num_epochs = (opt['num_epochs'] if opt['num_epochs'] > 0 else float('inf')) self.max_train_time = (opt['max_train_time'] if opt['max_train_time'] > 0 else float('inf')) self.log_every_n_secs = (opt['log_every_n_secs'] if opt['log_every_n_secs'] > 0 else float('inf')) self.val_every_n_secs = (opt['validation_every_n_secs'] if opt['validation_every_n_secs'] > 0 else float('inf')) self.save_every_n_secs = (opt['save_every_n_secs'] if opt['save_every_n_secs'] > 0 else float('inf')) self.val_every_n_epochs = (opt['validation_every_n_epochs'] if opt['validation_every_n_epochs'] > 0 else float('inf')) # smart defaults for --validation-metric-mode if opt['validation_metric'] in {'loss', 'ppl', 'mean_rank'}: opt['validation_metric_mode'] = 'min' elif opt['validation_metric'] in { 'accuracy', 'hits@1', 'hits@5', 'f1', 'bleu' }: opt['validation_metric_mode'] = 'max' if opt.get('validation_metric_mode') is None: opt['validation_metric_mode'] = 'max' self.last_valid_epoch = 0 self.valid_optim = (1 if opt['validation_metric_mode'] == 'max' else -1) self.valid_reports = [] self.best_valid = None if (opt.get('model_file') and isfile(opt['model_file'] + '.best_valid')): with open(opt['model_file'] + ".best_valid", 'r') as f: x = f.readline() self.best_valid = float(x) f.close() self.impatience = 0 self.saved = False self.valid_world = None self.opt = opt # we may have been preempted, make sure we note that amount self._preempted_epochs = 0.0 if (opt.get('model_file') and isfile(opt['model_file'] + trainstats_suffix)): # looks like we were preempted. make sure we load up our total # training stats, etc with open(opt['model_file'] + trainstats_suffix) as ts: obj = json.load(ts) self._preempted_epochs = obj.get('total_epochs', 0) self.train_time.total = obj.get('train_time', 0) self.impatience = obj.get('impatience', 0) self.valid_reports = obj.get('valid_reports', []) if opt['tensorboard_log'] is True: self.writer = TensorboardLogger(opt)
def __init__(self, opt): # if python is called from a non-interactive shell, like a bash script, # it will by-default ignore SIGINTs, and KeyboardInterrupt exceptions are # not produced. This line brings them back signal.signal(signal.SIGINT, signal.default_int_handler) if isinstance(opt, ParlaiParser): print( '[ Deprecated Warning: TrainLoop should be passed opt not Parser ]' ) opt = opt.parse_args() # Possibly load from checkpoint trainstats_suffix = '.trainstats' # we might load training statistics from here if opt['load_from_checkpoint'] and opt.get( 'model_file') and os.path.isfile(opt['model_file'] + '.checkpoint'): opt['init_model'] = opt['model_file'] + '.checkpoint' trainstats_suffix = '.checkpoint.trainstats' # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: # If data built via pytorch data teacher, we need to load prebuilt dict if opt.get('pytorch_teacher_task'): opt['dict_file'] = get_pyt_dict_file(opt) elif opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' print("[ building dictionary first... ]") build_dict(opt, skip_if_built=True) # Create model and assign it to the specified task self.agent = create_agent(opt) #specify model such as seq2seq self.world = create_task(opt, self.agent) # bacthworld or other world # set up timers self.train_time = Timer() self.validate_time = Timer() self.log_time = Timer() self.save_time = Timer() print('[ training... ]') self.parleys = 0 self.max_num_epochs = opt[ 'num_epochs'] if opt['num_epochs'] > 0 else float('inf') self.max_train_time = opt['max_train_time'] if opt['max_train_time'] > 0 \ else float('inf') self.log_every_n_secs = opt['log_every_n_secs'] if opt['log_every_n_secs'] > 0 \ else float('inf') self.val_every_n_secs = \ opt['validation_every_n_secs'] if opt['validation_every_n_secs'] > 0 \ else float('inf') self.save_every_n_secs = opt['save_every_n_secs'] if opt['save_every_n_secs'] \ > 0 else float('inf') self.val_every_n_epochs = \ opt['validation_every_n_epochs'] if opt['validation_every_n_epochs'] > 0 \ else float('inf') # smart defaults for --validation-metric-mode if opt['validation_metric'] in {'loss', 'ppl', 'mean_rank'}: opt['validation_metric_mode'] = 'min' elif opt['validation_metric'] in { 'accuracy', 'hits@1', 'hits@5', 'f1', 'bleu' }: opt['validation_metric_mode'] = 'max' if opt.get('validation_metric_mode') is None: opt['validation_metric_mode'] = 'max' self.last_valid_epoch = 0 self.valid_optim = 1 if opt['validation_metric_mode'] == 'max' else -1 self.best_valid = None if opt.get('model_file') and os.path.isfile(opt['model_file'] + '.best_valid'): with open(opt['model_file'] + ".best_valid", 'r') as f: x = f.readline() self.best_valid = float(x) f.close() self.impatience = 0 self.saved = False self.valid_world = None self.opt = opt # we may have been preempted, make sure we note that amount self._preempted_epochs = 0.0 if (opt.get('model_file') and os.path.isfile(opt['model_file'] + trainstats_suffix)): # looks like we were preempted. make sure we load up our total # training stats, etc with open(opt['model_file'] + trainstats_suffix) as ts: obj = json.load(ts) self._preempted_epochs = obj.get('total_epochs', 0) self.train_time.total = obj.get('train_time', 0) self.impatience = obj.get('impatience', 0) if opt['tensorboard_log'] is True: self.writer = TensorboardLogger(opt)
def train_model(opt): # Possibly build a dictionary (not all models do this). if opt['dict_build_first'] and 'dict_file' in opt: if opt['dict_file'] is None and opt.get('pretrained_model'): opt['dict_file'] = opt['pretrained_model'] + '.dict' if opt['dict_file'] is None and opt.get('model_file'): opt['dict_file'] = opt['model_file'] + '.dict' print("[ building dictionary first... ]") build_dict.build_dict(opt) # Create model and assign it to the specified task agent = create_agent(opt) if opt['datatype'].split(':')[0] == 'train': world = create_task(opt, agent) train_time = Timer() validate_time = Timer() log_time = Timer() print('[ training... ]') parleys = 0 total_exs = 0 max_exs = opt['num_epochs'] * len(world) epochs_done = 0 max_parleys = math.ceil(max_exs / opt['batchsize']) best_metric_name = opt['chosen_metric'] best_metric = 0 impatience = 0 saved = False valid_world = None try: while True: world.parley() parleys += 1 new_epoch = world.epoch_done() if new_epoch: world.reset() epochs_done += 1 if opt['num_epochs'] > 0 and parleys >= max_parleys: print('[ num_epochs completed: {} ]'.format(opt['num_epochs'])) break if 0 < opt['max_train_time'] < train_time.time(): print('[ max_train_time elapsed: {} ]'.format(train_time.time())) break if (0 < opt['log_every_n_secs'] < log_time.time()) or \ (opt['log_every_n_epochs'] > 0 and new_epoch and (epochs_done % opt['log_every_n_epochs']) == 0): if opt['display_examples']: print(world.display() + '\n~~') logs = list() # time elapsed logs.append('time:{}s'.format(math.floor(train_time.time()))) logs.append('parleys:{}'.format(parleys)) if epochs_done > 0: logs.append('epochs done:{}'.format(epochs_done)) # get report and update total examples seen so far if hasattr(agent, 'report'): train_report = agent.report() agent.reset_metrics() else: train_report = world.report() world.reset_metrics() if hasattr(train_report, 'get') and train_report.get('total'): total_exs += train_report['total'] logs.append('total_exs:{}'.format(total_exs)) # check if we should log amount of time remaining time_left = None if opt['num_epochs'] > 0 and total_exs > 0: exs_per_sec = train_time.time() / total_exs time_left = (max_exs - total_exs) * exs_per_sec if opt['max_train_time'] > 0: other_time_left = opt['max_train_time'] - train_time.time() if time_left is not None: time_left = min(time_left, other_time_left) else: time_left = other_time_left if time_left is not None: logs.append('time_left:{}s'.format(math.floor(time_left))) # join log string and add full metrics report to end of log log = '[ {} ] {}'.format(' '.join(logs), train_report) print(log) log_time.reset() if 0 < opt['validation_every_n_secs'] < validate_time.time() or \ (opt['validation_every_n_epochs'] > 0 and new_epoch and ( epochs_done % opt['validation_every_n_epochs']) == 0): valid_report, valid_world = run_eval(agent, opt, 'valid', opt['validation_max_exs'], valid_world=valid_world) if best_metric_name not in valid_report and 'accuracy' in valid_report: best_metric_name = 'accuracy' if valid_report[best_metric_name] > best_metric: best_metric = valid_report[best_metric_name] impatience = 0 print('[ new best ' + best_metric_name + ': ' + str(best_metric) + ' ]') world.save_agents() saved = True if best_metric == 1: print('[ task solved! stopping. ]') break else: impatience += 1 print('[ did not beat best ' + best_metric_name + ': {} impatience: {} ]'.format( round(best_metric, 4), impatience)) validate_time.reset() if 0 < opt['validation_patience'] <= impatience: print('[ ran out of patience! stopping training. ]') break except KeyboardInterrupt: print('Stopped training, starting testing') if not saved: world.save_agents() # else: world.shutdown() # reload best validation model opt['pretrained_model'] = opt['model_file'] agent = create_agent(opt) run_eval(agent, opt, 'valid', write_log=True) run_eval(agent, opt, 'test', write_log=True) else: run_eval(agent, opt, opt['datatype'], write_log=True) agent.shutdown()