def train(self): n_iter = 0 for epoch in range(self.configs["n_epoch"]): logger.info("Epoch: {}".format(epoch)) epoch_loss = 0 for values, indicators_train, indicators_eval in tqdm( self.iterator): values = values.to(self.device) indicators_train = indicators_train.to(self.device) self.opt.zero_grad() y, y_pred, indicators_y = self._iter( values=values, indicators=indicators_train) iter_loss = self.get_loss(y=y, y_pred=y_pred, indicators=indicators_y) iter_loss.backward() self.opt.step() n_iter += 1 epoch_loss += iter_loss logger.info("Training loss: {}".format(epoch_loss / len(self.iterator))) if self.is_eval: evaluator = Evaluator(configs=None, model=self.model, iterator=self.iterator_eval) eval_loss = evaluator.eval() logger.info("Eval loss: {}".format(eval_loss)) if self.early_stopping(loss=eval_loss): return else: if self.early_stopping(loss=epoch_loss / len(self.iterator)): return
def test_store_multiple(self): s = State() s.append(PredicateInstance('happy', Instance('a'))) s.append(PredicateInstance('unhappy', Instance('a'))) self.assertStateContains(PredicateInstance('happy', Instance('a')), Evaluator(state=s)) self.assertStateContains(PredicateInstance('unhappy', Instance('a')), Evaluator(state=s))
def main(): argv = sys.argv if len(argv) == 3: model_path, data_file = argv[1:] evaluator = Evaluator(model_path, data_file) evaluator.evaluate() else: print('Usage: "python evaluate.py $model_path $data_file"')
def test(): assert cfg.load_path is not None output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: raise RuntimeError('cannot find pre-processed dataset') else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] train_data_obj.filter_data('test') dev_data_obj.filter_data('test') test_data_obj.filter_data('test') emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove with tf.variable_scope(network_type) as scope: if network_type in model_type_set: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) if cfg.gpu_mem is None: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True) graph_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) else: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem) graph_config = tf.ConfigProto(gpu_options=gpu_options) # graph_config.gpu_options.allow_growth = True sess = tf.Session(config=graph_config) graphHandler.initialize(sess) # todo: test model # ---- dev ---- dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None) _logger.add('==> for dev, loss: %.4f, accuracy: %.4f' % (dev_loss, dev_accu)) # ---- test ---- test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None) _logger.add('~~> for test, loss: %.4f, accuracy: %.4f' % (test_loss, test_accu)) train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj, None) _logger.add('--> for test, loss: %.4f, accuracy: %.4f' % (train_loss, train_accu))
def search(): if request.method == 'POST': screen_name = request.form['screen_name'] evaluator = Evaluator(app.root_path) status, result = evaluator.evaluate(screen_name) if status != 200: raise Err(result, status_code=status) return json.dumps(result) else: return False
def main(argv): # initialize evaluator and evaluate evaluator = Evaluator( FLAGS.experiment_path, FLAGS.src_test_path, FLAGS.tgt_test_path, FLAGS.save_as_pretrained, ) evaluator.evaluate()
def score_embedding(embedding, groups): evaluator = Evaluator(groups) evaluator.evaluate(embedding) print(" RESULTS") print("==============") print("OPP score: %f" % evaluator.opp) print("Accuracy: %f" % evaluator.accuracy) print("---------------------------------") print("Total number of test groups: %d" % evaluator.num_total_groups) print("Number of filtered test groups: %d (%f%%)" % (evaluator.num_filtered_groups, evaluator.percent_filtered_groups)) print("Total number of non-OOV test cases: %d" % evaluator.num_cases) print("Number of filtered cluster entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_cluster_items, evaluator.num_total_cluster_items, evaluator.percent_filtered_cluster_items)) print("Number of filtered outlier entities: %d/%d (mean per %% cluster: %f%%)" % (evaluator.num_filtered_outliers, evaluator.num_total_outliers, evaluator.percent_filtered_outliers))
def test_general(self): ev = Evaluator() possible_keywords = [ 'family', 'food', 'outdoor', 'rest', 'indoor', 'sports', 'science', 'culture', 'history' ] dg = DataGenerator(possible_keywords) gen_query = dg.generate(1)[0] gen_data = dg.generate(10) cf1 = Type1(euclidean_distance, combined_cosine_similarity, 0.33, 0.33, 0.33, disable_thresholds=True) cf2 = Type2(euclidean_distance, combined_cosine_similarity, 0.33, 0.33, 0.33, disable_thresholds=True) cf3 = Type3(euclidean_distance, combined_cosine_similarity, 0.33, 0.33, 0.33, disable_thresholds=True) ns1 = NaiveSolver(gen_query, gen_data, cf1, result_length=10, max_subset_size=6) ns2 = NaiveSolver(gen_query, gen_data, cf2, result_length=10, max_subset_size=6) ns3 = NaiveSolver(gen_query, gen_data, cf3, result_length=10, max_subset_size=6) ev.add_solver(ns1) ev.add_solver(ns2) ev.add_solver(ns3) ev.evaluate() results = ev.get_results() self.assertEqual(len(results), 3) self.assertEqual(len(results[0]), 2) self.assertEqual(len(results[1]), 2) self.assertEqual(len(results[2]), 2) self.assertEqual(len(results[0][0]), 10) self.assertEqual(len(results[1][0]), 10) self.assertEqual(len(results[2][0]), 10)
def run(args) -> None: """ This function parses the arguments provided and runs the experiment :param args: the arguments provided by the user. :return: None """ selector = GeneratorSelector() generators = selector.getAllGenerators() benchmark = Benchmark(benchmarkType=args.benchmark, verbose=args.verbose) evaluator = Evaluator(benchmark=benchmark, generators=generators, timeLimit=benchmark.timeLimit, verbose=args.verbose) evaluator.evaluate() sys.exit(0)
def run_one(config): # create work dir for this run # moses can't handle paths with colons timestamp = datetime.datetime.now().strftime('%Y-%m-%dT%H.%M.%S') run_work_dir = os.path.join(base_work_dir, timestamp) assert not os.path.exists(run_work_dir) os.makedirs(run_work_dir) config.put('work_dir', run_work_dir) if os.path.exists('latest'): os.remove('latest') os.symlink(run_work_dir, 'latest') # set up logging if config.run == 'debug': logging.basicConfig(level=logging.DEBUG) else: log_path = os.path.join(run_work_dir, LOGFILE_NAME) logging.basicConfig(filename=log_path, level=logging.INFO) experiment = SMTSemparseExperiment(config) if config.run == 'debug': experiment.run_fold(1) elif config.run == 'dev': for i in range(10): experiment.run_fold(i) elif config.run == 'test': experiment.run_split() else: assert False if not config.nlg: logging.info('evaluating') Evaluator(config).run()
def set_local_parameters(self, random_seed, losses, metrics, loss_weights, checkpoint_every, print_every): self.random_seed = random_seed if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) self.losses = losses self.metrics = metrics self.loss_weights = loss_weights or len(losses)*[1.] self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics) self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every self.logger = logging.getLogger(__name__) self._stop_training = False
def test(): assert cfg.load_path is not None output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: raise RuntimeError('cannot find pre-processed dataset') else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] train_data_obj.filter_data() dev_data_obj.filter_data() test_data_obj.filter_data() emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove with tf.variable_scope(network_type) as scope: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) sess = tf.Session(config=sess_config_gene(cfg.gpu_mem)) graphHandler.initialize(sess) # test model # ---- dev ---- dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None) _logger.add('==> for dev, loss: %.4f, accuracy: %.4f' % (dev_loss, dev_accu)) # ---- test ---- test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None) _logger.add('~~> for test, loss: %.4f, accuracy: %.4f' % (test_loss, test_accu)) train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj, None) _logger.add('--> for test, loss: %.4f, accuracy: %.4f' % (train_loss, train_accu))
def inference_time(): assert cfg.load_path is not None output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: raise RuntimeError('cannot find pre-processed dataset') else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] train_data_obj.filter_data() dev_data_obj.filter_data() test_data_obj.filter_data() emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove with tf.variable_scope(network_type) as scope: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) sess = tf.Session(config=sess_config_gene(cfg.gpu_mem)) graphHandler.initialize(sess) repeat_num = 10 time_counter = TimeCounter() for i in range(5): evaluator.get_evaluation(sess, dev_data_obj, None) for t in range(repeat_num): dev_loss, dev_accu = evaluator.get_evaluation( sess, dev_data_obj, None, time_counter=time_counter) # _logger.add('~~> for test, loss: %.4f, accuracy: %.4f' % # (test_loss, test_accu)) time_rec = time_counter.update_data_round(t + 1) print('this: ', time_rec[0], '; all: ', time_rec[1])
def test_store_and_access(self): s = State() p = PredicateInstance('happy', Instance('a')) s.append(p) self.assertEqual([p], s.fetch('happy', [Instance('a')])) self.assertStateContains(PredicateInstance('happy', Instance('a')), Evaluator(state=s))
def __init__(self, data): self._evaluator = Evaluator(data) self._item_count = self._evaluator.generator.rule_set_len self._best_ind = None self._best_fit_val = 0 # To assure reproductibility, the RNG seed is set prior to the items # dict initialization. It is also seeded in main(). random.seed(64) # # Create the item dictionary: item name is an integer, and value is # # a weight. # items = {} # # Create random items and store them in the items' dictionary. (18 rule * 9) # for i in range(NBR_ITEMS): # items[i] = i; creator.create("FitnessMax", base.Fitness, weights=(1.0, )) creator.create("Individual", set, fitness=creator.FitnessMax) self.toolbox = base.Toolbox() # Attribute generator # define 'attr_item' to be an attribute ('gene') # which corresponds to integers sampled uniformly # from the range [1,10] (i.e. 1 to 10 with equal probability) self.toolbox.register("attr_item", random.randrange, self._item_count) # Structure initializers # define 'individual' to be an individual # consisting of 10 'attr_item' elements ('genes') self.toolbox.register("individual", tools.initRepeat, creator.Individual, self.toolbox.attr_item, IND_INIT_SIZE) # define the population to be a list of individuals self.toolbox.register("population", tools.initRepeat, list, self.toolbox.individual) self.toolbox.register("evaluate", self.eval_ind) self.toolbox.register("mate", self.cx_ind) self.toolbox.register("mutate", self.mutate_ind) self.toolbox.register("select", tools.selNSGA2)
def evaluate(self): logger.info('Begin evaluate of Impulso') if self.args.experiment_id: module = importlib.import_module( f'experiments.{self.args.experiment_id}.src.evaluator') Evaluator = getattr(module, 'Evaluator') else: from src.evaluator import Evaluator evaluator = Evaluator(self.args.exec_type, self.hparams) evaluator.load_data() evaluator.evaluate() logger.info('End evaluate of Impulso')
def test_random_distribution(self, mocked_randint): c = Character() simulation = Simulation( Evaluator(actors=[c], rules=[ Rule('rule1', [], [Predicate('p1', 0)], prob=1), Rule('rule2', [], [Predicate('p2', 0)], prob=4) ])) mocked_randint.return_value = 1 simulation.step() self.assertStateContains(PredicateInstance('p1', c), simulation.evaluator) self.assertStateDoesNotContain(PredicateInstance('p2', c), simulation.evaluator) mocked_randint.return_value = 2 simulation.step() self.assertStateContains(PredicateInstance('p1', c), simulation.evaluator) self.assertStateContains(PredicateInstance('p2', c), simulation.evaluator)
def main(): energy_list = [] garbage_list = [] f = open(map_path) z = env(f) for i in range(3): zone = copy.deepcopy(z) zone.maze = clean_maze(zone.maze) bond = Agent(zone) current_time = 0 bond.agent_map[bond.posX][bond.posY] = 1 calculus = Evaluator() while current_time < life_time: run(zone, bond, calculus, current_time, i) current_time += 1 print("Cleaned dirty: " + str(calculus.cleaned_dirty)) print("consumed energy: " + str(calculus.consumed_energy)) print("Avg dirty: " + str(calculus.total_dirty / life_time)) energy_list.append(calculus.consumed_energy) garbage_list.append(calculus.total_dirty / life_time) labels = ['random', 'reflex', 'model-based'] show(labels, garbage_list, energy_list)
def train(cfg): # ======= data =========== loaded_data = load_file(cfg['processed_path'], 'processed_datasets', mode='pickle') if loaded_data is None and cfg['load_preproc']: train_data_obj = Dataset(cfg['train_data_path'], 'train', bpe_path=cfg['bpe_data_dir']) dev_data_obj = Dataset(cfg['dev_data_path'], 'dev', train_data_obj.bpe_enc) test_data_obj = Dataset(cfg['test_data_path'], 'test', train_data_obj.bpe_enc) dev_data_obj.sort_example() test_data_obj.sort_example() save_file({'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj}, cfg['processed_path']) else: train_data_obj = loaded_data['train_data_obj'] dev_data_obj = loaded_data['dev_data_obj'] test_data_obj = loaded_data['test_data_obj'] # 1. delete too long sentence, max len: 50 train_data_obj.filter_example(cfg['max_sent_len']) # ========= build network ====== if cfg['model_class'] is None: print('Did not find the model, please check ' '1) module name 2) class name and 3) implementation of get_default_model_paramters(). exit!') exit() with tf.variable_scope("model") as scp: model = cfg['model_class']( cfg, train_data_obj.bpe_enc.n_vocab, 3, train_data_obj.bpe_enc.get_idx_from_token(Dataset.CLASSIFY_TOKEN), scp.name ) # ======= build session ======= graph_handler = GraphHandler(model, cfg) evaluator = Evaluator(model, cfg) performance_record = PerformanceRecoder(cfg['ckpt_dir'], cfg['save_model'], cfg['save_num']) sess = graph_handler.initialize() model.load_openai_pretrained_transformer_model( sess, cfg['pretrained_transformer_dir']) # ======== begin to train ======= loss_task_ma, loss_lm_ma, accuracy_ma = MovingAverage(), MovingAverage(), MovingAverage() for example_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_iter( cfg['train_batch_size'], cfg['n_steps']): global_step_val = sess.run(model.global_step) + 1 step_out = model.step(sess, example_batch, cfg['summary_period']) loss_task_ma(step_out['task_loss']), accuracy_ma(step_out['accuracy']), loss_lm_ma(step_out['lm_loss']) graph_handler.add_summary(step_out['summary'], global_step_val) if global_step_val % 100 == 0: log.info('data round: %d: %d/%d, global step:%d -- loss: %.4f, accu: %.4f' % (data_round, idx_b, batch_num, global_step_val, loss_task_ma.value, accuracy_ma.value)) if 'lm_loss' in step_out: log.info('\tauxiliary language model perplexity: %.4f' % math.exp(loss_lm_ma.value)) # eval if global_step_val % cfg['eval_period'] == 0: dev_res = evaluator.get_evaluation(sess, dev_data_obj, global_step_val) log.info('==> for dev, loss: %.4f, accuracy: %.4f' % (dev_res['loss'], dev_res['accuracy'])) criterion_metric = dev_res['accuracy'] if not test_data_obj.no_label: test_res = evaluator.get_evaluation(sess, test_data_obj, global_step_val) log.info('~~> for test, loss: %.4f, accuracy: %.4f' % (test_res['loss'], test_res['accuracy'])) is_in_top, deleted_step = performance_record.update_top_list(global_step_val, criterion_metric, sess) if is_in_top: # get prediction for non-labeled test data if test_data_obj.no_label and global_step_val > 0.4 * cfg['n_steps']: evaluator.get_evaluation(sess, test_data_obj, global_step_val, write_predict=True) # todo: time count log.info(str(performance_record.top_list))
help="the number of orders to parse", type=int, default=2400) parser.add_argument("--process_num", dest="NUM_PROCESSES", help="the number of processes", type=int, default=16) parser.add_argument("--weight_amount", dest="WEIGHT_AMOUNT", help="the weight of the amountCost in the objective function", type=float, default=15) parser.add_argument("--weight_time", dest="WEIGHT_TIME", help="the weight of the timeCost in the objective function", type=float, default=1) parser.add_argument("--hub_cost_const", dest="HUB_BUILT_COST_CONST", help="the constant part of cost of building a hub", type=int, default=3000) parser.add_argument("--hub_cost_vary", dest="HUB_BUILT_COST_VARY", help="the variable part cost of building a hub", type=float, default=2.0) parser.add_argument("--hub_ratio", dest="HUB_UNIT_COST_RATIO", help="the cutoff of unit price of a hub", type=float, default=0.7) parser.add_argument("--hub_capacity", dest="HUB_CAPACITY", help="the capacity of the hub(in prob3)", type=int, default=500) args = parser.parse_args() arg_dict = edict(vars(args)) merge_a_into_b(arg_dict, cfg) print("Using the config:") print(cfg) solver = Solver() evaluator = Evaluator() solver.solve(problem_id=args.PROBLEM_ID) # disable the tune mode but plot the distribution, see the outputs for more detail evaluator.evaluate(prob_id=args.PROBLEM_ID, tune=False, plot=True)
def train(): output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: train_data_obj = Dataset(cfg.train_data_path, 'train') dev_data_obj = Dataset(cfg.dev_data_path, 'dev', train_data_obj.dicts) save_file( { 'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj }, cfg.processed_path) else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove output_cls_num = len( train_data_obj.dicts['sub_cls']) if cfg.fine_grained else len( train_data_obj.dicts['cls']) with tf.variable_scope(network_type) as scope: if network_type in model_set: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], output_cls_num, scope=scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) performRecoder = PerformRecoder(3) if cfg.gpu_mem is None: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True) graph_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) else: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem) graph_config = tf.ConfigProto(gpu_options=gpu_options) # graph_config.gpu_options.allow_growth = True sess = tf.Session(config=graph_config) graphHandler.initialize(sess) # begin training steps_per_epoch = int( math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size)) num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch global_step = 0 for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter( num_steps): global_step = sess.run(model.global_step) + 1 if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0 loss, summary, train_op = model.step(sess, sample_batch, get_summary=if_get_summary) if global_step % 100 == 0: _logger.add('data round: %d: %d/%d, global step:%d -- loss: %.4f' % (data_round, idx_b, batch_num, global_step, loss)) if if_get_summary: graphHandler.add_summary(summary, global_step) # Occasional evaluation if global_step % (cfg.eval_period or steps_per_epoch) == 0: # ---- dev ---- dev_loss, dev_accu = evaluator.get_evaluation( sess, dev_data_obj, global_step) _logger.add('==> for dev, loss: %.4f, accuracy: %.4f' % (dev_loss, dev_accu)) is_in_top, deleted_step = performRecoder.update_top_list( global_step, dev_accu, sess) this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round( data_round) # if this_epoch_time is not None and mean_epoch_time is not None: # _logger.add('##> this epoch time: %f, mean epoch time: %f' % (this_epoch_time, mean_epoch_time)) do_analyse_qc(_logger.path)
class SupervisedTrainer(object): """ The SupervisedTrainer class helps in setting up a training framework in a supervised setting. Args: expt_dir (optional, str): experiment Directory to store details of the experiment, by default it makes a folder in the current directory to store the details (default: `experiment`). """ def __init__(self, expt_dir='experiment'): self._trainer = "Simple Trainer" if not os.path.isabs(expt_dir): expt_dir = os.path.join(os.getcwd(), expt_dir) self.expt_dir = expt_dir if not os.path.exists(self.expt_dir): os.makedirs(self.expt_dir) def set_local_parameters(self, random_seed, losses, metrics, loss_weights, checkpoint_every, print_every): self.random_seed = random_seed if random_seed is not None: random.seed(random_seed) torch.manual_seed(random_seed) self.losses = losses self.metrics = metrics self.loss_weights = loss_weights or len(losses)*[1.] self.evaluator = Evaluator(loss=self.losses, metrics=self.metrics) self.optimizer = None self.checkpoint_every = checkpoint_every self.print_every = print_every self.logger = logging.getLogger(__name__) self._stop_training = False def _train_batch(self, ins_variable, src_variable, ins_len_variable, ins_position_variables, position_variables, tgt_variable): # Forward propagation predicted_softmax, attn = self.model(ins_variable, src_variable, ins_len_variable, ins_position_variables, position_variables) losses = self.evaluator.compute_batch_loss(predicted_softmax, tgt_variable) # Backward propagation for i, loss in enumerate(losses, 0): loss.scale_loss(self.loss_weights[i]) loss.backward(retain_graph=True) self.optimizer.step() self.model.zero_grad() return losses def _train_epoches(self, data, n_epochs, start_epoch, start_step, callbacks, dev_data, monitor_data=[]): steps_per_epoch = len(data) total_steps = steps_per_epoch * n_epochs callbacks.set_info(start_step, start_epoch, steps_per_epoch, total_steps) # set data as attribute to trainer self.train_data = data self.val_data = dev_data self.monitor_data = monitor_data # ######################################## # This is used to resume training from same place in dataset # after loading from checkpoint s = start_step if start_epoch > 1: s -= (start_epoch - 1) * steps_per_epoch ######################################## callbacks.on_train_begin() for epoch in range(start_epoch, n_epochs + 1): callbacks.on_epoch_begin(epoch) self.model.train() for batch in data: # Skip over the batches that are below start step if epoch == start_epoch and s > 0: s -= 1 continue callbacks.on_batch_begin(batch) ins_variables, src_variables, tgt_variable = self.get_batch_data(batch) ins_position_variables = self.get_position_ids(ins_variables[1]) position_variables = self.get_position_ids(src_variables[1]) self.batch_losses = self._train_batch(ins_variables[0], src_variables[0], ins_variables[1], ins_position_variables, position_variables, tgt_variable) callbacks.on_batch_end(batch) callbacks.on_epoch_end(epoch) # Stop training early if flag _stop_training is True if self._stop_training: break logs = callbacks.on_train_end() return logs def train(self, model, data, dev_data, num_epochs=5, resume_training=False, monitor_data={}, optimizer=None, custom_callbacks=[], learning_rate=0.001, checkpoint_path=None, top_k=5, losses=[NLLLoss()], loss_weights=None, metrics=[], random_seed=None, checkpoint_every=100, print_every=100): """ Run training for a given model. Args: model (machine.models): model to run training on, if `resume=True`, it would be overwritten by the model loaded from the latest checkpoint. data (torchtext.data.Iterator: torchtext iterator object to train on num_epochs (int, optional): number of epochs to run (default 5) resume_training(bool, optional): resume training with the latest checkpoint up until the number of epochs (default False) dev_data (torchtext.data.Iterator): dev/validation set iterator Note: must not pass in the train iterator here as this gets evaluated during training (in between batches) If you want to evaluate on the full train during training then make two iterators and pass the second one here monitor_data (list of torchtext.data.Iterator, optional): list of iterators to test on (default None) Note: must not pass in the train iterator here as this gets evaluated during training (in between batches) If you want to evaluate on the full train during training then make two iterators and pass the second one here optimizer (machine.optim.Optimizer, optional): optimizer for training (default: Optimizer(pytorch.optim.Adam, max_grad_norm=5)) custom_callbacks (list, optional): list of custom call backs (see utils.callbacks.callback for base class) learing_rate (float, optional): learning rate used by the optimizer (default 0.001) checkpoint_path (str, optional): path to load checkpoint from in case training should be resumed top_k (int): how many models should be stored during training loss (list, optional): list of machine.loss.Loss objects for training (default: [machine.loss.NLLLoss]) metrics (list, optional): list of machine.metric.metric objects to be computed during evaluation checkpoint_every (int, optional): number of epochs to checkpoint after, (default: 100) print_every (int, optional): number of iterations to print after, (default: 100) Returns: model (machine.models): trained model. """ self.set_local_parameters(random_seed, losses, metrics, loss_weights, checkpoint_every, print_every) # If training is set to resume if resume_training: resume_checkpoint = Checkpoint.load(checkpoint_path) model = resume_checkpoint.model self.model = model self.optimizer = resume_checkpoint.optimizer # A walk around to set optimizing parameters properly resume_optim = self.optimizer.optimizer defaults = resume_optim.param_groups[0] defaults.pop('params', None) defaults.pop('initial_lr', None) self.optimizer.optimizer = resume_optim.__class__( self.model.parameters(), **defaults) start_epoch = resume_checkpoint.epoch step = resume_checkpoint.step else: start_epoch = 1 step = 0 self.model = model def get_optim(optim_name): optims = {'adam': optim.Adam, 'adagrad': optim.Adagrad, 'adadelta': optim.Adadelta, 'adamax': optim.Adamax, 'rmsprop': optim.RMSprop, 'sgd': optim.SGD, None: optim.Adam} return optims[optim_name] self.optimizer = Optimizer(get_optim(optimizer)(self.model.parameters(), lr=learning_rate), max_grad_norm=5) self.logger.info("Optimizer: %s, Scheduler: %s" % (self.optimizer.optimizer, self.optimizer.scheduler)) callbacks = CallbackContainer(self, [Logger(), ModelCheckpoint(top_k=top_k), History()] + custom_callbacks) logs = self._train_epoches(data, num_epochs, start_epoch, step, dev_data=dev_data, monitor_data=monitor_data, callbacks=callbacks) return self.model, logs @staticmethod def get_batch_data(batch): ins_variables = getattr(batch, 'ins') src_variables = getattr(batch, 'src') tgt_variable = getattr(batch, 'tgt')[0] return ins_variables, src_variables, tgt_variable @staticmethod def get_position_ids(length_list): batch_size = len(length_list) length = max(length_list).item() pos_tensor = torch.zeros(batch_size, length).long() for i in range(batch_size): pos_tensor[i] = torch.LongTensor(range(0, length)) return Variable(pos_tensor)
def main(params): # initialize the multi-GPU / multi-node training init_distributed_mode(params) # initialize the experiment / load data logger = initialize_exp(params) # Seed torch.manual_seed(params.seed) torch.cuda.manual_seed_all(params.seed) # initialize SLURM signal handler for time limit / pre-emption if params.is_slurm_job: init_signal_handler() # data loaders / samplers populate_dataset(params) train_data_loader, train_sampler, _ = get_data_loader( img_size=params.img_size, crop_size=params.crop_size, shuffle=True, batch_size=params.batch_size, num_classes=params.num_classes, nb_workers=params.nb_workers, distributed_sampler=params.multi_gpu, dataset=params.dataset, data_path=params.train_path, transform=params.train_transform, split='valid' if params.debug_train else 'train', seed=params.seed) valid_data_loader, _, _ = get_data_loader(img_size=params.img_size, crop_size=params.crop_size, shuffle=False, batch_size=params.batch_size, num_classes=params.num_classes, nb_workers=params.nb_workers, distributed_sampler=False, dataset=params.dataset, transform='center', split='valid', seed=params.seed) # build model / cuda logger.info("Building %s model ..." % params.architecture) ftmodel = build_model(params) ftmodel.fc = nn.Sequential() ftmodel.eval().cuda() linearmodel = nn.Linear(EMBEDDING_SIZE[params.architecture], params.num_classes).cuda() if params.from_ckpt != "": ckpt = torch.load(params.from_ckpt) state_dict = { k.replace("module.", ""): v for k, v in ckpt['model'].items() } del state_dict["fc.weight"] if "fc.bias" in state_dict: del state_dict["fc.bias"] missing_keys, unexcepted_keys = ftmodel.load_state_dict(state_dict, strict=False) print("Missing keys: ", missing_keys) print("Unexcepted keys: ", unexcepted_keys) # distributed # TODO: check this https://github.com/NVIDIA/apex/blob/master/examples/imagenet/main.py#L142 if params.multi_gpu: logger.info("Using nn.parallel.DistributedDataParallel ...") linearmodel = nn.parallel.DistributedDataParallel( linearmodel, device_ids=[params.local_rank], output_device=params.local_rank, broadcast_buffers=True) # build trainer / reload potential checkpoints / build evaluator trainer = Trainer(model=linearmodel, params=params, ftmodel=ftmodel) trainer.reload_checkpoint() evaluator = Evaluator(trainer, params) # evaluation if params.eval_only: scores = evaluator.run_all_evals(trainer, evals=['classif'], data_loader=valid_data_loader) for k, v in scores.items(): logger.info('%s -> %.6f' % (k, v)) logger.info("__log__:%s" % json.dumps(scores)) exit() # training for epoch in range(trainer.epoch, params.epochs): # update epoch / sampler / learning rate trainer.epoch = epoch logger.info("============ Starting epoch %i ... ============" % trainer.epoch) if params.multi_gpu: train_sampler.set_epoch(epoch) # update learning rate trainer.update_learning_rate() # train for i, (images, targets) in enumerate(train_data_loader): trainer.classif_step(images, targets) trainer.iter() logger.info("============ End of epoch %i ============" % trainer.epoch) # evaluate classification accuracy scores = evaluator.run_all_evals(trainer, evals=['classif'], data_loader=valid_data_loader) for name, val in trainer.get_scores().items(): scores[name] = val # print / JSON log for k, v in scores.items(): logger.info('%s -> %.6f' % (k, v)) if params.is_master: logger.info("__log__:%s" % json.dumps(scores)) # end of epoch trainer.save_best_model(scores) trainer.save_periodic() trainer.end_epoch(scores)
def train(): output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: train_data_obj = Dataset(cfg.train_data_path, 'train') dev_data_obj = Dataset(cfg.dev_data_path, 'dev', dicts=train_data_obj.dicts) test_data_obj = Dataset(cfg.test_data_path, 'test', dicts=train_data_obj.dicts) save_file( { 'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj }, cfg.processed_path) train_data_obj.save_dict(cfg.dict_path) else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] train_data_obj.filter_data() dev_data_obj.filter_data() test_data_obj.filter_data() emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove with tf.variable_scope(cfg.base_name) as scope: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) performRecoder = PerformRecoder(cfg.save_num) if cfg.gpu_mem is None: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True) graph_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) elif cfg.gpu_mem < 1.: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem) graph_config = tf.ConfigProto(gpu_options=gpu_options) else: gpu_options = tf.GPUOptions() graph_config = tf.ConfigProto(gpu_options=gpu_options) sess = tf.Session(config=graph_config) graphHandler.initialize(sess) # begin training steps_per_epoch = int( math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size)) num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch global_step = 0 for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter( num_steps): global_step = sess.run(model.global_step) + 1 if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0 loss, summary = model.step(sess, sample_batch, get_summary=if_get_summary, global_step_value=global_step) if global_step % 100 == 0: _logger.add( 'data round: %d: %d/%d, global step:%d -- loss_sl: %.4f, loss_rl: %.4f' % (data_round, idx_b, batch_num, global_step, loss[0], loss[1])) if if_get_summary: graphHandler.add_summary(summary, global_step) # Occasional evaluation evaluation = False if cfg.model_dir_suffix == 'test': if global_step % (cfg.eval_period or steps_per_epoch) == 0: evaluation = True elif is_base_training: if global_step > cfg.num_steps - 350000 and ( global_step % (cfg.eval_period or steps_per_epoch) == 0): evaluation = True else: if global_step % (cfg.eval_period or steps_per_epoch) == 0: if cfg.load_model: evaluation = True else: if global_step > 250000: evaluation = True if evaluation: # ---- dev ---- dev_loss, dev_accu, dev_perc = evaluator.get_evaluation( sess, dev_data_obj, global_step) _logger.add( '==> for dev, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' % (dev_loss[0], dev_loss[1], dev_perc, dev_accu)) # ---- test ---- test_loss, test_accu, test_perc = evaluator.get_evaluation( sess, test_data_obj, global_step) _logger.add( '~~> for test, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' % (test_loss[0], test_loss[1], test_perc, test_accu)) is_in_top, deleted_step = performRecoder.update_top_list( global_step, dev_accu, sess) this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round( data_round) if this_epoch_time is not None and mean_epoch_time is not None: _logger.add('##> this epoch time: %f, mean epoch time: %f' % (this_epoch_time, mean_epoch_time)) if is_base_training and global_step >= 200000 and global_step % 50000 == 0 and cfg.save_model: graphHandler.save(sess, global_step) _logger.writeToFile() do_analyse_snli_rl(_logger.path)
def multi_test(): assert cfg.load_path is not None output_model_params() loadFile = True ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: train_data_obj = Dataset(cfg.train_data_path, 'train') dev_data_obj = Dataset(cfg.dev_data_path, 'dev', dicts=train_data_obj.dicts) test_data_obj = Dataset(cfg.test_data_path, 'test', dicts=train_data_obj.dicts) save_file( { 'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj }, cfg.processed_path) train_data_obj.save_dict(cfg.dict_path) else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] train_data_obj.filter_data() dev_data_obj.filter_data() test_data_obj.filter_data() emb_mat_token, emb_mat_glove = train_data_obj.emb_mat_token, train_data_obj.emb_mat_glove with tf.variable_scope(cfg.base_name) as scope: model = Model(emb_mat_token, emb_mat_glove, len(train_data_obj.dicts['token']), len(train_data_obj.dicts['char']), train_data_obj.max_lens['token'], scope.name) graphHandler = GraphHandler(model) evaluator = Evaluator(model) if cfg.gpu_mem is None: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True) graph_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) elif cfg.gpu_mem < 1.: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem) graph_config = tf.ConfigProto(gpu_options=gpu_options) else: gpu_options = tf.GPUOptions() graph_config = tf.ConfigProto(gpu_options=gpu_options) sess = tf.Session(config=graph_config) graphHandler.initialize(sess) repeat_num = 10 time_counter = TimeCounter() for t in range(repeat_num): # ---- dev ---- test_loss, test_accu, test_perc = evaluator.get_evaluation( sess, test_data_obj, None, time_counter=time_counter) _logger.add( '==> for test, loss: %.4f %.4f, perc: %.4f, accuracy: %.4f' % (test_loss[0], test_loss[1], test_perc, test_accu)) print(time_counter.update_data_round(t + 1))
def main(): parser = argparse.ArgumentParser() parser.add_argument("model_dir", help="Directory containing the trained model") parser.add_argument("test_dir", help="Directory containing test files") parser.add_argument("result_dir", help="Directory to write result files") parser.add_argument("--max_distance", default=15, type=int, help="Set the max entity distance to build data") parser.add_argument("--triad", action='store_true', default=False, help="use triads") parser.add_argument("--keras", action='store_true', default=False, help="Use keras model") parser.add_argument("--clustering_only", action='store_true', default=False, help="Use saved linkage files to perform clustering") parser.add_argument("--compute_linkage", action='store_true', default=False, help="compute linkage, for clustering_only option") args = parser.parse_args() with open(os.path.join(args.model_dir, 'word_indexes.pkl'), 'rb') as f: word_indexes = pickle.load(f) with open(os.path.join(args.model_dir, 'pos_tags.pkl'), 'rb') as f: pos_tags = pickle.load(f) df = build_dataFrame(args.test_dir, threads=1, suffix='auto_conll') if args.clustering_only: model = None elif args.keras: from keras.models import load_model model = load_model(os.path.join(args.model_dir, 'model.h5')) else: import torch model = torch.load(os.path.join(args.model_dir, 'model.pt')) model.eval() print("Loaded model") if args.triad: n_files = len(df.doc_id.unique()) test_gen = DataGen(df, word_indexes, pos_tags) test_input_gen = test_gen.generate_triad_input( looping=True, test_data=True, threads=4, max_distance=args.max_distance) evaluator = TriadEvaluator(model, test_input_gen) evaluator.write_results(df, args.result_dir, n_iterations=n_files, clustering_only=args.clustering_only, compute_linkage=args.compute_linkage) else: test_gen = DataGen(df, word_indexes, pos_tags) test_data_q = next( test_gen.generate_input(looping=False, test_data=True)) print("Loaded test data") evaluator = Evaluator(model, test_data_q) print("Performing fast evaluation...") print(evaluator.fast_eval()) print("Saving result files...") evaluator.write_results(df, args.result_dir) scorer(args.result_dir)
def train(): # output_model_params() # need to fixed, reusability of data loadFile = True ifLoad, data = False, None loaddict, dicts = load_file(cfg.dict_path, 'dict', 'pickle') if not loaddict: raise (ValueError, 'dict load failed') if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: train_data_obj = Dataset(cfg.train_data_path, 'train', dicts=dicts, language_type='es', unlabeled_file_path=cfg.unlabeled_data_path, emb_file_path=cfg.emb_es_path) dev_data_obj = Dataset(cfg.dev_data_path, 'dev', dicts=dicts) test_data_obj = Dataset(cfg.test_data_path, 'test', dicts=dicts) save_file( { 'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj }, cfg.processed_path) # train_data_obj.save_dict(cfg.dict_path) else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] emb_mat_token = train_data_obj.emb_mat_token with tf.variable_scope(network_type) as scope: if network_type in model_type_set: model = Model(emb_mat_token, len(train_data_obj.dicts['es']), 100, scope.name) # TODO graphHandler = GraphHandler(model) evaluator = Evaluator(model) performRecorder = PerformRecorder(5) if cfg.gpu_mem < 1: gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=cfg.gpu_mem, allow_growth=True) else: gpu_options = tf.GPUOptions() graph_config = tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True) sess = tf.Session(config=graph_config) graphHandler.initialize(sess) # load model if cfg.load_model and cfg.load_step: saver = tf.train.Saver() step = cfg.load_step model_path = os.path.join(cfg.ckpt_dir, 'top_result_saver_step_%d.ckpt' % step) saver.restore(sess, model_path) # begin training steps_per_epoch = int( math.ceil(1.0 * train_data_obj.sample_num / cfg.train_batch_size)) num_steps = cfg.num_steps or steps_per_epoch * cfg.max_epoch global_step = 0 for sample_batch, batch_num, data_round, idx_b in train_data_obj.generate_batch_sample_iter( num_steps): global_step = sess.run(model.global_step) + 1 if_get_summary = global_step % (cfg.log_period or steps_per_epoch) == 0 loss, summary, train_op = model.step(sess, sample_batch, get_summary=if_get_summary) if global_step % 100 == 0 or global_step == 1: _logger.add('data round: %d: %d/%d, global_step:%d -- loss:%.4f' % (data_round, idx_b, batch_num, global_step, loss)) if if_get_summary: graphHandler.add_summary(summary, global_step) # Occasional evaluation #if global_step > int(cfg.num_steps - 100000) and (global_step % (cfg.eval_period or steps_per_epoch) == 0): if global_step % (cfg.eval_period or steps_per_epoch) == 0: # debug # ---- dev ---- dev_loss, dev_accu, dev_accu_0, dev_accu_1 = evaluator.get_evaluation( sess, dev_data_obj, global_step) _logger.add( '==> for dev, loss: %.4f, accuracy: %.4f, accuracy_0: %.4f, accuracy_1: %.4f' % (dev_loss, dev_accu, dev_accu_0, dev_accu_1)) # ---- test ---- test_loss, test_accu, test_accu_0, test_accu_1 = evaluator.get_evaluation( sess, test_data_obj, global_step) _logger.add( '~~> for test, loss: %.4f, accuracy: %.4f, accuracy_0: %.4f, accuracy_1: %.4f' % (test_loss, test_accu, test_accu_0, test_accu_1)) if global_step > cfg.update_lr_step: model.update_learning_rate(dev_loss, cfg.lr_decay) if global_step > cfg.record_model_step: is_in_top, deleted_step = performRecorder.update_top_list( global_step, test_accu, sess) this_epoch_time, mean_epoch_time = cfg.time_counter.update_data_round( data_round) if this_epoch_time is not None and mean_epoch_time is not None: _logger.add('##> this epoch time: %f, mean epoch time: %f' % (this_epoch_time, mean_epoch_time))
def test(): assert cfg.load_path is not None #TODO loadFile = False ifLoad, data = False, None if loadFile: ifLoad, data = load_file(cfg.processed_path, 'processed data', 'pickle') if not ifLoad or not loadFile: train_data_obj = Dataset(cfg.train_data_path, 'train', language_type='es', unlabeled_file_path=cfg.unlabeled_data_path, emb_file_path=cfg.emb_es_path) dev_data_obj = Dataset(cfg.dev_data_path, 'dev', dicts=train_data_obj.dicts) test_data_obj = Dataset(cfg.test_data_path, 'test', dicts=train_data_obj.dicts) save_file( { 'train_data_obj': train_data_obj, 'dev_data_obj': dev_data_obj, 'test_data_obj': test_data_obj }, cfg.processed_path) train_data_obj.save_dict(cfg.dict_path) else: train_data_obj = data['train_data_obj'] dev_data_obj = data['dev_data_obj'] test_data_obj = data['test_data_obj'] emb_mat_token = train_data_obj.emb_mat_token with tf.variable_scope(network_type) as scope: if network_type in model_type_set: model = Model(emb_mat_token, len(train_data_obj.dicts['es']), 100, scope.name) # TODO graphHandler = GraphHandler(model) evaluator = Evaluator(model) graph_config = tf.ConfigProto() sess = tf.Session(config=graph_config) graphHandler.initialize(sess) # ---- dev ---- dev_loss, dev_accu = evaluator.get_evaluation(sess, dev_data_obj, None) _logger.add('==> for dev, loss: %.4f, accuracy: %.4f' % (dev_loss, dev_accu)) # ---- test ---- test_loss, test_accu = evaluator.get_evaluation(sess, test_data_obj, None) _logger.add('~~> for test, loss: %.4f, accuracy: %.4f' % (test_loss, test_accu)) # ---- train ---- train_loss, train_accu = evaluator.get_evaluation(sess, train_data_obj, None) _logger.add('--> for train, loss: %.4f, accuracy: %.4f' % (train_loss, train_accu))
def main(params): # initialize the multi-GPU / multi-node training # initialize experiment / SLURM signal handler for time limit / pre-emption init_distributed_mode(params) logger = initialize_exp(params) init_signal_handler() # CPU / CUDA if params.cpu: assert not params.multi_gpu else: assert torch.cuda.is_available() src.utils.CUDA = not params.cpu # build environment / modules / trainer / evaluator env = build_env(params) modules = build_modules(env, params) trainer = Trainer(modules, env, params) evaluator = Evaluator(trainer) # evaluation if params.eval_only: scores = evaluator.run_all_evals() for k, v in scores.items(): logger.info("%s -> %.6f" % (k, v)) logger.info("__log__:%s" % json.dumps(scores)) exit() # training for _ in range(params.max_epoch): logger.info("============ Starting epoch %i ... ============" % trainer.epoch) trainer.n_equations = 0 while trainer.n_equations < trainer.epoch_size: # training steps for task_id in np.random.permutation(len(params.tasks)): task = params.tasks[task_id] if params.export_data: trainer.export_data(task) else: trainer.enc_dec_step(task) trainer.iter() logger.info("============ End of epoch %i ============" % trainer.epoch) # evaluate perplexity scores = evaluator.run_all_evals() # print / JSON log for k, v in scores.items(): logger.info("%s -> %.6f" % (k, v)) if params.is_master: logger.info("__log__:%s" % json.dumps(scores)) # end of epoch trainer.save_best_model(scores) trainer.save_periodic() trainer.end_epoch(scores)
def test_eval(self): evaluator = Evaluator({}) env = Environment() self.exp1.eval(evaluator, env) self.assertEqual(env.get_variable("x"), 65)