def docs(dataset_name): p = utils.Progbar(target=(utils.lines_in_file(directories.RAW + dataset_name))) for i, d in enumerate(utils.load_json_lines(directories.RAW + dataset_name)): p.update(i + 1) yield d
def train(model, training_data, optimizer): model.train() num_batches = math.ceil(len(training_data) / args.batch_size) bar = utils.Progbar(target=num_batches) train_loss = 0.0 train_total_instances = 0 for batch_id, batch in enumerate( utils.minibatches(training_data, args.batch_size)): model.zero_grad() for sentence, tags in batch: sentence_in = processor.tensor(sentence) targets = processor.tensor(tags) loss = model.neg_log_likelihood(sentence_in, targets) loss.backward() train_loss += loss train_total_instances += 1 optimizer.step() bar.update(batch_id + 1, exact=[("train loss", train_loss / train_total_instances)]) if args.save_checkpoint: save_model(model)
def run(model, sess, dataset, train=False): j = 0 prog = utils.Progbar(len(dataset)) avg_loss = 0.0 avg_correct = 0.0 for x, masks, y in dataset: input_feed = { model.inputs: utils.convert_to_np(x), model.masks: utils.convert_to_np(masks), model.labels: utils.convert_to_np(y), model.train: train } if train: output_feed = [ model.cost, model.accuracy, model.average_guess, model.op ] cost, num_correct, avg_guess, _ = sess.run(output_feed, input_feed) else: output_feed = [model.cost, model.accuracy, model.average_guess] cost, num_correct, avg_guess = sess.run(output_feed, input_feed) avg_loss += cost avg_correct += num_correct j += 1 prog.update(j, values=[('num_correct', num_correct)], exact=[('cost', cost)]) avg_loss /= len(dataset) avg_correct /= len(dataset) return avg_correct, avg_loss
def evaluate(model, eval_data, dataset_name): model.eval() num_batches = math.ceil(len(eval_data) / args.batch_size) bar = utils.Progbar(target=num_batches) eval_score = 0.0 eval_total_instances = 0 eval_total_characters = 0 eval_correct_characters = 0 with torch.no_grad(): for batch_id, batch in enumerate( utils.minibatches(eval_data, args.batch_size)): for sentence, tags in batch: score, tag_out = model(processor.tensor(sentence)) if len(tag_out) != len(tags): raise IndexError( 'Size of output tag sequence differs from that of reference.' ) length = len(tags) correct = [ tag_out[i] == tags[i] for i in range(1, length - 1) ].count(1) eval_score += score eval_total_instances += 1 eval_total_characters += length eval_correct_characters += correct bar.update(batch_id + 1, exact=[("eval score", eval_score / eval_total_instances) ]) logger.info('{} dataset accuracy: {}'.format( dataset_name, eval_correct_characters / eval_total_characters))
def run_model_over_docs(dataset_name, docs, model): docs_by_id = {doc.did: doc for doc in docs} prog = utils.Progbar(dataset_name.n_batches) for i, X in enumerate(dataset_name): if X['y'].size == 0: continue scores = model.predict_on_batch(X) update_doc(docs_by_id[X['did']], X, scores) prog.update(i + 1)
def build_dataset(vectors, name, tune_fraction=0.0, reduced=False, columns=None): doc_vectors = utils.load_pickle(directories.MISC + name.replace("_reduced", "") + "_document_vectors.pkl") main_pairs = PairDataBuilder(columns) tune_pairs = PairDataBuilder(columns) main_mentions = MentionDataBuilder(columns) tune_mentions = MentionDataBuilder(columns) main_docs = DocumentDataBuilder(columns) tune_docs = DocumentDataBuilder(columns) print("Building dataset", name + ("/tune" if tune_fraction > 0 else "")) p = utils.Progbar(target=(2 if reduced else utils.lines_in_file(directories.RAW + name))) for i, d in enumerate(utils.load_json_lines(directories.RAW + name)): if reduced and i > 2: break p.update(i + 1) if reduced and tune_fraction != 0: pairs, mentions, docs = (main_pairs, main_mentions, main_docs) \ if i == 0 else (tune_pairs, tune_mentions, tune_docs) else: pairs, mentions, docs = (main_pairs, main_mentions, main_docs) \ if random.random() > tune_fraction else (tune_pairs, tune_mentions, tune_docs) ms, ps = mentions.size(), pairs.size() mention_positions = {} for mention_num in sorted(d["mentions"].keys(), key=int): mention_positions[mention_num] = mentions.size() mentions.add_mention(d["mentions"][mention_num], vectors, doc_vectors[d["mentions"][mention_num]["doc_id"]]) for key in sorted(d["labels"].keys(), key=lambda k: (int(k.split()[1]), int(k.split()[0]))): k1, k2 = key.split() pairs.add_pair(d["labels"][key], mention_positions[k1], mention_positions[k2], int(d["mentions"][k1]["doc_id"]), int(d["mentions"][k1]["mention_id"]), int(d["mentions"][k2]["mention_id"]), d["pair_features"][key]) me, pe = mentions.size(), pairs.size() docs.add_doc(ms, me, ps, pe, d["document_features"]) suffix = ("_reduced" if reduced else "") if tune_mentions.size() > 0: tune_mentions.write(name + "_tune" + suffix) tune_pairs.write(name + "_tune" + suffix) tune_docs.write(name + "_tune" + suffix) main_mentions.write(name + "_train" + suffix) main_pairs.write(name + "_train" + suffix) main_docs.write(name + "_train" + suffix) else: main_mentions.write(name + suffix) main_pairs.write(name + suffix) main_docs.write(name + suffix)
def __init__(self, trainer, docs, data, message, replay_memory=None, beta=0, docs_per_iteration=10000): self.trainer = trainer self.data = data self.model = trainer.model self.message = message self.replay_memory = replay_memory self.beta = beta self.loss_aggregator = Aggregator() self.evaluators = [ evaluation.Evaluator(metric=evaluation.muc), evaluation.Evaluator(metric=evaluation.b_cubed), evaluation.Evaluator(metric=evaluation.ceafe) ] self.merged_pairs = {} self.training = self.replay_memory is not None print self.message random.shuffle(docs) if self.training: docs = docs[:docs_per_iteration] prog = utils.Progbar(len(docs)) for i, (doc, actionstate) in enumerate(docs): self.trainer.doc = doc self.trainer.actionstate = actionstate if len(actionstate.possible_pairs) != 0: actionstate.load(self.data, self.trainer.pair_model, self.trainer.anaphoricity_model) s = State(doc, actionstate) doc_merged_pairs = self.run_agent(s, beta, i) for evaluator in self.evaluators: evaluator.update(doc) self.merged_pairs[doc.did] = doc_merged_pairs doc.reset() actionstate.clear() muc, b3, ceafe = (self.evaluators[i].get_f1() for i in range(3)) exact = [('muc', 100 * muc), ('b3', 100 * b3), ('ceafe', 100 * ceafe), ('conll', 100 * (muc + b3 + ceafe) / 3), ('loss', self.loss_aggregator.get_avg())] prog.update(i + 1, exact=exact)
def train_all(self): timer.start("train") model_weights = self.model.get_weights() prog = utils.Progbar(len(self.memory)) random.shuffle(self.memory) for i, X in enumerate(self.memory): loss = self.train_on_example(X) prog.update(i + 1, [("loss", loss)]) self.size = 0 self.memory = [] timer.stop("train") weight_diffs = [(np.sum(np.abs(new_weight - old_weight)), new_weight.size) for new_weight, old_weight in zip( self.model.get_weights(), model_weights)] summed = np.sum(map(np.array, weight_diffs), axis=0) print "weight diffs", weight_diffs, summed
def set_costs(dataset, docs): docs_by_id = {doc.did: doc for doc in docs} prog = utils.Progbar(dataset.n_batches) for i, X in enumerate(dataset): if X['y'].size == 0: continue doc = docs_by_id[X['did']] doc_weight = (len(doc.mention_to_gold) + len(doc.mentions)) / 10.0 for (start, end) in zip(X['starts'][:, 0], X['ends'][:, 0]): ids = X['ids'][start:end] ana = ids[0, 1] old_ant = doc.ana_to_ant[ana] doc.unlink(ana) costs = X['cost_ptrs'][start:end] for ant_ind in range(end - start): costs[ant_ind] = doc.link(ids[ant_ind, 0], ana, hypothetical=True, beta=1) doc.link(old_ant, ana) costs -= costs.max() costs *= -doc_weight prog.update(i + 1)
def train(self): ''' train the model ''' def num_steps(bucket): return int(np.ceil(len(bucket) / float(config.BATCH_SIZE))) saver = tf.train.Saver() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) self._check_restore_parameters(sess, saver) # get checkpoint save state iteration = self.global_step.eval() # get number of batches per bucket bucket_sizes = [ num_steps(v['dec_input']) for k, v in self.train_data.iteritems() ] cur_epoch = self.epoch.eval() bucket_index = self.bucket_index.eval() step_iter = self.bucket_step.eval() print 'Starting at iter:', iteration, 'epoch:', cur_epoch+1, \ 'bucket index:', bucket_index, 'bucket step:', step_iter for epoch in range(cur_epoch, config.NUM_EPOCHS): epoch_start = time.time() total_losses = [] sess.run(tf.assign(self.epoch, epoch)) print '\n', 'Epoch:', epoch + 1 print 'Bucket sizes', bucket_sizes # print 'Learning rate:', self.lr # show progress bar if more than one batch if sum(bucket_sizes) > 1: prog = utils.Progbar(target=bucket_sizes[bucket_index]) end_while = False while True: # get data for iteration batch_start = time.time() batch_data = data.get_batch(self.train_data, bucket_index, config.BUCKETS, config.BATCH_SIZE, step_iter) encoder_inputs = batch_data[0] decoder_inputs = batch_data[1] decoder_masks = batch_data[2] next_bucket = batch_data[3] # run the model _, step_loss, _ = self.run_step(sess, encoder_inputs, decoder_inputs, decoder_masks, config.BATCH_SIZE, bucket_index, True) # reset iteration vars if done with the current bucket if next_bucket: step_iter = sess.run(tf.assign(self.bucket_step, 0)) bucket_index = sess.run( tf.assign(self.bucket_index, bucket_index + 1)) else: step_iter = sess.run( tf.assign(self.bucket_step, step_iter + 1)) if bucket_index >= len(config.BUCKETS): end_while = True bucket_index = sess.run(tf.assign( self.bucket_index, 0)) if next_bucket and sum(bucket_sizes) > 1: print # update progress prog.update( bucket_sizes[bucket_index - 1], [("train loss", step_loss), ('batch runtime', time.time() - batch_start)]) new_bucket = bucket_sizes[bucket_index] prog = utils.Progbar(target=new_bucket) total_losses.append(step_loss) # when to evaluate and save state if end_while or \ iteration == 200 or \ (iteration > 0 and iteration % 1000 == 0): saver.save(sess, os.path.join(self.checkpoint_path, 'summarizer'), global_step=iteration) if iteration == 200 or end_while or epoch < 2 \ or iteration % 2000 == 0: self.evaluate(sess, total_losses, iteration) iteration += 1 if sum(bucket_sizes) == 1: print 'Train loss', step_loss if end_while: print 'Epoch', epoch + 1, 'took', time.time( ) - epoch_start break if sum(bucket_sizes) > 1: prog.update( step_iter, [("train loss", step_loss), ('batch runtime', time.time() - batch_start)]) # eval on test set at the end self.evaluate(sess, total_losses, iteration, test=True) saver.save(sess, os.path.join(self.checkpoint_path, 'summarizer'), global_step=iteration)
logger.info("Number dev instances: {}".format(len(dev_instances))) training_total_tokens = 0 best_f1 = 0. for epoch in range(int(options.num_epochs)): logger.info("Epoch {} out of {}".format(epoch + 1, options.num_epochs)) random.shuffle(training_instances) train_loss = 0.0 train_total_instance = 0 # size of trained instances if options.dropout > 0: model.set_dropout(options.dropout) nbatches = (len(training_instances) + options.batch_size - 1) // options.batch_size bar = utils.Progbar(target=nbatches) for batch_id, batch in enumerate( utils.minibatches(training_instances, options.batch_size)): for idx, instance in enumerate(batch): if len(instance.sentence) == 0: continue train_total_instance += 1 loss_expr = model.neg_log_loss(instance.sentence, instance.tags) # Forward pass loss = loss_expr.scalar_value() # Do backward pass loss_expr.backward() # Bail if loss is NaN if math.isnan(loss):
def main(args): if not os.path.exists(args.checkpoint): os.makedirs(args.checkpoint) c = utils.make_config(args) d = input_pipeline.DataInputPipeline(args.inputs, args.vocab, args.labels, c) os.environ[ 'CUDA_VISIBLE_DEVICES'] = args.gpu # Or whichever device you would like to use # gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session( ) #config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)) m = models.Model(c, sess, d) sess.run(tf.global_variables_initializer()) if args.train: print 'INFO: starting training...' prog = utils.Progbar(target=d.get_num_batches()) epoch = 1 for _ in range(c.num_epochs): epoch_loss = 0 for i, batch in enumerate(d.batch_iter()): sales_hat, price_hat, shop_hat, category_hat, loss = \ m.train_on_batch(*batch[:-1]) prog.update(i + 1, [('train loss', loss)]) epoch_loss += loss print '\n INFO: EPOCH ', epoch, ' MEAN LOSS: ', epoch_loss / float( d.get_num_batches()) print 'INFO: saving checkpoint...' m.save(args.checkpoint) print 'INFO: ...done!' epoch += 1 else: print 'INFO: loading model from checkpoint...' m.load(dir=args.checkpoint) print 'INFO: done!' if args.output is not None: print 'INFO: starting inference...' prog = utils.Progbar(target=d.get_num_batches()) source_out = {'source': [], 'len': [], 'attn': [], 'ids': []} sales_out = {'label': [], 'pred': []} price_out = {'label': [], 'pred': []} shop_out = {'label': [], 'pred': []} category_out = {'label': [], 'pred': []} loss_out = [] for i, batch in enumerate(d.batch_iter()): sales_hat, price_hat, shop_hat, category_hat, loss, attn = \ m.test_on_batch(*batch[:-1]) prog.update(i, [('train loss', loss)]) # record results source, source_len, log_sales, price, shop, category, ids = batch source_out['source'] += source source_out['len'] += source_len source_out['attn'] += attn.tolist() source_out['ids'] += ids sales_out['label'] += log_sales sales_out['pred'] += sales_hat.tolist() price_out['label'] += price price_out['pred'] += price_hat.tolist() shop_out['label'] += shop shop_out['pred'] += shop_hat.tolist() category_out['label'] += category category_out['pred'] += category_hat.tolist() loss_out += loss print '\nINFO: dumping output to ', args.output utils.write_pickle(source_out, args.output) print '\n\nINFO: \(^o^)/ done \(^o^)/'
def train_model(dataset, paths, device): """The main function for executing network training. It loads the specified dataset iterator, saliency model, and helper classes. Training is then performed in a new session by iterating over all batches for a number of epochs. After validation on an independent set, the model is saved and the training history is updated. Args: dataset (str): Denotes the dataset to be used during training. paths (dict, str): A dictionary with all path elements. device (str): Represents either "cpu" or "gpu". """ iterator = data.get_dataset_iterator("train", dataset, paths["data"]) next_element, train_init_op, valid_init_op = iterator input_images, ground_truths = next_element[:2] input_plhd = tf.placeholder_with_default(input_images, (None, None, None, 3), name="input") #training = tf.placeholder(tf.bool, name="training") ## For BN msi_net = model_bn.MSINET(is_train=True) predicted_maps = msi_net.forward(input_plhd) optimizer, loss = msi_net.train(ground_truths, predicted_maps, config.PARAMS["learning_rate"]) n_train_data = getattr(data, dataset.upper()).n_train n_valid_data = getattr(data, dataset.upper()).n_valid n_train_batches = int(np.ceil(n_train_data / config.PARAMS["batch_size"])) n_valid_batches = int(np.ceil(n_valid_data / config.PARAMS["batch_size"])) history = utils.History(n_train_batches, n_valid_batches, dataset, paths["history"], device) progbar = utils.Progbar(n_train_data, n_train_batches, config.PARAMS["batch_size"], config.PARAMS["n_epochs"], history.prior_epochs) #training = tf.placeholder(tf.bool, name="training") ## For BN with tf.Session() as sess: sess.run(tf.global_variables_initializer()) saver = msi_net.restore(sess, dataset, paths, device) print(">> Start training on %s..." % dataset.upper()) for epoch in range(config.PARAMS["n_epochs"]): sess.run(train_init_op) for batch in range(n_train_batches): #_, error = sess.run([optimizer, loss], feed_dict={training: True}) _, error = sess.run([optimizer, loss]) history.update_train_step(error) progbar.update_train_step(batch) sess.run(valid_init_op) for batch in range(n_valid_batches): #error = sess.run(loss, feed_dict={training: False}) error = sess.run(loss) history.update_valid_step(error) progbar.update_valid_step() msi_net.save(saver, sess, dataset, paths["latest"], device) history.save_history() progbar.write_summary(history.get_mean_train_error(), history.get_mean_valid_error()) if history.valid_history[-1] == min(history.valid_history): msi_net.save(saver, sess, dataset, paths["best"], device) msi_net.optimize(sess, dataset, paths["best"], device) print("\tBest model!", flush=True)
def main(args): c = utils.make_config(args) d = input_pipeline.DataInputPipeline(args.inputs, args.vocab, args.labels, c) os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu # gpu_options = tf.GPUOptions(allow_growth=True) sess = tf.Session( ) #config=tf.ConfigProto(gpu_options=gpu_options, allow_soft_placement=True)) m = models.Model(c, sess, d) print 'INFO: loading model from checkpoint...' m.load(dir=args.checkpoint) print 'INFO: done!' sess.run(tf.global_variables_initializer()) print 'INFO: starting inference...' prog = utils.Progbar(target=d.get_num_batches()) source_out = {'source': [], 'len': [], 'attn': [], 'ids': []} sales_out = {'label': [], 'pred': []} price_out = {'label': [], 'pred': []} shop_out = {'label': [], 'pred': []} category_out = {'label': [], 'pred': []} loss_out = [] for i, batch in enumerate(d.batch_iter()): sales_hat, price_hat, shop_hat, category_hat, loss, attn = \ m.test_on_batch(*batch[:-1]) prog.update(i, [('train loss', loss)]) # record results source, source_len, log_sales, price, shop, category, ids = batch source_out['source'] += source source_out['len'] += source_len source_out['attn'] += attn.tolist() source_out['ids'] += ids sales_out['label'] += log_sales sales_out['pred'] += sales_hat.tolist() price_out['label'] += price price_out['pred'] += price_hat.tolist() shop_out['label'] += shop shop_out['pred'] += shop_hat.tolist() category_out['label'] += category category_out['pred'] += category_hat.tolist() loss_out += loss if args.output is not None: print '\nINFO: dumping output to ', args.output write_pickle(source_out, args.output) print 'INFO: done \(^o^)/'
def __init__(self, list_words, max_distance=3): self.max_distance = max_distance self.dictionary = {} self.max_length = 0 self.progbar = utils.Progbar(len(list_words)) self.build(list_words)
def train(model_props, n_epochs=10000, reduced=False, dev_set_name='dev'): print "Training", model_props.path pprint(model_props.__dict__) model_props.write(model_props.path + 'model_props.pkl') utils.rmkdir(model_props.path + 'src') for fname in os.listdir('.'): if fname.endswith('.py'): shutil.copyfile(fname, model_props.path + 'src/' + fname) if model_props.ranking or \ model_props.top_pairs: write_start = 0 write_every = 10 else: write_start = 80 write_every = 20 print "Loading data" vectors = np.load(directories.RELEVANT_VECTORS + 'word_vectors.npy') train = datasets.DocumentBatchedDataset( "train_reduced" if reduced else "train", model_props, with_ids=True) dev = datasets.DocumentBatchedDataset( dev_set_name + "_reduced" if reduced else dev_set_name, model_props, with_ids=True) print "Building model" model, _ = pairwise_models.get_model(dev, vectors, model_props) json_string = model.to_json() open(model_props.path + 'architecture.json', 'w').write(json_string) best_val_score = 1000 best_val_score_in_window = 1000 history = [] for epoch in range(n_epochs): timer.start("train") print "EPOCH {:}, model = {:}".format((epoch + 1), model_props.path) epoch_stats = {} model_weights = model.get_weights() train_docs = utils.load_pickle(directories.DOCUMENTS + 'train_docs.pkl') dev_docs = utils.load_pickle(directories.DOCUMENTS + dev_set_name + '_docs.pkl') if reduced: dev_docs = dev_docs[:3] if model_props.ranking: print "Running over training set" run_model_over_docs(train, train_docs, model) epoch_stats.update(compute_metrics(train_docs, "train")) if model_props.use_rewards: print "Setting costs" set_costs(train, train_docs) print "Training" prog = utils.Progbar(train.n_batches) train.shuffle() loss_sum, n_examples = 0, 0 for i, X in enumerate(train): if X['y'].size == 0: continue batch_loss = model.train_on_batch(X) loss_sum += batch_loss * train.scale_factor n_examples += X['y'].size prog.update(i + 1, exact=[("train loss", loss_sum / n_examples)]) epoch_stats["train time"] = time.time() - prog.start for k in prog.unique_values: epoch_stats[k] = prog.sum_values[k][0] / max( 1, prog.sum_values[k][1]) epoch_stats["weight diffs"] = [ (np.sum(np.abs(new_weight - old_weight)), new_weight.size) for new_weight, old_weight in zip(model.get_weights(), model_weights) ] summed = np.sum(map(np.array, epoch_stats["weight diffs"][1:]), axis=0) epoch_stats["total weight diff"] = tuple(summed) print "Testing on dev set" evaluate_model(dev, dev_docs, model, model_props, epoch_stats) history.append(epoch_stats) utils.write_pickle(history, model_props.path + 'history.pkl') score = -epoch_stats["dev conll"] if model_props.ranking else \ (epoch_stats["dev loss"] if not model_props.anaphoricity_only else epoch_stats["dev anaphoricity loss"]) if score < best_val_score: best_val_score = score print "New best {:}, saving model".format( "CoNLL F1" if model_props.ranking else "validation loss") model.save_weights(model_props.path + "best_weights.hdf5", overwrite=True) if score < best_val_score_in_window and epoch > write_start: print "Best in last {:}, saved to weights_{:}".format( write_every, write_every * (epoch / write_every)) best_val_score_in_window = score model.save_weights( model_props.path + "weights_{:}.hdf5".format(write_every * (epoch / write_every)), overwrite=True) if epoch + write_every >= n_epochs: model.save_weights(model_props.path + "final_weights.hdf5", overwrite=True) if epoch % write_every == 0: best_val_score_in_window = 1000 timer.stop("train") timer.print_totals() print timer.clear()
def evaluate_model(dataset, docs, model, model_props, stats, save_output=False, save_scores=False, print_table=False): prog = utils.Progbar(dataset.n_batches) mt = RankingMetricsTracker(dataset.name, model_props=model_props) \ if model_props.ranking else ClassificationMetricsTracker(dataset.name) mta = ClassificationMetricsTracker(dataset.name + " anaphoricity", anaphoricity=True) docs_by_id = {doc.did: doc for doc in docs} if model_props.ranking else {} saved_links, saved_scores = (defaultdict(list) if save_output else None, defaultdict(dict) if save_scores else None) for i, X in enumerate(dataset): if X['y'].size == 0: continue progress = [] scores = model.predict_on_batch(X) if model_props.ranking: update_doc(docs_by_id[X['did']], X, scores, saved_links=saved_links, saved_scores=saved_scores) if model_props.anaphoricity and not model_props.ranking: progress.append( ("anaphoricity loss", mta.update(X, scores[0][:, 0]))) if not model_props.anaphoricity_only: progress.append( ("loss", mt.update( X, scores if model_props.ranking else scores[1 if model_props.anaphoricity else 0][:, 0]))) prog.update(i + 1, exact=progress) if save_scores: print "Writing scores" utils.write_pickle(saved_scores, model_props.path + dataset.name + '_scores.pkl') if save_output: print "Writing output" utils.write_pickle(saved_links, model_props.path + dataset.name + '_links.pkl') utils.write_pickle( docs, model_props.path + dataset.name + '_processed_docs.pkl') timer.start("metrics") if model_props.ranking: stats.update(compute_metrics(docs, dataset.name)) stats["validate time"] = time.time() - prog.start if model_props.anaphoricity and not model_props.ranking: mta.finish(stats) if not model_props.anaphoricity_only: mt.finish(stats) timer.stop("metrics") if print_table: print " & ".join( map(lambda x: "{:.2f}".format(x * 100), [ stats[dataset.name + " muc precision"], stats[dataset.name + " muc recall"], stats[dataset.name + " muc"], stats[dataset.name + " b3 precision"], stats[dataset.name + " b3 recall"], stats[dataset.name + " b3"], stats[dataset.name + " ceafe precision"], stats[dataset.name + " ceafe recall"], stats[dataset.name + " ceafe"], stats[dataset.name + " conll"], ]))