Python count_lines示例

示例#1

文件： training.py 项目： luofuli/Hackathon

def load_vocab(vocab_file):
  """Returns a lookup table and the vocabulary size."""
  vocab_size = count_lines(vocab_file) + 1  # Add UNK.
  vocab = tf.contrib.lookup.index_table_from_file(
      vocab_file,
      vocab_size=vocab_size - 1,
      num_oov_buckets=1)
  return vocab, vocab_size

示例#2

文件： sequence_classifier.py 项目： yhgon/OpenNMT-tf

 def _initialize(self, metadata):
   self.inputter.initialize(metadata)
   self.labels_vocabulary_file = metadata[self.labels_vocabulary_file_key]
   self.num_labels = count_lines(self.labels_vocabulary_file)

示例#3

文件： inference.py 项目： luofuli/Hackathon

      "length": tf.shape(x)[0]})
  dataset = dataset.padded_batch(64, {
      "ids": [None],
      "length": []})
  return dataset.make_initializable_iterator()

if args.direction == 1:
  src_file, tgt_file = args.src, args.tgt
  src_vocab_file, tgt_vocab_file = args.src_vocab, args.tgt_vocab
else:
  src_file, tgt_file = args.tgt, args.src
  src_vocab_file, tgt_vocab_file = args.tgt_vocab, args.src_vocab

from opennmt.utils.misc import count_lines

tgt_vocab_size = count_lines(tgt_vocab_file) + 1
src_vocab_size = count_lines(src_vocab_file) + 1
src_vocab = tf.contrib.lookup.index_table_from_file(
    src_vocab_file,
    vocab_size=src_vocab_size - 1,
    num_oov_buckets=1)

with tf.device("cpu:0"):
  src_iterator = load_data(src_file, src_vocab)

src = src_iterator.get_next()


# Step 2

示例#4

 def _initialize(self, metadata):
   self.inputter.initialize(metadata)
   self.labels_vocabulary_file = metadata[self.labels_vocabulary_file_key]
   self.num_labels = count_lines(self.labels_vocabulary_file)

示例#5

文件： transformerinference.py 项目： AnonymousAuthor2013/PostRec

    dataset = dataset.map(input_vocab.lookup)
    dataset = dataset.map(lambda x: {"ids": x, "length": tf.shape(x)[0]})
    dataset = dataset.padded_batch(64, {"ids": [None], "length": []})
    return dataset.make_initializable_iterator()


if args.direction == 1:
    src_file, tgt_file = args.src, args.tgt
    src_vocab_file, tgt_vocab_file = args.src_vocab, args.tgt_vocab
else:
    src_file, tgt_file = args.tgt, args.src
    src_vocab_file, tgt_vocab_file = args.tgt_vocab, args.src_vocab

from opennmt.utils.misc import count_lines

tgt_vocab_size = count_lines(tgt_vocab_file) + 1
src_vocab_size = count_lines(src_vocab_file) + 1
from tensorflow.contrib import lookup
src_vocab = lookup.index_table_from_file(src_vocab_file,
                                         vocab_size=src_vocab_size - 1,
                                         num_oov_buckets=1)

with tf.device("cpu:0"):
    src_iterator = load_data(src_file, src_vocab)

src = src_iterator.get_next()

# Step 2

hidden_size = 768
from bert.modeling import BertModel, BertConfig

示例#6

文件： text_inputter.py 项目： zlpmichelle/OpenNMT-tf

 def get_dataset_size(self, data_file):
     return count_lines(data_file)

示例#7

文件： sequence_classifier.py 项目： yuhonghong66/OpenNMT-tf

 def _initialize(self, metadata, asset_dir=None):
     assets = super(SequenceClassifier,
                    self)._initialize(metadata, asset_dir=asset_dir)
     self.labels_vocabulary_file = metadata[self.labels_vocabulary_file_key]
     self.num_labels = count_lines(self.labels_vocabulary_file)
     return assets