def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
vocab_freqs = defaultdict(int)
doc_counts = defaultdict(int)
# Fill vocabulary frequencies map and document counts map
for doc in document_generators.documents(
dataset='train',
include_unlabeled=FLAGS.use_unlabeled,
include_validation=FLAGS.include_validation):
fill_vocab_from_doc(doc, vocab_freqs, doc_counts)
# Filter out low-occurring terms
vocab_freqs = dict((term, freq) for term, freq in vocab_freqs.iteritems()
if doc_counts[term] > FLAGS.doc_count_threshold)
# Sort by frequency
ordered_vocab_freqs = data_utils.sort_vocab_by_frequency(vocab_freqs)
# Limit vocab size
ordered_vocab_freqs = ordered_vocab_freqs[:MAX_VOCAB_SIZE]
# Add EOS token
ordered_vocab_freqs.append((data_utils.EOS_TOKEN, 1))
# Write
tf.gfile.MakeDirs(FLAGS.output_dir)
data_utils.write_vocab_and_frequency(ordered_vocab_freqs, FLAGS.output_dir)
def main(_):
tf.logging.set_verbosity(tf.logging.INFO)
vocab_freqs = defaultdict(int)
doc_counts = defaultdict(int)
# Fill vocabulary frequencies map and document counts map
for doc in document_generators.documents(
dataset='train',
include_unlabeled=FLAGS.use_unlabeled,
include_validation=FLAGS.include_validation):
fill_vocab_from_doc(doc, vocab_freqs, doc_counts)
# Filter out low-occurring terms
vocab_freqs = dict((term, freq) for term, freq in iteritems(vocab_freqs)
if doc_counts[term] > FLAGS.doc_count_threshold)
# Sort by frequency
ordered_vocab_freqs = data_utils.sort_vocab_by_frequency(vocab_freqs)
# Limit vocab size
ordered_vocab_freqs = ordered_vocab_freqs[:MAX_VOCAB_SIZE]
# Add EOS token
ordered_vocab_freqs.append((data_utils.EOS_TOKEN, 1))
# Write
tf.gfile.MakeDirs(FLAGS.output_dir)
data_utils.write_vocab_and_frequency(ordered_vocab_freqs, FLAGS.output_dir)