def build_input_sequence(doc, vocab_ids): """Builds input sequence from file. Splits lines on whitespace. Treats punctuation as whitespace. For word-level sequences, only keeps terms that are in the vocab. Terms are added as token in the SequenceExample. The EOS_TOKEN is also appended. Label and weight features are set to 0. Args: doc: Document (defined in `document_generators`) from which to build the sequence. vocab_ids: dict<term, id>. Returns: SequenceExampleWrapper. """ seq = data.SequenceWrapper() for token in document_generators.tokens(doc): if token in vocab_ids: seq.add_timestep().set_token(vocab_ids[token]) # Add EOS token to end seq.add_timestep().set_token(vocab_ids[data.EOS_TOKEN]) return seq
def build_input_sequence(doc, vocab_ids): """Builds input sequence from file. Splits lines on whitespace. Treats punctuation as whitespace. For word-level sequences, only keeps terms that are in the vocab. Terms are added as token in the SequenceExample. The EOS_TOKEN is also appended. Label and weight features are set to 0. Args: doc: Document (defined in `document_generators`) from which to build the sequence. vocab_ids: dict<term, id>. Returns: SequenceExampleWrapper. """ seq = data.SequenceWrapper() for token in document_generators.tokens(doc): if token in vocab_ids: seq.add_timestep().set_token(vocab_ids[token]) # Add EOS token to end seq.add_timestep().set_token(vocab_ids[data.EOS_TOKEN]) return seq
def fill_vocab_from_doc(doc, vocab_freqs, doc_counts): """Fills vocabulary and doc counts with tokens from doc. Args: doc: Document to read tokens from. vocab_freqs: dict<token, frequency count> doc_counts: dict<token, document count> Returns: None """ doc_seen = set() for token in document_generators.tokens(doc): if doc.add_tokens or token in vocab_freqs: vocab_freqs[token] += 1 if token not in doc_seen: doc_counts[token] += 1 doc_seen.add(token)
def fill_vocab_from_doc(doc, vocab_freqs, doc_counts): """Fills vocabulary and doc counts with tokens from doc. Args: doc: Document to read tokens from. vocab_freqs: dict<token, frequency count> doc_counts: dict<token, document count> Returns: None """ doc_seen = set() for token in document_generators.tokens(doc): if doc.add_tokens or token in vocab_freqs: vocab_freqs[token] += 1 if token not in doc_seen: doc_counts[token] += 1 doc_seen.add(token)