def ensure_stream_exhausted(toks, caller, context_length=DefaultContextLength): """Raises a DocParseException unless the stream _toks_ is empty. If not empty, displays a preview of the first few remaining tokens in the stream.""" if toks.peek() is not None: raise DocParseException, "%s: Tokens at end of input. {next tokens: %s}" % ( caller, list(take(context_length, toks)), )
def ensure_stream_exhausted(toks, caller, context_length=DefaultContextLength): '''Raises a DocParseException unless the stream _toks_ is empty. If not empty, displays a preview of the first few remaining tokens in the stream.''' if toks.peek() is not None: raise DocParseException, "%s: Tokens at end of input. {next tokens: %s}" % ( caller, list(take(context_length, toks)))
def get_context(toks, ntokens=10): return ", ".join(take(ntokens, toks))