def parse(self, stream=None, inline=False, enable_escaping=False): """ Parse an existing stream or the current `raw` document, apply node-filters and return a node-tree. :param stream: An existing stream. If `None` the `raw` attribute is processed into a `TokenStream`. :param inline: If `True` only child-nodes are returned and no `Document` node as the top level one. :return: A node-tree that represents the finished document in an abstract form. """ if stream is None: stream = self.tokenize(enable_escaping=enable_escaping) # create the node-tree document = events.emit_ovr("define-document-node")() while not stream.eof: node = self.dispatch_node(stream) if node is not None: document.children.append(node) else: stream.next() # apply node-filters ctx = Context(self, enable_escaping) for callback in events.iter_callbacks("process-doc-tree"): ret = callback(document, ctx) if ret is not None: document = ret if inline: return document.children return document
def parse(self, stream=None, inline=False, enable_escaping=False): """ Parse an existing stream or the current `raw` document, apply node-filters and return a node-tree. :param stream: An existing stream. If `None` the `raw` attribute is processed into a `TokenStream`. :param inline: If `True` only child-nodes are returned and no `Document` node as the top level one. :return: A node-tree that represents the finished document in an abstract form. """ if stream is None: stream = self.tokenize(enable_escaping=enable_escaping) # create the node-tree document = events.emit_ovr('define-document-node')() while not stream.eof: node = self.dispatch_node(stream) if node is not None: document.children.append(node) else: stream.next() # apply node-filters ctx = Context(self, enable_escaping) for callback in events.iter_callbacks('process-doc-tree'): ret = callback(document, ctx) if ret is not None: document = ret if inline: return document.children return document
def tokenize(self, raw=None, enable_escaping=False): """ Tokenize the raw document, apply stream-filters and return the processing-ready token stream. :param raw: The raw document. :return: A `TokenStream` instance. """ ctx = Context(self, enable_escaping) stream = TokenStream.from_tuple_iter(self._process_lexing_rules(raw or self.raw, enable_escaping)) for callback in events.iter_callbacks("process-stream"): ret = callback(stream, ctx) if ret is not None: stream = ret return stream
def tokenize(self, raw=None, enable_escaping=False): """ Tokenize the raw document, apply stream-filters and return the processing-ready token stream. :param raw: The raw document. :return: A `TokenStream` instance. """ ctx = Context(self, enable_escaping) stream = TokenStream.from_tuple_iter( self._process_lexing_rules(raw or self.raw, enable_escaping)) for callback in events.iter_callbacks('process-stream'): ret = callback(stream, ctx) if ret is not None: stream = ret return stream