def test_span(self):
doc = self._getdoc(' ab\\cc def ghi ')
root = syntax_highlight.Tokenizer(tokens=[
('span1', syntax_highlight.Span('span', 'a', 'c',
escape='\\')),
('span2', syntax_highlight.Span('span', 'd', 'f',
capture_end=False)),
('span3', syntax_highlight.Span('span', 'g', 'i', terminate_tokens='h')),
])
tokenizer = syntax_highlight.begin_tokenizer(
doc, root, 0)
styles = list(tokenizer)
assert styles == [
(0, 1, root.styleid_default),
(1, 2, root.tokens.span1.styleid_span),
(2, 5, root.tokens.span1.styleid_span),
(5, 6, root.tokens.span1.styleid_span),
(6, 7, root.styleid_default),
(7, 8, root.tokens.span2.styleid_span),
(8, 9, root.tokens.span2.styleid_span),
(9, 11, root.styleid_default),
(11, 12, root.tokens.span3.styleid_span),
]
def test_span(self):
doc = self._getdoc(' ab\\cc def ghi ')
root = syntax_highlight.Tokenizer(tokens=[
('span1', syntax_highlight.Span('span', 'a', 'c', escape='\\')),
('span2',
syntax_highlight.Span('span', 'd', 'f', capture_end=False)),
('span3',
syntax_highlight.Span('span', 'g', 'i', terminate_tokens='h')),
])
tokenizer = syntax_highlight.begin_tokenizer(doc, root, 0)
styles = list(tokenizer)
assert styles == [
(0, 1, root.styleid_default),
(1, 2, root.tokens.span1.styleid_span),
(2, 5, root.tokens.span1.styleid_span),
(5, 6, root.tokens.span1.styleid_span),
(6, 7, root.styleid_default),
(7, 8, root.tokens.span2.styleid_span),
(8, 9, root.tokens.span2.styleid_span),
(9, 11, root.styleid_default),
(11, 12, root.tokens.span3.styleid_span),
]
def test_resume(self):
doc = self._getdoc(' abc')
root = syntax_highlight.Tokenizer(
tokens=[('keyword',
syntax_highlight.Keywords('keyword', ['abc', 'ghi']))])
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 2)
styles = list(tokeniter)
assert styles == [(0, 4, root.styleid_default),
(4, 7, root.tokens.keyword.styleid_token)]
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 6)
doc.styles.setints(4, 7, root.tokens.keyword.styleid_token)
styles = list(tokeniter)
assert styles == [(0, 4, root.styleid_default),
(4, 7, root.tokens.keyword.styleid_token)]
def test_resume(self):
doc = self._getdoc(' abc')
root = syntax_highlight.Tokenizer(tokens=[
('keyword', syntax_highlight.Keywords('keyword', ['abc', 'ghi']))])
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 2)
styles = list(tokeniter)
assert styles == [
(0, 4, root.styleid_default),
(4, 7, root.tokens.keyword.styleid_token)]
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 6)
doc.styles.setints(4, 7, root.tokens.keyword.styleid_token)
styles = list(tokeniter)
assert styles == [
(0, 4, root.styleid_default),
(4, 7, root.tokens.keyword.styleid_token)]
def test_token(self):
doc = self._getdoc(' abc def ghi ')
root = syntax_highlight.Tokenizer(tokens=[
('keyword', syntax_highlight.Keywords('keyword', ['abc', 'ghi']))
])
tokenizer = syntax_highlight.begin_tokenizer(
doc, root, 0)
styles = list(tokenizer)
assert styles == [
(0, 1, root.styleid_default),
(1, 4, root.tokens.keyword.styleid_token),
(4, 9, root.styleid_default),
(9, 12, root.tokens.keyword.styleid_token),
(12, 13, root.styleid_default), ]
def test_token(self):
doc = self._getdoc(' abc def ghi ')
root = syntax_highlight.Tokenizer(
tokens=[('keyword',
syntax_highlight.Keywords('keyword', ['abc', 'ghi']))])
tokenizer = syntax_highlight.begin_tokenizer(doc, root, 0)
styles = list(tokenizer)
assert styles == [
(0, 1, root.styleid_default),
(1, 4, root.tokens.keyword.styleid_token),
(4, 9, root.styleid_default),
(9, 12, root.tokens.keyword.styleid_token),
(12, 13, root.styleid_default),
]
def run_tokenizer(self, batch=HIGHLIGHTBATCH):
range_start, range_end = self._get_highlight_range()
if self._highlight_done < range_start:
self._highlight_done = range_start
if not self._highlight_iter:
f = max(range_start, self._highlight_done - 1)
self._highlight_iter = syntax_highlight.begin_tokenizer(
self.document, self.tokenizer, f)
updatefrom = self.document.endpos()
updateto = range_start
updated = False
finished = False
for n, (f, t, style) in enumerate(self._highlight_iter):
f = max(range_start, f)
t = min(range_end, t)
if f < t:
self.document.styles.setints(f, t, style)
updated = True
updatefrom = min(f, updatefrom)
updateto = max(t, updateto)
if batch and (n > batch):
break
if t >= range_end:
finished = True
break
else:
finished = True
if self.document.endpos() == 0 or updated and (updatefrom != updateto):
self.document.style_updated(updatefrom, updateto)
self._highlight_done = updateto
# returns False if finished to terminate idle loop.
return not finished
def test_nest_resume(self):
doc = self._getdoc(' abc 123 x abc')
class SubKeyword(syntax_highlight.Keywords):
def on_start(self, doc, match):
pos, terminates = yield from super().on_start(doc, match)
return (yield from sub.run(doc, pos)), False
root = syntax_highlight.Tokenizer(tokens=[
('subkeyword', SubKeyword('keyword', ['abc']))])
sub = syntax_highlight.Tokenizer(parent=root, terminates='x', tokens=[
('keyword', syntax_highlight.Keywords('keyword', ['123']))])
doc.styles.setints(0, 3, root.styleid_default)
doc.styles.setints(3, 6, root.tokens.subkeyword.styleid_token)
doc.styles.setints(6, 9, sub.styleid_default)
doc.styles.setints(9, 12, sub.tokens.keyword.styleid_token)
# parse at blank
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 2)
styles = list(tokeniter)
assert styles == [
(0, 3, root.styleid_default),
(3, 6, root.tokens.subkeyword.styleid_token),
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at first token
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 4)
styles = list(tokeniter)
assert styles == [
(0, 3, root.styleid_default),
(3, 6, root.tokens.subkeyword.styleid_token),
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at blank of sub2
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 8)
styles = list(tokeniter)
assert styles == [
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at blank of keyword2
tokeniter = syntax_highlight.begin_tokenizer(
doc, root, 11)
styles = list(tokeniter)
assert styles == [
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
def test_nest_resume(self):
doc = self._getdoc(' abc 123 x abc')
class SubKeyword(syntax_highlight.Keywords):
def on_start(self, doc, match):
pos, terminates = yield from super().on_start(doc, match)
return (yield from sub.run(doc, pos)), False
root = syntax_highlight.Tokenizer(
tokens=[('subkeyword', SubKeyword('keyword', ['abc']))])
sub = syntax_highlight.Tokenizer(
parent=root,
terminates='x',
tokens=[('keyword', syntax_highlight.Keywords('keyword',
['123']))])
doc.styles.setints(0, 3, root.styleid_default)
doc.styles.setints(3, 6, root.tokens.subkeyword.styleid_token)
doc.styles.setints(6, 9, sub.styleid_default)
doc.styles.setints(9, 12, sub.tokens.keyword.styleid_token)
# parse at blank
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 2)
styles = list(tokeniter)
assert styles == [(0, 3, root.styleid_default),
(3, 6, root.tokens.subkeyword.styleid_token),
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at first token
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 4)
styles = list(tokeniter)
assert styles == [(0, 3, root.styleid_default),
(3, 6, root.tokens.subkeyword.styleid_token),
(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at blank of sub2
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 8)
styles = list(tokeniter)
assert styles == [(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
# parse at blank of keyword2
tokeniter = syntax_highlight.begin_tokenizer(doc, root, 11)
styles = list(tokeniter)
assert styles == [(6, 9, sub.styleid_default),
(9, 12, sub.tokens.keyword.styleid_token),
(12, 13, sub.styleid_default),
(13, 15, root.styleid_default),
(15, 18, root.tokens.subkeyword.styleid_token)]
