def test_scan_until_(self):
lexer = Lexer("foo'bar", "test.fea")
self.assertEqual(lexer.pos_, 0)
lexer.scan_until_("'")
self.assertEqual(lexer.pos_, 3)
lexer.scan_until_("'")
self.assertEqual(lexer.pos_, 3)
def test_scan_over_(self):
lexer = Lexer("abbacabba12", "test.fea")
self.assertEqual(lexer.pos_, 0)
lexer.scan_over_("xyz")
self.assertEqual(lexer.pos_, 0)
lexer.scan_over_("abc")
self.assertEqual(lexer.pos_, 9)
lexer.scan_over_("abc")
self.assertEqual(lexer.pos_, 9)
lexer.scan_over_("0123456789")
self.assertEqual(lexer.pos_, 11)
def test_scan_over_(self):
lexer = Lexer("abbacabba12", "test.fea")
self.assertEqual(lexer.pos_, 0)
lexer.scan_over_("xyz")
self.assertEqual(lexer.pos_, 0)
lexer.scan_over_("abc")
self.assertEqual(lexer.pos_, 9)
lexer.scan_over_("abc")
self.assertEqual(lexer.pos_, 9)
lexer.scan_over_("0123456789")
self.assertEqual(lexer.pos_, 11)
def test_location(self):
locs = lambda s: [
"%s:%d:%d" % loc for (_, _, loc) in Lexer(s, "test.fea")
]
self.assertEqual(
locs("a b # Comment\n12 @x"),
["test.fea:1:1", "test.fea:1:3", "test.fea:2:1", "test.fea:2:4"])
def next_(self, recurse=False):
if len(self.tokens) and not recurse:
t = self.tokens.pop(0)
if t[0] != VARIABLE:
return (t[0], t[1], self.location_())
return self.parse_variable(t[1])
try:
return Lexer.next_(self)
except FeatureLibError as e:
if u"Unexpected character" not in str(e):
raise e
location = self.location_()
text = self.text_
start = self.pos_
cur_char = text[start]
if cur_char == '$':
self.pos_ += 1
self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_)
varname = text[start + 1:self.pos_]
if len(varname) < 1 or len(varname) > 63:
raise FeatureLibError("Bad variable name length", location)
return (VARIABLE, varname, location)
else:
raise FeatureLibError("Unexpected character: %r" % cur_char,
location)
def test_scan_until_(self):
lexer = Lexer("foo'bar", "test.fea")
self.assertEqual(lexer.pos_, 0)
lexer.scan_until_("'")
self.assertEqual(lexer.pos_, 3)
lexer.scan_until_("'")
self.assertEqual(lexer.pos_, 3)
def next_(self, recurse=False):
while (not self.empty_):
if self.tokens_ is not None:
res = self.tokens_.pop(0)
if not len(self.tokens_):
self.popstack()
if res[0] != VARIABLE:
return (res[0], res[1], self.location_())
return self.parse_variable(res[1])
try:
res = Lexer.next_(self)
except IndexError as e:
self.popstack()
continue
except StopIteration as e:
self.popstack()
continue
except FeatureLibError as e:
if u"Unexpected character" not in str(e):
raise e
# only executes if exception occurred
location = self.location_()
text = self.text_
start = self.pos_
cur_char = text[start]
if cur_char == '$':
self.pos_ += 1
self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_)
varname = text[start+1:self.pos_]
if len(varname) < 1 or len(varname) > 63:
raise FeatureLibError("Bad variable name length", location)
res = (VARIABLE, varname, location)
else:
raise FeatureLibError("Unexpected character: %r" % cur_char, location)
return res
raise StopIteration
def locs(s):
return [str(loc) for (_, _, loc) in Lexer(s, "test.fea")]
def lines(s):
return [loc.line for (_, _, loc) in Lexer(s, "test.fea")]
def lex(s):
return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")]
def locs(s):
return ["%s:%d:%d" % loc for (_, _, loc) in Lexer(s, "test.fea")]
def test_newline(self):
lines = lambda s: [loc[1] for (_, _, loc) in Lexer(s, "test.fea")]
self.assertEqual(lines("FOO\n\nBAR\nBAZ"), [1, 3, 4]) # Unix
self.assertEqual(lines("FOO\r\rBAR\rBAZ"), [1, 3, 4]) # Macintosh
self.assertEqual(lines("FOO\r\n\r\n BAR\r\nBAZ"), [1, 3, 4]) # Windows
self.assertEqual(lines("FOO\n\rBAR\r\nBAZ"), [1, 3, 4]) # mixed
def __init__(self, *a):
Lexer.__init__(self, *a)
self.tokens_ = None
self.stack_ = []
self.empty_ = False
def __init__(self, *a):
Lexer.__init__(self, *a)
self.tokens = []
