def test_scan_until_(self): lexer = Lexer("foo'bar", "test.fea") self.assertEqual(lexer.pos_, 0) lexer.scan_until_("'") self.assertEqual(lexer.pos_, 3) lexer.scan_until_("'") self.assertEqual(lexer.pos_, 3)
def test_scan_over_(self): lexer = Lexer("abbacabba12", "test.fea") self.assertEqual(lexer.pos_, 0) lexer.scan_over_("xyz") self.assertEqual(lexer.pos_, 0) lexer.scan_over_("abc") self.assertEqual(lexer.pos_, 9) lexer.scan_over_("abc") self.assertEqual(lexer.pos_, 9) lexer.scan_over_("0123456789") self.assertEqual(lexer.pos_, 11)
def test_location(self): locs = lambda s: [ "%s:%d:%d" % loc for (_, _, loc) in Lexer(s, "test.fea") ] self.assertEqual( locs("a b # Comment\n12 @x"), ["test.fea:1:1", "test.fea:1:3", "test.fea:2:1", "test.fea:2:4"])
def next_(self, recurse=False): if len(self.tokens) and not recurse: t = self.tokens.pop(0) if t[0] != VARIABLE: return (t[0], t[1], self.location_()) return self.parse_variable(t[1]) try: return Lexer.next_(self) except FeatureLibError as e: if u"Unexpected character" not in str(e): raise e location = self.location_() text = self.text_ start = self.pos_ cur_char = text[start] if cur_char == '$': self.pos_ += 1 self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_) varname = text[start + 1:self.pos_] if len(varname) < 1 or len(varname) > 63: raise FeatureLibError("Bad variable name length", location) return (VARIABLE, varname, location) else: raise FeatureLibError("Unexpected character: %r" % cur_char, location)
def next_(self, recurse=False): while (not self.empty_): if self.tokens_ is not None: res = self.tokens_.pop(0) if not len(self.tokens_): self.popstack() if res[0] != VARIABLE: return (res[0], res[1], self.location_()) return self.parse_variable(res[1]) try: res = Lexer.next_(self) except IndexError as e: self.popstack() continue except StopIteration as e: self.popstack() continue except FeatureLibError as e: if u"Unexpected character" not in str(e): raise e # only executes if exception occurred location = self.location_() text = self.text_ start = self.pos_ cur_char = text[start] if cur_char == '$': self.pos_ += 1 self.scan_over_(Lexer.CHAR_NAME_CONTINUATION_) varname = text[start+1:self.pos_] if len(varname) < 1 or len(varname) > 63: raise FeatureLibError("Bad variable name length", location) res = (VARIABLE, varname, location) else: raise FeatureLibError("Unexpected character: %r" % cur_char, location) return res raise StopIteration
def locs(s): return [str(loc) for (_, _, loc) in Lexer(s, "test.fea")]
def lines(s): return [loc.line for (_, _, loc) in Lexer(s, "test.fea")]
def lex(s): return [(typ, tok) for (typ, tok, _) in Lexer(s, "test.fea")]
def locs(s): return ["%s:%d:%d" % loc for (_, _, loc) in Lexer(s, "test.fea")]
def test_newline(self): lines = lambda s: [loc[1] for (_, _, loc) in Lexer(s, "test.fea")] self.assertEqual(lines("FOO\n\nBAR\nBAZ"), [1, 3, 4]) # Unix self.assertEqual(lines("FOO\r\rBAR\rBAZ"), [1, 3, 4]) # Macintosh self.assertEqual(lines("FOO\r\n\r\n BAR\r\nBAZ"), [1, 3, 4]) # Windows self.assertEqual(lines("FOO\n\rBAR\r\nBAZ"), [1, 3, 4]) # mixed
def __init__(self, *a): Lexer.__init__(self, *a) self.tokens_ = None self.stack_ = [] self.empty_ = False
def __init__(self, *a): Lexer.__init__(self, *a) self.tokens = []