def __init__(self, bnf_text):
super(DemoGrammar, self).__init__(
bnf_text,
tokenizer=self._tokenize_lines,
parser=PythonParser,
# diff_parser=DiffParser
)
self.version_info = parse_version_string("1.0")
def load_grammar(language='python', version='', path=None):
if language not in ['python', 'move', 'mvir', 'demo']:
raise NotImplementedError("No support for language %s." % language)
if language == 'python':
version_info = parse_version_string(version)
file = path or os.path.join(
'grammar', '%s%s%s.txt' %
(language, version_info.major, version_info.minor))
else:
file = path or os.path.join('grammar', '%s%s.txt' %
(language, version))
global _loaded_grammars
path = os.path.join(os.path.dirname(__file__), file)
if path in _loaded_grammars:
return _loaded_grammars[path]
else:
try:
with open(path) as f:
bnf_text = f.read()
if language == 'python':
grammar = PythonGrammar(version_info, bnf_text)
elif language == 'demo':
from marso.demo.demo_grammar import DemoGrammar
grammar = DemoGrammar(bnf_text)
else:
raise FileNotFoundError()
return _loaded_grammars.setdefault(path, grammar)
except FileNotFoundError:
message = "Python version %s.%s is currently not supported." % (
version_info.major, version_info.minor)
raise NotImplementedError(message)
new_found = found + char
if str.isidentifier(new_found):
found = new_found
else:
if found:
yield create_token()
prefix = ''
pos = start_pos[0], start_pos[1] + i
found = char
is_illegal = True
if found:
yield create_token()
if __name__ == "__main__":
if len(sys.argv) >= 2:
path = sys.argv[1]
with open(path) as f:
code = f.read()
else:
code = sys.stdin.read()
from marso.utils import python_bytes_to_unicode, parse_version_string
if isinstance(code, bytes):
code = python_bytes_to_unicode(code)
for token in tokenize(code, parse_version_string()):
print(token)
def test_parse_version_string(version_str, version):
parsed_version = parse_version_string(version_str)
if len(version) == 1:
assert parsed_version[0] == version[0]
else:
assert parsed_version == version
def works_ge_py39(each_version):
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (3, 9))
def works_ge_py27(each_version):
version_info = parse_version_string(each_version)
return Checker(each_version, version_info >= (2, 7))