def test_load_grammar_from_pickle(self):
# Make a copy of the grammar file in a temp directory we are
# guaranteed to be able to write to.
tmpdir = tempfile.mkdtemp()
try:
grammar_copy = os.path.join(tmpdir, os.path.basename(support.grammar_path))
shutil.copy(support.grammar_path, grammar_copy)
pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
self.assertTrue(os.path.exists(pickle_name))
os.unlink(grammar_copy) # Only the pickle remains...
pgen2_driver.load_grammar(grammar_copy, save=False, force=False)
finally:
shutil.rmtree(tmpdir)
def test_load_grammar_from_subprocess(self):
tmpdir = tempfile.mkdtemp()
tmpsubdir = os.path.join(tmpdir, "subdir")
try:
os.mkdir(tmpsubdir)
grammar_base = os.path.basename(support.grammar_path)
grammar_copy = os.path.join(tmpdir, grammar_base)
grammar_sub_copy = os.path.join(tmpsubdir, grammar_base)
shutil.copy(support.grammar_path, grammar_copy)
shutil.copy(support.grammar_path, grammar_sub_copy)
pickle_name = pgen2_driver._generate_pickle_name(grammar_copy)
pickle_sub_name = pgen2_driver._generate_pickle_name(
grammar_sub_copy)
self.assertNotEqual(pickle_name, pickle_sub_name)
# Generate a pickle file from this process.
pgen2_driver.load_grammar(grammar_copy, save=True, force=True)
self.assertTrue(os.path.exists(pickle_name))
# Generate a new pickle file in a subprocess with a most likely
# different hash randomization seed.
sub_env = dict(os.environ)
sub_env["PYTHONHASHSEED"] = "random"
subprocess.check_call(
[
sys.executable,
"-c",
"""
from fissix.pgen2 import driver as pgen2_driver
pgen2_driver.load_grammar(%r, save=True, force=True)
""" % (grammar_sub_copy, ),
],
env=sub_env,
)
self.assertTrue(os.path.exists(pickle_sub_name))
with open(pickle_name,
"rb") as pickle_f_1, open(pickle_sub_name,
"rb") as pickle_f_2:
self.assertEqual(
pickle_f_1.read(),
pickle_f_2.read(),
msg="Grammar caches generated using different hash seeds"
" were not identical.",
)
finally:
shutil.rmtree(tmpdir)
def main():
gr = driver.load_grammar("Grammar.txt")
dr = driver.Driver(gr, convert=pytree.convert)
fn = "example.py"
tree = dr.parse_file(fn, debug=True)
if not diff(fn, tree):
print("No diffs.")
if not sys.argv[1:]:
return # Pass a dummy argument to run the complete test suite below
problems = []
# Process every imported module
for name in sys.modules:
mod = sys.modules[name]
if mod is None or not hasattr(mod, "__file__"):
continue
fn = mod.__file__
if fn.endswith(".pyc"):
fn = fn[:-1]
if not fn.endswith(".py"):
continue
print("Parsing", fn, file=sys.stderr)
tree = dr.parse_file(fn, debug=True)
if diff(fn, tree):
problems.append(fn)
# Process every single module on sys.path (but not in packages)
for dir in sys.path:
try:
names = os.listdir(dir)
except OSError:
continue
print("Scanning", dir, "...", file=sys.stderr)
for name in names:
if not name.endswith(".py"):
continue
print("Parsing", name, file=sys.stderr)
fn = os.path.join(dir, name)
try:
tree = dr.parse_file(fn, debug=True)
except pgen2.parse.ParseError as err:
print("ParseError:", err)
else:
if diff(fn, tree):
problems.append(fn)
# Show summary of problem files
if not problems:
print("No problems. Congratulations!")
else:
print("Problems in following files:")
for fn in problems:
print("***", fn)
def test_load_grammar_from_txt_file(self):
pgen2_driver.load_grammar(support.grammar_path, save=False, force=True)
from fissix.pgen2 import driver as pgen2_driver
from fissix.pgen2 import token as pgen2_token
from fissix.pygram import python_symbols
from fissix import patcomp
# it's handy to import this from here when developing
from bowler.helpers import print_tree as dumpTree
import os
from textwrap import dedent
# --------------------------------------------------------------------------------------------------
# load grammar & make a driver we can use to parse
fissix_dir = os.path.dirname( fissix.__file__ )
grammar_path = os.path.join( fissix_dir, "Grammar.txt" )
m_grammar = pgen2_driver.load_grammar( grammar_path )
driver = pgen2_driver.Driver( m_grammar, convert=fissix.pytree.convert )
def parseString( string ):
return driver.parse_string( dedent( string ) + "\n\n", debug=True )
def getGrammar():
return m_grammar
# --------------------------------------------------------------------------------------------------
def makeLeaf( type_name, value, prefix="" ):
type_num = typeNameToNum( type_name )
return Leaf( type_num, value, prefix=prefix )
def makeStatement():
# Author: Collin Winter
# Python imports
import unittest
import os
import os.path
from textwrap import dedent
# Local imports
from fissix import pytree, refactor
from fissix.pgen2 import driver as pgen2_driver
test_dir = os.path.dirname(__file__)
proj_dir = os.path.normpath(os.path.join(test_dir, ".."))
grammar_path = os.path.join(test_dir, "..", "Grammar.txt")
grammar = pgen2_driver.load_grammar(grammar_path)
grammar_no_print_statement = pgen2_driver.load_grammar(grammar_path)
del grammar_no_print_statement.keywords["print"]
driver = pgen2_driver.Driver(grammar, convert=pytree.convert)
driver_no_print_statement = pgen2_driver.Driver(
grammar_no_print_statement, convert=pytree.convert
)
def parse_string(string):
return driver.parse_string(reformat(string), debug=True)
def run_all_tests(test_mod=None, tests=None):
if tests is None:
tests = unittest.TestLoader().loadTestsFromModule(test_mod)