def load_packaged_grammar(package, grammar_source): data = slew.load_resource( 'data/' + driver._generate_pickle_name( os.path.basename(grammar_source))) g = driver.grammar.Grammar() g.loads(data) return g
def test_load_grammar_from_subprocess(self): tmpdir = tempfile.mkdtemp() tmpsubdir = os.path.join(tmpdir, 'subdir') try: os.mkdir(tmpsubdir) grammar_base = os.path.basename(support.grammar_path) grammar_copy = os.path.join(tmpdir, grammar_base) grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) shutil.copy(support.grammar_path, grammar_copy) shutil.copy(support.grammar_path, grammar_sub_copy) pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) pickle_sub_name = pgen2_driver._generate_pickle_name( grammar_sub_copy) self.assertNotEqual(pickle_name, pickle_sub_name) # Generate a pickle file from this process. pgen2_driver.load_grammar(grammar_copy, save=True, force=True) self.assertTrue(os.path.exists(pickle_name)) # Generate a new pickle file in a subprocess with a most likely # different hash randomization seed. sub_env = dict(os.environ) sub_env['PYTHONHASHSEED'] = 'random' code = """ from lib2to3.pgen2 import driver as pgen2_driver pgen2_driver.load_grammar(%r, save=True, force=True) """ % (grammar_sub_copy, ) msg = ("lib2to3 package is deprecated and may not be able " "to parse Python 3.10+") cmd = [ sys.executable, f'-Wignore:{msg}:PendingDeprecationWarning', '-c', code ] subprocess.check_call(cmd, env=sub_env) self.assertTrue(os.path.exists(pickle_sub_name)) with open(pickle_name, 'rb') as pickle_f_1, \ open(pickle_sub_name, 'rb') as pickle_f_2: self.assertEqual( pickle_f_1.read(), pickle_f_2.read(), msg='Grammar caches generated using different hash seeds' ' were not identical.') finally: shutil.rmtree(tmpdir)
def test_load_grammar_from_subprocess(self): tmpdir = tempfile.mkdtemp() tmpsubdir = os.path.join(tmpdir, 'subdir') try: os.mkdir(tmpsubdir) grammar_base = os.path.basename(support.grammar_path) grammar_copy = os.path.join(tmpdir, grammar_base) grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) shutil.copy(support.grammar_path, grammar_copy) shutil.copy(support.grammar_path, grammar_sub_copy) pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) pickle_sub_name = pgen2_driver._generate_pickle_name( grammar_sub_copy) self.assertNotEqual(pickle_name, pickle_sub_name) # Generate a pickle file from this process. pgen2_driver.load_grammar(grammar_copy, save=True, force=True) self.assertTrue(os.path.exists(pickle_name)) # Generate a new pickle file in a subprocess with a most likely # different hash randomization seed. sub_env = dict(os.environ) sub_env['PYTHONHASHSEED'] = 'random' subprocess.check_call( [sys.executable, '-c', """ from lib2to3.pgen2 import driver as pgen2_driver pgen2_driver.load_grammar(%r, save=True, force=True) """ % (grammar_sub_copy,)], env=sub_env) self.assertTrue(os.path.exists(pickle_sub_name)) with open(pickle_name, 'rb') as pickle_f_1, \ open(pickle_sub_name, 'rb') as pickle_f_2: self.assertEqual( pickle_f_1.read(), pickle_f_2.read(), msg='Grammar caches generated using different hash seeds' ' were not identical.') finally: shutil.rmtree(tmpdir)
def test_load_grammar_from_pickle(self): tmpdir = tempfile.mkdtemp() try: grammar_copy = os.path.join(tmpdir, os.path.basename(support.grammar_path)) shutil.copy(support.grammar_path, grammar_copy) pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) pgen2_driver.load_grammar(grammar_copy, save=True, force=True) self.assertTrue(os.path.exists(pickle_name)) os.unlink(grammar_copy) pgen2_driver.load_grammar(grammar_copy, save=False, force=False) finally: shutil.rmtree(tmpdir)
def test_load_grammar_from_subprocess(self): tmpdir = tempfile.mkdtemp() tmpsubdir = os.path.join(tmpdir, 'subdir') try: os.mkdir(tmpsubdir) grammar_base = os.path.basename(support.grammar_path) grammar_copy = os.path.join(tmpdir, grammar_base) grammar_sub_copy = os.path.join(tmpsubdir, grammar_base) shutil.copy(support.grammar_path, grammar_copy) shutil.copy(support.grammar_path, grammar_sub_copy) pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) pickle_sub_name = pgen2_driver._generate_pickle_name( grammar_sub_copy) self.assertNotEqual(pickle_name, pickle_sub_name) pgen2_driver.load_grammar(grammar_copy, save=True, force=True) self.assertTrue(os.path.exists(pickle_name)) sub_env = dict(os.environ) sub_env['PYTHONHASHSEED'] = 'random' subprocess.check_call([ sys.executable, '-c', """ from lib2to3.pgen2 import driver as pgen2_driver pgen2_driver.load_grammar(%r, save=True, force=True) """ % (grammar_sub_copy, ) ], env=sub_env) self.assertTrue(os.path.exists(pickle_sub_name)) with open(pickle_name, 'rb') as pickle_f_1, open(pickle_sub_name, 'rb') as pickle_f_2: self.assertEqual( pickle_f_1.read(), pickle_f_2.read(), msg= 'Grammar caches generated using different hash seeds were not identical.' ) finally: shutil.rmtree(tmpdir)
def test_load_grammar_from_pickle(self): # Make a copy of the grammar file in a temp directory we are # guaranteed to be able to write to. tmpdir = tempfile.mkdtemp() try: grammar_copy = os.path.join( tmpdir, os.path.basename(support.grammar_path)) shutil.copy(support.grammar_path, grammar_copy) pickle_name = pgen2_driver._generate_pickle_name(grammar_copy) pgen2_driver.load_grammar(grammar_copy, save=True, force=True) self.assertTrue(os.path.exists(pickle_name)) os.unlink(grammar_copy) # Only the pickle remains... pgen2_driver.load_grammar(grammar_copy, save=False, force=False) finally: shutil.rmtree(tmpdir)
# gen_2to3_grammar.py: input import sys from lib2to3.pgen2 import driver gp = driver._generate_pickle_name(sys.argv[1]) driver.load_grammar(sys.argv[1], gp, force=True) print(gp)