def test_path_to_module_name(self):
self.assertIsNone(module_utils.path_to_module_name("../foo.py"))
self.assertEqual("x.y.z",
module_utils.path_to_module_name("x/y/z.pyi"))
self.assertEqual("x.y.z",
module_utils.path_to_module_name("x/y/z.pytd"))
self.assertEqual(
"x.y.z", module_utils.path_to_module_name("x/y/z/__init__.pyi"))
def testPathToModuleName(self):
self.assertIsNone(module_utils.path_to_module_name("../foo.py"))
self.assertEqual("x.y.z",
module_utils.path_to_module_name("x/y/z.pyi"))
self.assertEqual("x.y.z",
module_utils.path_to_module_name("x/y/z.pytd"))
self.assertEqual(
"x.y.z", module_utils.path_to_module_name("x/y/z/__init__.pyi"))
self.assertEqual(
"x.y.z.__init__",
module_utils.path_to_module_name("x/y/z/__init__.pyi",
preserve_init=True))
def blacklisted_modules(self, python_version):
"""Return the blacklist, as a list of module names. E.g. ["x", "y.z"]."""
for full_filename in self.read_blacklist():
filename = os.path.splitext(full_filename)[0]
path = filename.split(os.path.sep) # E.g. ["stdlib", "html", "parser"]
if python_version[0] in self.get_python_major_versions(full_filename):
yield module_utils.path_to_module_name(os.path.sep.join(path[2:]))
def _get_module_names_in_path(lister, path):
names = set()
try:
contents = list(lister(path))
except pytype_source_utils.NoSuchDirectory:
pass
else:
for filename in contents:
names.add(module_utils.path_to_module_name(filename))
return names
def _get_module_names_in_path(lister, path, python_version):
"""Get module names for all .pyi files in the given path."""
names = set()
try:
contents = list(lister(path, python_version))
except pytype_source_utils.NoSuchDirectory:
pass
else:
for filename in contents:
names.add(module_utils.path_to_module_name(filename))
return names
def blacklisted_modules(self, python_version):
"""Return the blacklist, as a list of module names. E.g. ["x", "y.z"]."""
for full_filename in self.read_blacklist():
filename = os.path.splitext(full_filename)[0]
path = filename.split("/") # E.g. ["stdlib", "2", "html", "parser.pyi"]
# It's possible that something is blacklisted with a more
# specific version (e.g. stdlib/3.4/...). That usually just means
# that this module didn't exist in earlier Python versions. So
# we can still just use python_version[0].
if (path[1].startswith(str(python_version[0])) or
path[1] == "2and3"):
yield module_utils.path_to_module_name("/".join(path[2:]))
def blacklisted_modules(self):
"""Return the blacklist, as a list of module names. E.g. ["x", "y.z"]."""
for path in self.read_blacklist():
parts = path.split(
os.path.sep) # E.g. ["stdlib", "html", "parser.pyi"]
if parts[0] == "stdlib":
filename = os.path.sep.join(parts[1:])
else:
filename = os.path.sep.join(parts[2:])
mod = module_utils.path_to_module_name(filename)
if mod:
yield mod
def process_imports_map(imports_map):
"""Generate a map of {module name: canonical relative path}."""
if not imports_map:
return {}
# Store maps of the full path and canonical relative path.
mod_to_fp = {}
fp_to_cp = {}
for path, v in imports_map.items():
mod = module_utils.path_to_module_name(path)
mod_to_fp[mod] = v
if v not in fp_to_cp or len(path) > len(fp_to_cp[v]):
fp_to_cp[v] = path
return {mod: fp_to_cp[fp] for mod, fp in mod_to_fp.items()}
def get_all_module_names(self, python_version):
"""Get the names of all modules in typeshed or bundled with pytype."""
if self._env_home:
raise NotImplementedError(
"Not implemented: Can't scan external typeshed")
paths = (self.get_typeshed_paths(python_version) +
self.get_pytd_paths(python_version))
subdirs = [d.rpartition("pytype/")[-1] for d in paths]
module_names = set()
for subdir in subdirs:
try:
contents = list(pytype_source_utils.list_pytype_files(subdir))
except pytype_source_utils.NoSuchDirectory:
pass
else:
for filename in contents:
module_names.add(
module_utils.path_to_module_name(filename))
assert "ctypes" in module_names # sanity check
return module_names
def test_file(filename):
with open(filename, 'r') as f:
src = f.read()
version = sys.version_info[:2]
# get module name from filename
mod = module_utils.path_to_module_name(filename)
if mod.startswith('third_party.py.typeshed'):
mod = mod.split('.')[5:]
mod = '.'.join(mod)
# Parse using typed ast parser
out = ast_parser.parse_pyi(src,
filename=filename,
module_name=mod,
python_version=version)
new = pytd_utils.Print(out).replace('"', "'").splitlines(True)
# Parse using bison parser
pytd = parser.old_parse_string(src,
name=mod,
filename=filename,
python_version=version)
old = pytd_utils.Print(pytd).replace('"', "'").splitlines(True)
if old != new:
print(''.join(old))
diff = difflib.unified_diff(old,
new,
fromfile=filename,
tofile=filename)
# Display first diff we find and exit.
sys.stdout.writelines(diff)
print()
print(f"FAILED: '{filename}'")
sys.exit(1)
"""Testing code to run the typed_ast based pyi parser."""
import sys
from pytype import module_utils
from pytype.pyi import parser
from pytype.pyi.types import ParseError # pylint: disable=g-importing-member
from pytype.pytd import pytd_utils
if __name__ == '__main__':
filename = sys.argv[1]
with open(filename, 'r') as f:
src = f.read()
module_name = module_utils.path_to_module_name(filename)
try:
out, _ = parser.parse_pyi_debug(src, filename, module_name)
except ParseError as e:
print(e)
sys.exit(1)
print('------pytd--------------')
print(out)
print('------round trip--------------')
print(pytd_utils.Print(out))
