def pyff_package(old_package: PackageSummary,
new_package: PackageSummary) -> Optional[PackagePyfference]:
"""Given summaries of two versions of a package, return differences between them"""
old_files: Set[pathlib.Path] = {
pathlib.Path(module)
for module in get_module_files(old_package.path, ())
}
new_files: Set[pathlib.Path] = {
pathlib.Path(module)
for module in get_module_files(new_package.path, ())
}
LOGGER.debug("Files of the old package %s: %s", str(old_package.path),
old_files)
LOGGER.debug("Files of the new package %s: %s", str(new_package.path),
new_files)
old_modules: Set[pathlib.Path] = {
pathlib.Path(module)
for module in extract_modules(old_files, old_package)
}
new_modules: Set[pathlib.Path] = {
pathlib.Path(module)
for module in extract_modules(new_files, new_package)
}
LOGGER.debug("Old modules: %s", str(old_modules))
LOGGER.debug("New modules: %s", str(new_modules))
removed = old_modules - new_modules
new = new_modules - old_modules
both = old_modules.intersection(new_modules)
LOGGER.debug("Removed modules: %s", str(removed))
LOGGER.debug("Modules in both packages: %s", str(both))
LOGGER.debug("New modules: %s", str(new))
removed_summaries = {
module: _summarize_module_in_package(module, old_package)
for module in removed
}
new_summaries = {
module: _summarize_module_in_package(module, new_package)
for module in new
}
changed = {
module: change
for module, change in [(
module,
_compare_module_in_packages(module, old_package, new_package))
for module in both] if change is not None
}
modules = pm.ModulesPyfference(removed_summaries, changed, new_summaries)
if modules:
return PackagePyfference(modules)
return None
def test_get_module_files_1(self):
package = resources.find('data/find_test')
modules = set(modutils.get_module_files(package, []))
expected = ['__init__.py', 'module.py', 'module2.py',
'noendingnewline.py', 'nonregr.py']
self.assertEqual(modules,
{os.path.join(package, x) for x in expected})
def project_from_files(files,
func_wrapper=_astroid_wrapper,
project_name="no name",
black_list=("CVS", )):
"""return a Project from a list of files or modules"""
# build the project representation
astroid_manager = manager.AstroidManager()
project = Project(project_name)
for something in files:
if not os.path.exists(something):
fpath = modutils.file_from_modpath(something.split("."))
elif os.path.isdir(something):
fpath = os.path.join(something, "__init__.py")
else:
fpath = something
ast = func_wrapper(astroid_manager.ast_from_file, fpath)
if ast is None:
continue
project.path = project.path or ast.file
project.add_module(ast)
base_name = ast.name
# recurse in package except if __init__ was explicitly given
if ast.package and something.find("__init__") == -1:
# recurse on others packages / modules if this is a package
for fpath in modutils.get_module_files(os.path.dirname(ast.file),
black_list):
ast = func_wrapper(astroid_manager.ast_from_file, fpath)
if ast is None or ast.name == base_name:
continue
project.add_module(ast)
return project
def project_from_files(files, func_wrapper=_astroid_wrapper,
project_name="no name",
black_list=('CVS',)):
"""return a Project from a list of files or modules"""
# build the project representation
astroid_manager = manager.AstroidManager()
project = Project(project_name)
for something in files:
if not os.path.exists(something):
fpath = modutils.file_from_modpath(something.split('.'))
elif os.path.isdir(something):
fpath = os.path.join(something, '__init__.py')
else:
fpath = something
ast = func_wrapper(astroid_manager.ast_from_file, fpath)
if ast is None:
continue
# XXX why is first file defining the project.path ?
project.path = project.path or ast.file
project.add_module(ast)
base_name = ast.name
# recurse in package except if __init__ was explicitly given
if ast.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in modutils.get_module_files(os.path.dirname(ast.file),
black_list):
ast = func_wrapper(astroid_manager.ast_from_file, fpath)
if ast is None or ast.name == base_name:
continue
project.add_module(ast)
return project
def project_from_files(self,
files,
func_wrapper=astroid_wrapper,
project_name=None,
black_list=None):
"""return a Project from a list of files or modules"""
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = modutils.file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None:
continue
# XXX why is first file defining the project.path ?
project.path = project.path or astroid.file
project.add_module(astroid)
base_name = astroid.name
# recurse in package except if __init__ was explicitly given
if astroid.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in modutils.get_module_files(dirname(astroid.file),
black_list):
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None or astroid.name == base_name:
continue
project.add_module(astroid)
return project
def test_get_module_files_1(self):
package = resources.find('data/find_test')
modules = set(modutils.get_module_files(package, []))
expected = ['__init__.py', 'module.py', 'module2.py',
'noendingnewline.py', 'nonregr.py']
self.assertEqual(modules,
{os.path.join(package, x) for x in expected})
def project_from_files(self, files, func_wrapper=astroid_wrapper,
project_name=None, black_list=None):
"""return a Project from a list of files or modules"""
# build the project representation
project_name = project_name or self.config.project
black_list = black_list or self.config.black_list
project = Project(project_name)
for something in files:
if not exists(something):
fpath = file_from_modpath(something.split('.'))
elif isdir(something):
fpath = join(something, '__init__.py')
else:
fpath = something
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None:
continue
# XXX why is first file defining the project.path ?
project.path = project.path or astroid.file
project.add_module(astroid)
base_name = astroid.name
# recurse in package except if __init__ was explicitly given
if astroid.package and something.find('__init__') == -1:
# recurse on others packages / modules if this is a package
for fpath in get_module_files(dirname(astroid.file),
black_list):
astroid = func_wrapper(self.ast_from_file, fpath)
if astroid is None or astroid.name == base_name:
continue
project.add_module(astroid)
return project
def walk_child_modules(self):
for child_path in modutils.get_module_files(self.file, self.blacklist):
if child_path.endswith("__init__.py"):
continue
orig_file = self.file
self.file = child_path
yield self.load_module()
self.file = orig_file
def test_knownValues_get_module_files_1(self): # XXXFIXME: TOWRITE
"""given a directory return a list of all available python module's files, even
in subdirectories
"""
import data
modules = sorted(modutils.get_module_files(path.join(DATADIR, 'find_test'),
data.__path__[0]))
self.assertEqual(modules,
[path.join(DATADIR, 'find_test', x) for x in ['__init__.py', 'module.py', 'module2.py', 'noendingnewline.py', 'nonregr.py']])
def test_get_all_files(self):
"""test that list_all returns all Python files from given location
"""
non_package = resources.find('data/notamodule')
modules = modutils.get_module_files(non_package, [], list_all=True)
self.assertEqual(
modules,
[os.path.join(non_package, 'file.py')],
)
def test_get_module_files_1(self):
"""given a directory return a list of all available python module's files, even
in subdirectories
"""
package = resources.find('data/find_test')
modules = set(modutils.get_module_files(package, []))
self.assertEqual(
modules,
{os.path.join(package, x) for x in ['__init__.py', 'module.py', 'module2.py', 'noendingnewline.py', 'nonregr.py']})
def test_get_all_files(self):
"""test that list_all returns all Python files from given location
"""
non_package = resources.find('data/notamodule')
modules = modutils.get_module_files(non_package, [], list_all=True)
self.assertEqual(
modules,
[os.path.join(non_package, 'file.py')],
)
def expand_modules(files_or_modules, black_list):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
errors.append({
'key': 'ignored-builtin-module',
'mod': modname
})
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
result.append({
'path': filepath,
'name': modname,
'isarg': True,
'basepath': filepath,
'basename': modname
})
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append({
'path': subfilepath,
'name': submodname,
'isarg': False,
'basepath': filepath,
'basename': modname
})
return result, errors
def test_get_module_files_1(self) -> None:
package = resources.find("data/find_test")
modules = set(modutils.get_module_files(package, []))
expected = [
"__init__.py",
"module.py",
"module2.py",
"noendingnewline.py",
"nonregr.py",
]
self.assertEqual(modules, {os.path.join(package, x) for x in expected})
def test_get_module_files_1(self):
"""given a directory return a list of all available python module's files, even
in subdirectories
"""
package = resources.find('data/find_test')
modules = set(modutils.get_module_files(package, []))
self.assertEqual(
modules, {
os.path.join(package, x)
for x in [
'__init__.py', 'module.py', 'module2.py',
'noendingnewline.py', 'nonregr.py'
]
})
def test_knownValues_get_module_files_1(self): # XXXFIXME: TOWRITE
"""given a directory return a list of all available python module's files, even
in subdirectories
"""
import data
modules = sorted(
modutils.get_module_files(path.join(DATADIR, 'find_test'),
data.__path__[0]))
self.assertEqual(modules, [
path.join(DATADIR, 'find_test', x) for x in [
'__init__.py', 'module.py', 'module2.py', 'noendingnewline.py',
'nonregr.py'
]
])
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
result.append({'path': filepath, 'name': modname, 'isarg': True,
'basepath': filepath, 'basename': modname})
if not (modname.endswith('.__init__') or modname == '__init__') \
and '__init__.py' in filepath:
for subfilepath in get_module_files(dirname(filepath), black_list):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
submodname = '.'.join(modpath_from_file(subfilepath))
result.append({'path': subfilepath, 'name': submodname,
'isarg': False,
'basepath': filepath, 'basename': modname})
return result, errors
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if basename(something) in black_list:
continue
if _basename_in_blacklist_re(basename(something), black_list_re):
continue
if exists(something):
# this is a file or a directory
try:
modname = ".".join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split("."))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({"key": "fatal", "mod": modname, "ex": ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split(".")
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = modutils.is_namespace(spec)
is_directory = modutils.is_directory(spec)
if not is_namespace:
result.append(
{
"path": filepath,
"name": modname,
"isarg": True,
"basepath": filepath,
"basename": modname,
}
)
has_init = (
not (modname.endswith(".__init__") or modname == "__init__")
and basename(filepath) == "__init__.py"
)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
dirname(filepath), black_list, list_all=is_namespace
):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = ".".join(modpath)
result.append(
{
"path": subfilepath,
"name": submodname,
"isarg": False,
"basepath": filepath,
"basename": modname,
}
)
return result, errors
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
path = sys.path.copy()
for something in files_or_modules:
if os.path.basename(something) in black_list:
continue
if _basename_in_blacklist_re(os.path.basename(something),
black_list_re):
continue
module_path = get_python_path(something)
additional_search_path = [".", module_path] + path
if os.path.exists(something):
# this is a file or a directory
try:
modname = ".".join(
modutils.modpath_from_file(something,
path=additional_search_path))
except ImportError:
modname = os.path.splitext(os.path.basename(something))[0]
if os.path.isdir(something):
filepath = os.path.join(something, "__init__.py")
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(
modname.split("."), path=additional_search_path)
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# The SyntaxError is a Python bug and should be
# removed once we move away from imp.find_module: https://bugs.python.org/issue10588
errors.append({"key": "fatal", "mod": modname, "ex": ex})
continue
filepath = os.path.normpath(filepath)
modparts = (modname or something).split(".")
try:
spec = modutils.file_info_from_modpath(modparts,
path=additional_search_path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = os.path.isdir(something)
else:
is_namespace = modutils.is_namespace(spec)
is_directory = modutils.is_directory(spec)
if not is_namespace:
result.append({
"path": filepath,
"name": modname,
"isarg": True,
"basepath": filepath,
"basename": modname,
})
has_init = (
not (modname.endswith(".__init__") or modname == "__init__")
and os.path.basename(filepath) == "__init__.py")
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
os.path.dirname(filepath), black_list,
list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(os.path.basename(subfilepath),
black_list_re):
continue
modpath = _modpath_from_file(subfilepath,
is_namespace,
path=additional_search_path)
submodname = ".".join(modpath)
result.append({
"path": subfilepath,
"name": submodname,
"isarg": False,
"basepath": filepath,
"basename": modname,
})
return result, errors
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split('.')
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = spec.type == modutils.ModuleType.PY_NAMESPACE
is_directory = spec.type == modutils.ModuleType.PKG_DIRECTORY
if not is_namespace:
result.append({
'path': filepath,
'name': modname,
'isarg': True,
'basepath': filepath,
'basename': modname
})
has_init = (
not (modname.endswith('.__init__') or modname == '__init__')
and '__init__.py' in filepath)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(
dirname(filepath), black_list, list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath),
black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = '.'.join(modpath)
result.append({
'path': subfilepath,
'name': submodname,
'isarg': False,
'basepath': filepath,
'basename': modname
})
return result, errors
def expand_modules(files_or_modules, black_list, black_list_re):
"""take a list of files/modules/packages and return the list of tuple
(file, module name) which have to be actually checked
"""
result = []
errors = []
for something in files_or_modules:
if exists(something):
# this is a file or a directory
try:
modname = '.'.join(modutils.modpath_from_file(something))
except ImportError:
modname = splitext(basename(something))[0]
if isdir(something):
filepath = join(something, '__init__.py')
else:
filepath = something
else:
# suppose it's a module or package
modname = something
try:
filepath = modutils.file_from_modpath(modname.split('.'))
if filepath is None:
continue
except (ImportError, SyntaxError) as ex:
# FIXME p3k : the SyntaxError is a Python bug and should be
# removed as soon as possible http://bugs.python.org/issue10588
errors.append({'key': 'fatal', 'mod': modname, 'ex': ex})
continue
filepath = normpath(filepath)
modparts = (modname or something).split('.')
try:
spec = modutils.file_info_from_modpath(modparts, path=sys.path)
except ImportError:
# Might not be acceptable, don't crash.
is_namespace = False
is_directory = isdir(something)
else:
is_namespace = spec.type == modutils.ModuleType.PY_NAMESPACE
is_directory = spec.type == modutils.ModuleType.PKG_DIRECTORY
if not is_namespace:
result.append({'path': filepath, 'name': modname, 'isarg': True,
'basepath': filepath, 'basename': modname})
has_init = (not (modname.endswith('.__init__') or modname == '__init__')
and '__init__.py' in filepath)
if has_init or is_namespace or is_directory:
for subfilepath in modutils.get_module_files(dirname(filepath), black_list,
list_all=is_namespace):
if filepath == subfilepath:
continue
if _basename_in_blacklist_re(basename(subfilepath), black_list_re):
continue
modpath = _modpath_from_file(subfilepath, is_namespace)
submodname = '.'.join(modpath)
result.append({'path': subfilepath, 'name': submodname,
'isarg': False,
'basepath': filepath, 'basename': modname})
return result, errors
