def test_from_import_object(self):
# foo_a => from foo.foo_c import obj_c
modset = ModuleSet([FOO.init, FOO.a, FOO.b, FOO.c])
got = modset.get_imports(modset.by_name['foo.foo_a'])
assert len(got) == 2
assert FOO.b in got # doesnt matter for this test
assert FOO.c in got
def test_return_module_name(self):
# foo_a => import bar
modset = ModuleSet([FOO.a, BAR])
got = modset.get_imports(modset.by_name['foo.foo_a'], return_fqn=True)
name = got.pop()
assert len(got) == 0
assert name == 'bar'
def test_init_no_packge(self):
# if a module of a package is added but no __init__.py
# its packages is not added to the list of packages
modset = ModuleSet([FOO.a])
assert 0 == len(modset.pkgs)
assert 1 == len(modset.by_path)
assert modset.by_path[FOO.a].fqn == ['foo', 'foo_a']
def test_init_with_packge(self):
modset = ModuleSet([FOO.init, FOO.a])
assert set(['foo']) == modset.pkgs
assert 2 == len(modset.by_path)
assert modset.by_path[FOO.init].fqn == ['foo', '__init__']
assert modset.by_path[FOO.a].fqn == ['foo', 'foo_a']
assert 2 == len(modset.by_name)
assert modset.by_name['foo.__init__'].fqn == ['foo', '__init__']
assert modset.by_name['foo.foo_a'].fqn == ['foo', 'foo_a']
def task_imports():
"""find imports from a python module"""
base_path = pathlib.Path("youtube_dl_gui")
pkg_modules = ModuleSet(base_path.glob("**/*.py"))
for name, module in pkg_modules.by_name.items():
yield {
"name": name,
"file_dep": [module.path],
"actions": [(get_imports, (pkg_modules, module.path))],
}
def task_imports():
"""find imports from a python module"""
base_path = pathlib.Path('projects/requests/requests')
pkg_modules = ModuleSet(base_path.glob('**/*.py'))
for name, module in pkg_modules.by_name.items():
yield {
'name': name,
'file_dep': [module.path],
'actions': [(get_imports, (pkg_modules, module.path))],
}
def test_relative_parent(self):
# foo.sub.sub_a => from .. import foo_d
modset = ModuleSet([FOO.init, FOO.d, SUB.init, SUB.a])
got = modset.get_imports(modset.by_name['foo.sub.sub_a'])
assert len(got) == 1
assert FOO.d in got
def get_imports(module_path):
module = PyModule(module_path)
base_path = module.pkg_path().resolve()
mset = ModuleSet(base_path.glob('**/*.py'))
imports = mset.get_imports(module, return_fqn=True)
return {'modules': list(sorted(imports))}
def get_imports(module_path):
module = PyModule(module_path)
base_path = module.pkg_path().resolve()
mset = ModuleSet(base_path.glob('**/*.py'))
imports = mset.get_imports(module, return_fqn=True)
return {'modules': list(sorted(imports))}
def test_import_not_tracked(self):
modset = ModuleSet([FOO.a])
got = modset.get_imports(modset.by_name['foo.foo_a'])
assert len(got) == 0
import pathlib
import pygraphviz
from import_deps import PyModule, ModuleSet
DOIT_CONFIG = {
'default_tasks': ['imports', 'dot', 'draw'],
}
base_path = pathlib.Path('projects/requests/requests')
PKG_MODULES = ModuleSet(base_path.glob('**/*.py'))
def get_imports(pkg_modules, module_path):
module = pkg_modules.by_path[module_path]
imports = pkg_modules.get_imports(module, return_fqn=True)
return {'modules': list(sorted(imports))}
def task_imports():
"""find imports from a python module"""
for name, module in PKG_MODULES.by_name.items():
yield {
'name': name,
'file_dep': [module.path],
'actions': [(get_imports, (PKG_MODULES, module.path))],
}
def test_mod_imports(self):
# foo_a => import bar
modset = ModuleSet([FOO.init, FOO.a, FOO.b, FOO.c, BAR])
got = modset.mod_imports('foo.foo_a')
imports = list(sorted(got))
assert imports == ['bar', 'foo.foo_b', 'foo.foo_c']
def test_import_pkg(self):
# bar => import foo
modset = ModuleSet([FOO.init, BAR])
got = modset.get_imports(modset.by_name['bar'])
assert len(got) == 1
assert FOO.init in got
class PyTasks(object):
"""generate doit tasks related to python modules import dependencies
:ivar ModuleSet py_mods:
:ivar py_files: (list - str) files being watched for changes
:ivar json_file str: name of intermediate file with import info from all
modules
"""
def __init__(self, py_files, json_file='deps.json'):
self.json_file = json_file
self.py_files = list(set(py_files))
self.py_mods = ModuleSet(self.py_files)
self._graph = None # DepGraph cached on first use
def create_graph(self):
"""create Graph from json file"""
with open(self.json_file) as fp:
deps = json.load(fp)
return DepGraph(deps)
@property
def graph(self):
"""cache graph object"""
if self._graph is None:
self._graph = self.create_graph()
return self._graph
def action_get_dep(self, module_path):
"""action: return list of direct imports from a single py module
:return dict: single value 'imports', value set of str file paths
"""
mod = self.py_mods.by_path[module_path]
return {'imports': list(str(s) for s in self.py_mods.get_imports(mod))}
def action_write_json_deps(self, imports):
"""write JSON file with direct imports of all modules"""
result = {k: v['imports'] for k, v in imports.items()}
with open(self.json_file, 'w') as fp:
json.dump(result, fp)
def gen_deps(self):
"""generate doit tasks to find imports
generated tasks:
* get_dep:<path> => find imported moudules
* dep-json => save import info in a JSON file
"""
watched_modules = str(list(sorted(self.py_files)))
for mod in self.py_files:
# direct dependencies
yield {
'basename': 'get_dep',
'name': mod,
'actions': [(self.action_get_dep, [mod])],
'file_dep': [mod],
'uptodate': [config_changed(watched_modules)],
}
# Create an intermediate json file with import information.
# It is required to create an intermediate file because DelayedTasks
# can not have get_args to use values from other tasks.
yield {
'basename': 'dep-json',
'actions': [self.action_write_json_deps],
'task_dep': ['get_dep'],
'getargs': {
'imports': ('get_dep', None)
},
'targets': [self.json_file],
'doc': 'save dep info in {}'.format(self.json_file),
}
@staticmethod
def action_print_dependencies(node):
'''print a node's name and its dependencies to SDTOUT'''
node_list = sorted(n.name for n in node.all_deps())
node_path = os.path.relpath(node.name)
deps_path = (os.path.relpath(p) for p in node_list)
print(' - {}: {}'.format(node_path, ', '.join(deps_path)))
@gen_after(name='print-deps', after_task='dep-json')
def gen_print_deps(self):
'''create tasks for printing node info to STDOUT'''
for node in self.graph.nodes.values():
yield {
'basename': 'print-deps',
'name': node.name,
'actions': [(self.action_print_dependencies, [node])],
'verbosity': 2,
}
@staticmethod
def action_write_dot(file_name, graph):
"""write a dot-file(graphviz) with import relation of modules"""
with open(file_name, "w") as fp:
graph.write_dot(fp)
@gen_after(name='dep-dot', after_task='dep-json')
def gen_dep_graph_dot(self, dot_file='deps.dot'):
"""generate tasks for creating a `dot` graph of module imports"""
yield {
'basename': 'dep-dot',
'actions': [(self.action_write_dot, ['deps.dot', self.graph])],
'file_dep': [self.json_file],
'targets': [dot_file],
}
@gen_after(name='dep-image', after_task='dep-json')
def gen_dep_graph_image(self, dot_file='deps.dot', img_file='deps.svg'):
# generate SVG with bottom-up tree
dot_cmd = 'dot -Tsvg '
yield {
'basename': 'dep-image',
'actions': [dot_cmd + " -o %(targets)s %(dependencies)s"],
'file_dep': [dot_file],
'targets': [img_file],
}
def test_import_module(self):
# foo_a => import bar
modset = ModuleSet([FOO.a, BAR])
got = modset.get_imports(modset.by_name['foo.foo_a'])
assert len(got) == 1
assert BAR in got
def test_relative_intra_import_module(self):
# foo_d => from . import foo_c
modset = ModuleSet([FOO.init, FOO.c, FOO.d])
got = modset.get_imports(modset.by_name['foo.foo_d'])
assert len(got) == 1
assert FOO.c in got
def test_relative_intra_import_pkg_obj(self):
# foo_c => from . import foo_i
modset = ModuleSet([FOO.init, FOO.c])
got = modset.get_imports(modset.by_name['foo.foo_c'])
assert len(got) == 1
assert FOO.init in got
def test_import_obj(self):
# foo_b => import baz.obj_baz
modset = ModuleSet([FOO.b, BAZ])
got = modset.get_imports(modset.by_name['foo.foo_b'])
assert len(got) == 1
assert BAZ in got
def test_from_pkg_import_obj(self):
# baz => from foo import obj_1
modset = ModuleSet([FOO.init, BAZ])
got = modset.get_imports(modset.by_name['baz'])
assert len(got) == 1
assert FOO.init in got
def test_from_pkg_import_module(self):
# foo_a => from foo import foo_b
modset = ModuleSet([FOO.init, FOO.a, FOO.b])
got = modset.get_imports(modset.by_name['foo.foo_a'])
assert len(got) == 1
assert FOO.b in got
def __init__(self, py_files, json_file='deps.json'):
self.json_file = json_file
self.py_files = list(set(py_files))
self.py_mods = ModuleSet(self.py_files)
self._graph = None # DepGraph cached on first use
def test_init_subpackge(self):
modset = ModuleSet([FOO.init, SUB.init, SUB.a])
assert set(['foo', 'foo.sub']) == modset.pkgs
assert 3 == len(modset.by_path)
assert modset.by_path[SUB.a].fqn == ['foo', 'sub', 'sub_a']