def get_dependencies_of_target(
module_id: str, module_tree: MypyFile, target: Node,
type_map: Dict[Expression, Type],
python_version: Tuple[int, int]) -> Dict[str, Set[str]]:
"""Get dependencies of a target -- don't recursive into nested targets."""
# TODO: Add tests for this function.
visitor = DependencyVisitor(type_map, python_version,
module_tree.alias_deps)
visitor.scope.enter_file(module_id)
if isinstance(target, MypyFile):
# Only get dependencies of the top-level of the module. Don't recurse into
# functions.
for defn in target.defs:
# TODO: Recurse into top-level statements and class bodies but skip functions.
if not isinstance(
defn, (ClassDef, Decorator, FuncDef, OverloadedFuncDef)):
defn.accept(visitor)
elif isinstance(target, FuncBase) and target.info:
# It's a method.
# TODO: Methods in nested classes.
visitor.scope.enter_class(target.info)
target.accept(visitor)
visitor.scope.leave()
else:
target.accept(visitor)
visitor.scope.leave()
return visitor.map
def get_dependencies_of_target(module_id: str,
module_tree: MypyFile,
target: Node,
type_map: Dict[Expression, Type],
python_version: Tuple[int, int]) -> Dict[str, Set[str]]:
"""Get dependencies of a target -- don't recursive into nested targets."""
# TODO: Add tests for this function.
visitor = DependencyVisitor(type_map, python_version, module_tree.alias_deps)
visitor.scope.enter_file(module_id)
if isinstance(target, MypyFile):
# Only get dependencies of the top-level of the module. Don't recurse into
# functions.
for defn in target.defs:
# TODO: Recurse into top-level statements and class bodies but skip functions.
if not isinstance(defn, (ClassDef, Decorator, FuncDef, OverloadedFuncDef)):
defn.accept(visitor)
elif isinstance(target, FuncBase) and target.info:
# It's a method.
# TODO: Methods in nested classes.
visitor.scope.enter_class(target.info)
target.accept(visitor)
visitor.scope.leave()
else:
target.accept(visitor)
visitor.scope.leave()
return visitor.map
def generate_html_report(tree: Node, path: str, type_map: Dict[Node, Type],
output_dir: str) -> None:
if is_special_module(path):
return
# There may be more than one right answer for "what should we do here?"
# but this is a reasonable one.
path = os.path.relpath(path)
if path.startswith('..'):
return
visitor = StatisticsVisitor(inferred=True,
typemap=type_map,
all_nodes=True)
tree.accept(visitor)
assert not os.path.isabs(path) and not path.startswith('..')
# This line is *wrong* if the preceding assert fails.
target_path = os.path.join(output_dir, 'html', path)
# replace .py or .pyi with .html
target_path = os.path.splitext(target_path)[0] + '.html'
assert target_path.endswith('.html')
ensure_dir_exists(os.path.dirname(target_path))
output = [] # type: List[str]
append = output.append
append('''\
<html>
<head>
<style>
.red { background-color: #faa; }
.yellow { background-color: #ffa; }
.white { }
.lineno { color: #999; }
</style>
</head>
<body>
<pre>''')
num_imprecise_lines = 0
num_lines = 0
with open(path) as input_file:
for i, line in enumerate(input_file):
lineno = i + 1
status = visitor.line_map.get(lineno, TYPE_PRECISE)
style_map = {
TYPE_PRECISE: 'white',
TYPE_IMPRECISE: 'yellow',
TYPE_ANY: 'red'
}
style = style_map[status]
append('<span class="lineno">%4d</span> ' % lineno +
'<span class="%s">%s</span>' % (style, cgi.escape(line)))
if status != TYPE_PRECISE:
num_imprecise_lines += 1
if line.strip():
num_lines += 1
append('</pre>')
append('</body></html>')
with open(target_path, 'w') as output_file:
output_file.writelines(output)
target_path = target_path[len(output_dir) + 1:]
html_files.append((path, target_path, num_lines, num_imprecise_lines))
def find_classes(node: Node) -> Set[str]:
results = set() # type: Set[str]
class ClassTraverser(mypy.traverser.TraverserVisitor):
def visit_class_def(self, o: ClassDef) -> None:
results.add(o.name)
node.accept(ClassTraverser())
return results
def generate_html_report(tree: Node, path: str, type_map: Dict[Node, Type],
output_dir: str) -> None:
if is_special_module(path):
return
# There may be more than one right answer for "what should we do here?"
# but this is a reasonable one.
path = os.path.relpath(path)
if path.startswith('..'):
return
visitor = StatisticsVisitor(inferred=True, typemap=type_map, all_nodes=True)
tree.accept(visitor)
assert not os.path.isabs(path) and not path.startswith('..')
# This line is *wrong* if the preceding assert fails.
target_path = os.path.join(output_dir, 'html', path)
# replace .py or .pyi with .html
target_path = os.path.splitext(target_path)[0] + '.html'
assert target_path.endswith('.html')
ensure_dir_exists(os.path.dirname(target_path))
output = [] # type: List[str]
append = output.append
append('''\
<html>
<head>
<style>
.red { background-color: #faa; }
.yellow { background-color: #ffa; }
.white { }
.lineno { color: #999; }
</style>
</head>
<body>
<pre>''')
num_imprecise_lines = 0
num_lines = 0
with open(path) as input_file:
for i, line in enumerate(input_file):
lineno = i + 1
status = visitor.line_map.get(lineno, TYPE_PRECISE)
style_map = {TYPE_PRECISE: 'white',
TYPE_IMPRECISE: 'yellow',
TYPE_ANY: 'red'}
style = style_map[status]
append('<span class="lineno">%4d</span> ' % lineno +
'<span class="%s">%s</span>' % (style,
cgi.escape(line)))
if status != TYPE_PRECISE:
num_imprecise_lines += 1
if line.strip():
num_lines += 1
append('</pre>')
append('</body></html>')
with open(target_path, 'w') as output_file:
output_file.writelines(output)
target_path = target_path[len(output_dir) + 1:]
html_files.append((path, target_path, num_lines, num_imprecise_lines))
def get_dependencies_of_target(prefix: str, node: Node,
type_map: Dict[Expression, Type]) -> Dict[str, Set[str]]:
"""Get dependencies of a target -- don't recursive into nested targets."""
visitor = DependencyVisitor(prefix, type_map)
if isinstance(node, MypyFile):
for defn in node.defs:
if not isinstance(defn, (ClassDef, FuncDef)):
defn.accept(visitor)
else:
node.accept(visitor)
return visitor.map
def generate_html_report(tree: Node, path: str, type_map: Dict[Node, Type],
output_dir: str) -> None:
if is_special_module(path):
return
visitor = StatisticsVisitor(inferred=True,
typemap=type_map,
all_nodes=True)
tree.accept(visitor)
target_path = os.path.join(output_dir, 'html', path)
target_path = re.sub(r'\.py$', '.html', target_path)
ensure_dir_exists(os.path.dirname(target_path))
output = [] # type: List[str]
append = output.append
append('''\
<html>
<head>
<style>
.red { background-color: #faa; }
.yellow { background-color: #ffa; }
.white { }
.lineno { color: #999; }
</style>
</head>
<body>
<pre>''')
num_imprecise_lines = 0
num_lines = 0
with open(path) as input_file:
for i, line in enumerate(input_file):
lineno = i + 1
status = visitor.line_map.get(lineno, TYPE_PRECISE)
style_map = {
TYPE_PRECISE: 'white',
TYPE_IMPRECISE: 'yellow',
TYPE_ANY: 'red'
}
style = style_map[status]
append('<span class="lineno">%4d</span> ' % lineno +
'<span class="%s">%s</span>' % (style, cgi.escape(line)))
if status != TYPE_PRECISE:
num_imprecise_lines += 1
if line.strip():
num_lines += 1
append('</pre>')
append('</body></html>')
with open(target_path, 'w') as output_file:
output_file.writelines(output)
target_path = target_path[len(output_dir) + 1:]
html_files.append((path, target_path, num_lines, num_imprecise_lines))
def get_file(manager, node: Node, mypy_file: MypyFile) -> Optional[str]:
# print(f'looking for {type(node)}')
if isinstance(node, MypyFile):
return node.path
mypy_files = [mypy_file]
if isinstance(node, Var):
tup = lookup_fully_qualified(node.fullname(), manager.modules)
if tup is None:
# print('Var not found in modules')
return None
else:
var, mod = tup
if var.node == node:
return mod.path
else:
# print(f'Found var but not identical. Found type is {short_type(var.node)}')
if mod != mypy_file:
mypy_files.append(mod)
# Search in current file first because the definition is usually in the same file.
mypy_files.extend(
[f for f in manager.modules.values() if f not in mypy_files])
if isinstance(node, TypeInfo):
node = node.defn
finder = NodeFinder(node)
for file in mypy_files:
# print('looking in %s' % file.path)
file.accept(finder)
if finder.found:
return file.path
return None
def accept(self,
node: Node,
target: Register = INVALID_REGISTER) -> Register:
self.targets.append(target)
actual = node.accept(self)
self.targets.pop()
return actual
def generate_html_report(tree: Node, path: str, type_map: Dict[Node, Type],
output_dir: str) -> None:
if is_special_module(path):
return
visitor = StatisticsVisitor(inferred=True, typemap=type_map, all_nodes=True)
tree.accept(visitor)
target_path = os.path.join(output_dir, 'html', path)
target_path = re.sub(r'\.py$', '.html', target_path)
ensure_dir_exists(os.path.dirname(target_path))
output = [] # type: List[str]
append = output.append
append('''\
<html>
<head>
<style>
.red { background-color: #faa; }
.yellow { background-color: #ffa; }
.white { }
.lineno { color: #999; }
</style>
</head>
<body>
<pre>''')
num_imprecise_lines = 0
num_lines = 0
with open(path) as input_file:
for i, line in enumerate(input_file):
lineno = i + 1
status = visitor.line_map.get(lineno, TYPE_PRECISE)
style_map = { TYPE_PRECISE: 'white',
TYPE_IMPRECISE: 'yellow',
TYPE_ANY: 'red' }
style = style_map[status]
append('<span class="lineno">%4d</span> ' % lineno +
'<span class="%s">%s</span>' % (style,
cgi.escape(line)))
if status != TYPE_PRECISE:
num_imprecise_lines += 1
if line.strip():
num_lines += 1
append('</pre>')
append('</body></html>')
with open(target_path, 'w') as output_file:
output_file.writelines(output)
target_path = target_path[len(output_dir) + 1:]
html_files.append((path, target_path, num_lines, num_imprecise_lines))
def dump_type_stats(tree: Node, path: str, inferred: bool = False, typemap: Dict[Node, Type] = None) -> None:
if is_special_module(path):
return
print(path)
visitor = StatisticsVisitor(inferred, typemap)
tree.accept(visitor)
for line in visitor.output:
print(line)
print(" ** precision **")
print(" precise ", visitor.num_precise)
print(" imprecise", visitor.num_imprecise)
print(" any ", visitor.num_any)
print(" ** kinds **")
print(" simple ", visitor.num_simple)
print(" generic ", visitor.num_generic)
print(" function ", visitor.num_function)
print(" tuple ", visitor.num_tuple)
print(" typevar ", visitor.num_typevar)
print(" complex ", visitor.num_complex)
print(" any ", visitor.num_any)
def dump_type_stats(tree: Node, path: str, inferred: bool = False,
typemap: Dict[Node, Type] = None) -> None:
if is_special_module(path):
return
print(path)
visitor = StatisticsVisitor(inferred, typemap)
tree.accept(visitor)
for line in visitor.output:
print(line)
print(' ** precision **')
print(' precise ', visitor.num_precise)
print(' imprecise', visitor.num_imprecise)
print(' any ', visitor.num_any)
print(' ** kinds **')
print(' simple ', visitor.num_simple)
print(' generic ', visitor.num_generic)
print(' function ', visitor.num_function)
print(' tuple ', visitor.num_tuple)
print(' TypeVar ', visitor.num_typevar)
print(' complex ', visitor.num_complex)
print(' any ', visitor.num_any)
def all_yield_expressions(node: Node) -> List[Tuple[YieldExpr, bool]]:
v = YieldCollector()
node.accept(v)
return v.yield_expressions
def accept(self, node: Node) -> None:
try:
node.accept(self)
except Exception as err:
report_internal_error(err, self.errors.file, node.line,
self.errors, self.options)
def get_dependencies(prefix: str, node: Node,
type_map: Dict[Expression, Type]) -> Dict[str, Set[str]]:
"""Get all dependencies of a node, recursively."""
visitor = DependencyVisitor(prefix, type_map)
node.accept(visitor)
return visitor.map
def all_return_statements(node: Node) -> List[ReturnStmt]:
v = ReturnCollector()
node.accept(v)
return v.return_statements
def get_subexpressions(node: Node) -> List[Expression]:
visitor = SubexpressionFinder()
node.accept(visitor)
return visitor.expressions
def node(self, n: Node) -> None:
n.accept(self)
def node(self, node: Node) -> Node:
new = node.accept(self)
new.set_line(node.line)
return new
def accept(self, n: Node, target: int = -1) -> int:
self.targets.append(target)
actual = n.accept(self)
self.targets.pop()
return actual
def perform_transform(self, node: Node, transform: Callable[[Type], Type]) -> None:
"""Apply transform to all types associated with node."""
if isinstance(node, ForStmt):
if node.index_type:
node.index_type = transform(node.index_type)
self.transform_types_in_lvalue(node.index, transform)
if isinstance(node, WithStmt):
node.analyzed_types = [transform(typ) for typ in node.analyzed_types]
for n in node.target:
if isinstance(n, NameExpr) and isinstance(n.node, Var) and n.node.type:
n.node.type = transform(n.node.type)
if isinstance(node, (FuncDef, OverloadedFuncDef, CastExpr, AssignmentStmt,
TypeAliasExpr, Var)):
assert node.type, "Scheduled patch for non-existent type"
node.type = transform(node.type)
if isinstance(node, TypeAlias):
node.target = transform(node.target)
if isinstance(node, NewTypeExpr):
assert node.old_type, "Scheduled patch for non-existent type"
node.old_type = transform(node.old_type)
if node.info:
new_bases = [] # type: List[Instance]
for b in node.info.bases:
new_b = transform(b)
# TODO: this code can be combined with code in second pass.
if isinstance(new_b, Instance):
new_bases.append(new_b)
elif isinstance(new_b, TupleType):
new_bases.append(new_b.partial_fallback)
else:
self.fail("Argument 2 to NewType(...) must be subclassable"
" (got {})".format(new_b), node)
new_bases.append(self.builtin_type('object'))
node.info.bases = new_bases
if isinstance(node, TypeVarExpr):
if node.upper_bound:
node.upper_bound = transform(node.upper_bound)
if node.values:
node.values = [transform(v) for v in node.values]
if isinstance(node, TypedDictExpr):
assert node.info.typeddict_type, "Scheduled patch for non-existent type"
node.info.typeddict_type = cast(TypedDictType,
transform(node.info.typeddict_type))
if isinstance(node, NamedTupleExpr):
assert node.info.tuple_type, "Scheduled patch for non-existent type"
node.info.tuple_type = cast(TupleType,
transform(node.info.tuple_type))
if isinstance(node, TypeApplication):
node.types = [transform(t) for t in node.types]
if isinstance(node, TypeInfo):
for tvar in node.defn.type_vars:
if tvar.upper_bound:
tvar.upper_bound = transform(tvar.upper_bound)
if tvar.values:
tvar.values = [transform(v) for v in tvar.values]
new_bases = []
for base in node.bases:
new_base = transform(base)
if isinstance(new_base, Instance):
new_bases.append(new_base)
else:
# Don't fix the NamedTuple bases, they are Instance's intentionally.
# Patch the 'args' just in case, although generic tuple types are
# not supported yet.
alt_base = Instance(base.type, [transform(a) for a in base.args])
new_bases.append(alt_base)
node.bases = new_bases
if node.tuple_type:
new_tuple_type = transform(node.tuple_type)
assert isinstance(new_tuple_type, TupleType)
node.tuple_type = new_tuple_type
def perform_transform(self, node: Node, transform: Callable[[Type],
Type]) -> None:
"""Apply transform to all types associated with node."""
if isinstance(node, ForStmt):
if node.index_type:
node.index_type = transform(node.index_type)
self.transform_types_in_lvalue(node.index, transform)
if isinstance(node, WithStmt):
node.analyzed_types = [
transform(typ) for typ in node.analyzed_types
]
for n in node.target:
if isinstance(n, NameExpr) and isinstance(n.node,
Var) and n.node.type:
n.node.type = transform(n.node.type)
if isinstance(node, (FuncDef, OverloadedFuncDef, CastExpr,
AssignmentStmt, TypeAliasExpr, Var)):
assert node.type, "Scheduled patch for non-existent type"
node.type = transform(node.type)
if isinstance(node, TypeAlias):
node.target = transform(node.target)
if isinstance(node, NewTypeExpr):
assert node.old_type, "Scheduled patch for non-existent type"
node.old_type = transform(node.old_type)
if node.info:
new_bases = [] # type: List[Instance]
for b in node.info.bases:
new_b = transform(b)
# TODO: this code can be combined with code in second pass.
if isinstance(new_b, Instance):
new_bases.append(new_b)
elif isinstance(new_b, TupleType):
new_bases.append(new_b.partial_fallback)
else:
self.fail(
"Argument 2 to NewType(...) must be subclassable"
" (got {})".format(new_b), node)
new_bases.append(self.builtin_type('object'))
node.info.bases = new_bases
if isinstance(node, TypeVarExpr):
if node.upper_bound:
node.upper_bound = transform(node.upper_bound)
if node.values:
node.values = [transform(v) for v in node.values]
if isinstance(node, TypedDictExpr):
assert node.info.typeddict_type, "Scheduled patch for non-existent type"
node.info.typeddict_type = cast(
TypedDictType, transform(node.info.typeddict_type))
if isinstance(node, NamedTupleExpr):
assert node.info.tuple_type, "Scheduled patch for non-existent type"
node.info.tuple_type = cast(TupleType,
transform(node.info.tuple_type))
if isinstance(node, TypeApplication):
node.types = [transform(t) for t in node.types]
if isinstance(node, TypeInfo):
for tvar in node.defn.type_vars:
if tvar.upper_bound:
tvar.upper_bound = transform(tvar.upper_bound)
if tvar.values:
tvar.values = [transform(v) for v in tvar.values]
new_bases = []
for base in node.bases:
new_base = transform(base)
if isinstance(new_base, Instance):
new_bases.append(new_base)
else:
# Don't fix the NamedTuple bases, they are Instance's intentionally.
# Patch the 'args' just in case, although generic tuple types are
# not supported yet.
alt_base = Instance(base.type,
[transform(a) for a in base.args])
new_bases.append(alt_base)
node.bases = new_bases
if node.tuple_type:
new_tuple_type = transform(node.tuple_type)
assert isinstance(new_tuple_type, TupleType)
node.tuple_type = new_tuple_type
def node(self, n: Node) -> None:
n.accept(self)
def node(self, node: Node) -> Node:
new = node.accept(self)
new.set_line(node.line)
return new
def accept(self, n: Node, target: int = -1) -> int:
self.targets.append(target)
actual = n.accept(self)
self.targets.pop()
return actual
def accept(self, node: Node) -> None:
try:
node.accept(self)
except Exception as err:
report_internal_error(err, self.errors.file, node.line, self.errors, self.options)
def get_subexpressions(node: Node) -> List[Expression]:
visitor = SubexpressionFinder()
node.accept(visitor)
return visitor.expressions