Example #1
0
File: deps.py Project: sixolet/mypy
def get_dependencies(target: MypyFile,
                     type_map: Dict[Expression, Type],
                     python_version: Tuple[int, int]) -> Dict[str, Set[str]]:
    """Get all dependencies of a node, recursively."""
    visitor = DependencyVisitor(type_map, python_version, target.alias_deps)
    target.accept(visitor)
    return visitor.map
Example #2
0
 def strip_file_top_level(self, file_node: MypyFile) -> None:
     """Strip a module top-level (don't recursive into functions)."""
     self.names = file_node.names
     self.file_node = file_node
     self.recurse_into_functions = False
     file_node.plugin_deps.clear()
     file_node.accept(self)
Example #3
0
def merge_asts(old: MypyFile, old_symbols: SymbolTable,
               new: MypyFile, new_symbols: SymbolTable) -> None:
    """Merge a new version of a module AST to a previous version.

    The main idea is to preserve the identities of externally visible
    nodes in the old AST (that have a corresponding node in the new AST).
    All old node state (outside identity) will come from the new AST.

    When this returns, 'old' will refer to the merged AST, but 'new_symbols'
    will be the new symbol table. 'new' and 'old_symbols' will no longer be
    valid.
    """
    assert new.fullname() == old.fullname()
    # Find the mapping from new to old node identities for all nodes
    # whose identities should be preserved.
    replacement_map = replacement_map_from_symbol_table(
        old_symbols, new_symbols, prefix=old.fullname())
    # Also replace references to the new MypyFile node.
    replacement_map[new] = old
    # Perform replacements to everywhere within the new AST (not including symbol
    # tables).
    node = replace_nodes_in_ast(new, replacement_map)
    assert node is old
    # Also replace AST node references in the *new* symbol table (we'll
    # continue to use the new symbol table since it has all the new definitions
    # that have no correspondence in the old AST).
    replace_nodes_in_symbol_table(new_symbols, replacement_map)
Example #4
0
    def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type]) -> None:
        self.last_xml = None
        path = os.path.relpath(tree.path)
        if stats.is_special_module(path):
            return
        if path.startswith('..'):
            return
        if 'stubs' in path.split('/'):
            return

        visitor = stats.StatisticsVisitor(inferred=True, typemap=type_map, all_nodes=True)
        tree.accept(visitor)

        root = etree.Element('mypy-report-file', name=path, module=tree._fullname)
        doc = etree.ElementTree(root)
        file_info = FileInfo(path, tree._fullname)

        with tokenize.open(path) as input_file:
            for lineno, line_text in enumerate(input_file, 1):
                status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
                file_info.counts[status] += 1
                etree.SubElement(root, 'line',
                                 number=str(lineno),
                                 precision=stats.precision_names[status],
                                 content=line_text[:-1])
        # Assumes a layout similar to what XmlReporter uses.
        xslt_path = os.path.relpath('mypy-html.xslt', path)
        transform_pi = etree.ProcessingInstruction('xml-stylesheet',
                'type="text/xsl" href="%s"' % cgi.escape(xslt_path, True))
        root.addprevious(transform_pi)
        self.schema.assertValid(doc)

        self.last_xml = doc
        self.files.append(file_info)
Example #5
0
    def on_file(self,
                tree: MypyFile,
                type_map: Dict[Expression, Type],
                options: Options) -> None:
        path = os.path.relpath(tree.path)
        visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
                                          typemap=type_map, all_nodes=True)
        tree.accept(visitor)

        class_name = os.path.basename(path)
        file_info = FileInfo(path, tree._fullname)
        class_element = etree.Element('class',
                                      filename=path,
                                      complexity='1.0',
                                      name=class_name)
        etree.SubElement(class_element, 'methods')
        lines_element = etree.SubElement(class_element, 'lines')

        with tokenize.open(path) as input_file:
            class_lines_covered = 0
            class_total_lines = 0
            for lineno, _ in enumerate(input_file, 1):
                status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
                hits = 0
                branch = False
                if status == stats.TYPE_EMPTY:
                    continue
                class_total_lines += 1
                if status != stats.TYPE_ANY:
                    class_lines_covered += 1
                    hits = 1
                if status == stats.TYPE_IMPRECISE:
                    branch = True
                file_info.counts[status] += 1
                line_element = etree.SubElement(lines_element, 'line',
                                                number=str(lineno),
                                                precision=stats.precision_names[status],
                                                hits=str(hits),
                                                branch=str(branch).lower())
                if branch:
                    line_element.attrib['condition-coverage'] = '50% (1/2)'
            class_element.attrib['branch-rate'] = '0'
            class_element.attrib['line-rate'] = get_line_rate(class_lines_covered,
                                                              class_total_lines)
            # parent_module is set to whichever module contains this file.  For most files, we want
            # to simply strip the last element off of the module.  But for __init__.py files,
            # the module == the parent module.
            parent_module = file_info.module.rsplit('.', 1)[0]
            if file_info.name.endswith('__init__.py'):
                parent_module = file_info.module

            if parent_module not in self.root_package.packages:
                self.root_package.packages[parent_module] = CoberturaPackage(parent_module)
            current_package = self.root_package.packages[parent_module]
            packages_to_update = [self.root_package, current_package]
            for package in packages_to_update:
                package.total_lines += class_total_lines
                package.covered_lines += class_lines_covered
            current_package.classes[class_name] = class_element
Example #6
0
 def visit_mypy_file(self, node: MypyFile) -> MypyFile:
     # NOTE: The 'names' and 'imports' instance variables will be empty!
     new = MypyFile(self.statements(node.defs), [], node.is_bom,
                    ignored_lines=set(node.ignored_lines))
     new._fullname = node._fullname
     new.path = node.path
     new.names = SymbolTable()
     return new
Example #7
0
 def visit_mypy_file(self, o: MypyFile) -> None:
     self.scope.enter_file(o.fullname())
     self.is_package_init_file = o.is_package_init_file()
     self.add_type_alias_deps(self.scope.current_target())
     for trigger, targets in o.plugin_deps.items():
         self.map.setdefault(trigger, set()).update(targets)
     super().visit_mypy_file(o)
     self.scope.leave()
Example #8
0
 def visit_mypy_file(self, node: MypyFile) -> Node:
     # NOTE: The 'names' and 'imports' instance variables will be empty!
     new = MypyFile(self.nodes(node.defs), [], node.is_bom)
     new._name = node._name
     new._fullname = node._fullname
     new.path = node.path
     new.names = SymbolTable()
     return new
Example #9
0
def find_classes(node: MypyFile) -> Set[str]:
    results = set()  # type: Set[str]

    class ClassTraverser(mypy.traverser.TraverserVisitor):
        def visit_class_def(self, o: ClassDef) -> None:
            results.add(o.name)

    node.accept(ClassTraverser())
    return results
Example #10
0
 def get_python_out_path(self, f: MypyFile) -> str:
     if f.fullname() == '__main__':
         return os.path.join(self.output_dir, basename(f.path))
     else:
         components = f.fullname().split('.')
         if os.path.basename(f.path) == '__init__.py':
             components.append('__init__.py')
         else:
             components[-1] += '.py'
         return os.path.join(self.output_dir, *components)
Example #11
0
def generate_html_report(tree: MypyFile, path: str, type_map: Dict[Expression, Type],
                         output_dir: str) -> None:
    if is_special_module(path):
        return
    # There may be more than one right answer for "what should we do here?"
    # but this is a reasonable one.
    path = os.path.relpath(path)
    if path.startswith('..'):
        return
    visitor = StatisticsVisitor(inferred=True, typemap=type_map, all_nodes=True)
    tree.accept(visitor)
    assert not os.path.isabs(path) and not path.startswith('..')
    # This line is *wrong* if the preceding assert fails.
    target_path = os.path.join(output_dir, 'html', path)
    # replace .py or .pyi with .html
    target_path = os.path.splitext(target_path)[0] + '.html'
    assert target_path.endswith('.html')
    ensure_dir_exists(os.path.dirname(target_path))
    output = []  # type: List[str]
    append = output.append
    append('''\
<html>
<head>
  <style>
    .red { background-color: #faa; }
    .yellow { background-color: #ffa; }
    .white { }
    .lineno { color: #999; }
  </style>
</head>
<body>
<pre>''')
    num_imprecise_lines = 0
    num_lines = 0
    with open(path) as input_file:
        for i, line in enumerate(input_file):
            lineno = i + 1
            status = visitor.line_map.get(lineno, TYPE_PRECISE)
            style_map = {TYPE_PRECISE: 'white',
                         TYPE_IMPRECISE: 'yellow',
                         TYPE_ANY: 'red'}
            style = style_map[status]
            append('<span class="lineno">%4d</span>   ' % lineno +
                   '<span class="%s">%s</span>' % (style,
                                                   cgi.escape(line)))
            if status != TYPE_PRECISE:
                num_imprecise_lines += 1
            if line.strip():
                num_lines += 1
    append('</pre>')
    append('</body></html>')
    with open(target_path, 'w') as output_file:
        output_file.writelines(output)
    target_path = target_path[len(output_dir) + 1:]
    html_files.append((path, target_path, num_lines, num_imprecise_lines))
Example #12
0
 def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
                patches: List[Callable[[], None]]) -> None:
     self.recurse_into_functions = True
     self.errors.set_file(fnam, file_node.fullname())
     self.options = options
     self.sem.options = options
     self.patches = patches
     self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
     self.sem.cur_mod_id = file_node.fullname()
     self.sem.globals = file_node.names
     with experiments.strict_optional_set(options.strict_optional):
         self.accept(file_node)
Example #13
0
    def on_file(self, tree: MypyFile, type_map: Dict[Node, Type]) -> None:
        tree_source = open(tree.path).readlines()

        coverage_visitor = LineCoverageVisitor(tree_source)
        tree.accept(coverage_visitor)

        covered_lines = []
        for line_number, (_, typed) in enumerate(coverage_visitor.lines_covered):
            if typed:
                covered_lines.append(line_number + 1)

        self.lines_covered[os.path.abspath(tree.path)] = covered_lines
Example #14
0
    def on_file(self, tree: MypyFile, type_map: Dict[Node, Type]) -> None:
        physical_lines = len(open(tree.path).readlines())

        func_counter = FuncCounterVisitor()
        tree.accept(func_counter)
        unannotated_funcs, annotated_funcs = func_counter.counts
        total_funcs = annotated_funcs + unannotated_funcs

        imputed_annotated_lines = (physical_lines * annotated_funcs // total_funcs
                                   if total_funcs else physical_lines)

        self.counts[tree._fullname] = (imputed_annotated_lines, physical_lines,
                                       annotated_funcs, total_funcs)
Example #15
0
    def on_file(self, tree: MypyFile, type_map: Dict[Node, Type]) -> None:
        # Count physical lines.  This assumes the file's encoding is a
        # superset of ASCII (or at least uses \n in its line endings).
        physical_lines = len(open(tree.path, "rb").readlines())

        func_counter = FuncCounterVisitor()
        tree.accept(func_counter)
        unannotated_funcs, annotated_funcs = func_counter.counts
        total_funcs = annotated_funcs + unannotated_funcs

        imputed_annotated_lines = physical_lines * annotated_funcs // total_funcs if total_funcs else physical_lines

        self.counts[tree._fullname] = (imputed_annotated_lines, physical_lines, annotated_funcs, total_funcs)
Example #16
0
 def on_file(self,
             tree: MypyFile,
             type_map: Dict[Expression, Type],
             options: Options) -> None:
     visitor = stats.StatisticsVisitor(inferred=True, filename=tree.fullname(),
                                       typemap=type_map, all_nodes=True,
                                       visit_untyped_defs=False)
     tree.accept(visitor)
     self.any_types_counter[tree.fullname()] = visitor.type_of_any_counter
     num_unanalyzed_lines = list(visitor.line_map.values()).count(stats.TYPE_UNANALYZED)
     # count each line of dead code as one expression of type "Any"
     num_any = visitor.num_any_exprs + num_unanalyzed_lines
     num_total = visitor.num_imprecise_exprs + visitor.num_precise_exprs + num_any
     if num_total > 0:
         self.counts[tree.fullname()] = (num_any, num_total)
Example #17
0
 def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type],
             options: Options) -> None:
     visitor = stats.StatisticsVisitor(inferred=True,
                                       filename=tree.fullname(),
                                       typemap=type_map,
                                       all_nodes=True,
                                       visit_untyped_defs=False)
     tree.accept(visitor)
     self.any_types_counter[tree.fullname()] = visitor.type_of_any_counter
     num_unanalyzed_lines = list(visitor.line_map.values()).count(
         stats.TYPE_UNANALYZED)
     # count each line of dead code as one expression of type "Any"
     num_any = visitor.num_any_exprs + num_unanalyzed_lines
     num_total = visitor.num_imprecise_exprs + visitor.num_precise_exprs + num_any
     if num_total > 0:
         self.counts[tree.fullname()] = (num_any, num_total)
def create_indirect_imported_name(
        file_node: MypyFile, module: str, relative: int,
        imported_name: str) -> Optional[SymbolTableNode]:
    """Create symbol table entry for a name imported from another module.

    These entries act as indirect references.
    """
    target_module, ok = correct_relative_import(
        file_node.fullname(), relative, module,
        file_node.is_package_init_file())
    if not ok:
        return None
    target_name = '%s.%s' % (target_module, imported_name)
    link = ImportedName(target_name)
    # Use GDEF since this refers to a module-level definition.
    return SymbolTableNode(GDEF, link)
Example #19
0
def get_all_leaf_targets(file: MypyFile) -> List[TargetInfo]:
    """Return all leaf targets in a symbol table (module-level and methods)."""
    result = []  # type: List[TargetInfo]
    for fullname, node, active_type in file.local_definitions():
        if isinstance(node.node, (FuncDef, OverloadedFuncDef, Decorator)):
            result.append((fullname, node.node, active_type))
    return result
Example #20
0
def add_new_class_for_module(module: MypyFile,
                             name: str,
                             bases: List[Instance],
                             fields: Optional[Dict[str, MypyType]] = None
                             ) -> TypeInfo:
    new_class_unique_name = checker.gen_unique_name(name, module.names)

    # make new class expression
    classdef = ClassDef(new_class_unique_name, Block([]))
    classdef.fullname = module.fullname + '.' + new_class_unique_name

    # make new TypeInfo
    new_typeinfo = TypeInfo(SymbolTable(), classdef, module.fullname)
    new_typeinfo.bases = bases
    calculate_mro(new_typeinfo)
    new_typeinfo.calculate_metaclass_type()

    # add fields
    if fields:
        for field_name, field_type in fields.items():
            var = Var(field_name, type=field_type)
            var.info = new_typeinfo
            var._fullname = new_typeinfo.fullname + '.' + field_name
            new_typeinfo.names[field_name] = SymbolTableNode(MDEF, var, plugin_generated=True)

    classdef.info = new_typeinfo
    module.names[new_class_unique_name] = SymbolTableNode(GDEF, new_typeinfo, plugin_generated=True)
    return new_typeinfo
Example #21
0
def parse(source: Union[str, bytes],
          fnam: str = None,
          errors: Errors = None,
          pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
          custom_typing_module: str = None) -> MypyFile:
    """Parse a source file, without doing any semantic analysis.

    Return the parse tree. If errors is not provided, raise ParseError
    on failure. Otherwise, use the errors object to report parse errors.

    The pyversion (major, minor) argument determines the Python syntax variant.
    """
    is_stub_file = bool(fnam) and fnam.endswith('.pyi')
    try:
        assert pyversion[0] >= 3 or is_stub_file
        ast = ast35.parse(source, fnam, 'exec')

        tree = ASTConverter(
            pyversion=pyversion,
            is_stub=is_stub_file,
            custom_typing_module=custom_typing_module,
        ).visit(ast)
        tree.path = fnam
        tree.is_stub = is_stub_file
        return tree
    except (SyntaxError, TypeCommentParseError) as e:
        if errors:
            errors.set_file('<input>' if fnam is None else fnam)
            errors.report(e.lineno, e.offset, e.msg)
        else:
            raise

    return MypyFile([], [], False, set())
Example #22
0
    def visit_mypy_file(self, mfile: MypyFile) -> int:
        if mfile.fullname() in ('typing', 'abc'):
            # These module are special; their contents are currently all
            # built-in primitives.
            return -1

        self.enter()

        # Initialize non-int global variables.
        for name in sorted(mfile.names):
            node = mfile.names[name].node
            if (isinstance(node, Var)
                    and name not in nodes.implicit_module_attrs):
                v = cast(Var, node)
                if (not is_named_instance(v.type, 'builtins.int')
                        and v.fullname() != 'typing.Undefined'):
                    tmp = self.alloc_register()
                    self.add(SetRNone(tmp))
                    self.add(SetGR(v.fullname(), tmp))

        for d in mfile.defs:
            d.accept(self)
        self.add_implicit_return()
        self.generated['__init'] = FuncIcode(0, self.blocks,
                                             self.register_types)
        # TODO leave?
        return -1
Example #23
0
 def visit_mypy_file(self, mfile: MypyFile) -> int:
     if mfile.fullname() in ('typing', 'abc'):
         # These module are special; their contents are currently all
         # built-in primitives.
         return -1
     
     self.enter()
     
     # Initialize non-int global variables.
     for name in sorted(mfile.names):
         node = mfile.names[name].node
         if (isinstance(node, Var) and
                 name not in nodes.implicit_module_attrs):
             v = cast(Var, node)
             if (not is_named_instance(v.type, 'builtins.int')
                     and v.fullname() != 'typing.Undefined'):
                 tmp = self.alloc_register()
                 self.add(SetRNone(tmp))
                 self.add(SetGR(v.fullname(), tmp))
     
     for d in mfile.defs:
         d.accept(self)
     self.add_implicit_return()
     self.generated['__init'] = FuncIcode(0, self.blocks,
                                          self.register_types)
     # TODO leave?
     return -1
Example #24
0
def parse(source: Union[str, bytes],
          fnam: str,
          module: Optional[str],
          errors: Optional[Errors] = None,
          options: Optional[Options] = None) -> MypyFile:
    """Parse a source file, without doing any semantic analysis.

    Return the parse tree. If errors is not provided, raise ParseError
    on failure. Otherwise, use the errors object to report parse errors.
    """
    raise_on_error = False
    if errors is None:
        errors = Errors()
        raise_on_error = True
    if options is None:
        options = Options()
    errors.set_file(fnam, module)
    is_stub_file = fnam.endswith('.pyi')
    try:
        assert options.python_version[0] < 3 and not is_stub_file
        ast = ast27.parse(source, fnam, 'exec')
        tree = ASTConverter(options=options,
                            errors=errors,
                            ).visit(ast)
        assert isinstance(tree, MypyFile)
        tree.path = fnam
        tree.is_stub = is_stub_file
    except SyntaxError as e:
        errors.report(e.lineno, e.offset, e.msg, blocker=True)
        tree = MypyFile([], [], False, set())

    if raise_on_error and errors.is_errors():
        errors.raise_error()

    return tree
Example #25
0
 def visit_mypy_file(self, o: MypyFile) -> None:
     with self.scope.module_scope(o.fullname):
         self.is_package_init_file = o.is_package_init_file()
         self.add_type_alias_deps(self.scope.current_target())
         for trigger, targets in o.plugin_deps.items():
             self.map.setdefault(trigger, set()).update(targets)
         super().visit_mypy_file(o)
Example #26
0
    def on_file(self,
                tree: MypyFile,
                type_map: Dict[Expression, Type],
                options: Options) -> None:
        with open(tree.path) as f:
            tree_source = f.readlines()

        coverage_visitor = LineCoverageVisitor(tree_source)
        tree.accept(coverage_visitor)

        covered_lines = []
        for line_number, (_, typed) in enumerate(coverage_visitor.lines_covered):
            if typed:
                covered_lines.append(line_number + 1)

        self.lines_covered[os.path.abspath(tree.path)] = covered_lines
Example #27
0
    def on_file(self,
                tree: MypyFile,
                type_map: Dict[Expression, Type],
                options: Options) -> None:
        with open(tree.path) as f:
            tree_source = f.readlines()

        coverage_visitor = LineCoverageVisitor(tree_source)
        tree.accept(coverage_visitor)

        covered_lines = []
        for line_number, (_, typed) in enumerate(coverage_visitor.lines_covered):
            if typed:
                covered_lines.append(line_number + 1)

        self.lines_covered[os.path.abspath(tree.path)] = covered_lines
Example #28
0
def parse(source: Union[str, bytes], fnam: str = None, errors: Errors = None,
          pyversion: Tuple[int, int] = defaults.PYTHON3_VERSION,
          custom_typing_module: str = None) -> MypyFile:
    """Parse a source file, without doing any semantic analysis.

    Return the parse tree. If errors is not provided, raise ParseError
    on failure. Otherwise, use the errors object to report parse errors.

    The pyversion (major, minor) argument determines the Python syntax variant.
    """
    raise_on_error = False
    if errors is None:
        errors = Errors()
        raise_on_error = True
    errors.set_file('<input>' if fnam is None else fnam)
    is_stub_file = bool(fnam) and fnam.endswith('.pyi')
    try:
        assert pyversion[0] < 3 and not is_stub_file
        ast = ast27.parse(source, fnam, 'exec')
        tree = ASTConverter(pyversion=pyversion,
                            is_stub=is_stub_file,
                            errors=errors,
                            custom_typing_module=custom_typing_module,
                            ).visit(ast)
        assert isinstance(tree, MypyFile)
        tree.path = fnam
        tree.is_stub = is_stub_file
    except SyntaxError as e:
        errors.report(e.lineno, e.offset, e.msg)
        tree = MypyFile([], [], False, set())

    if raise_on_error and errors.is_errors():
        errors.raise_error()

    return tree
Example #29
0
def get_all_leaf_targets(file: MypyFile) -> List[TargetInfo]:
    """Return all leaf targets in a symbol table (module-level and methods)."""
    result = []  # type: List[TargetInfo]
    for fullname, node, active_type in file.local_definitions():
        if isinstance(node.node, (FuncDef, OverloadedFuncDef, Decorator)):
            result.append((fullname, node.node, active_type))
    return result
Example #30
0
    def visit_mypy_file(self, mypyfile: MypyFile) -> Register:
        if mypyfile.fullname() in ('typing', 'abc'):
            # These module are special; their contents are currently all
            # built-in primitives.
            return INVALID_REGISTER

        # First pass: Build ClassIRs and TypeInfo-to-ClassIR mapping.
        for node in mypyfile.defs:
            if isinstance(node, ClassDef):
                self.prepare_class_def(node)

        # Second pass: Generate ops.
        self.current_module_name = mypyfile.fullname()
        for node in mypyfile.defs:
            node.accept(self)

        return INVALID_REGISTER
Example #31
0
 def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
                patches: List[Tuple[int, Callable[[], None]]]) -> None:
     self.recurse_into_functions = True
     self.errors.set_file(fnam, file_node.fullname())
     self.options = options
     self.sem.options = options
     self.patches = patches
     self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
     self.sem.cur_mod_id = file_node.fullname()
     self.cur_mod_node = file_node
     self.sem.globals = file_node.names
     with experiments.strict_optional_set(options.strict_optional):
         self.scope.enter_file(file_node.fullname())
         self.accept(file_node)
         self.scope.leave()
     del self.cur_mod_node
     self.patches = []
Example #32
0
    def visit_Module(self, mod: ast27.Module) -> MypyFile:
        body = self.fix_function_overloads(self.translate_stmt_list(mod.body))

        return MypyFile(body,
                        self.imports,
                        False,
                        {ti.lineno for ti in mod.type_ignores},
                        )
Example #33
0
    def visit_Module(self, mod: ast35.Module) -> Node:
        body = self.fix_function_overloads(self.visit_list(mod.body))

        return MypyFile(body,
                        self.imports,
                        False,
                        {ti.lineno for ti in mod.type_ignores},
                        weak_opts=set())
Example #34
0
    def on_file(self, tree: MypyFile, type_map: Dict[Expression,
                                                     Type]) -> None:
        # Count physical lines.  This assumes the file's encoding is a
        # superset of ASCII (or at least uses \n in its line endings).
        physical_lines = len(open(tree.path, 'rb').readlines())

        func_counter = FuncCounterVisitor()
        tree.accept(func_counter)
        unannotated_funcs, annotated_funcs = func_counter.counts
        total_funcs = annotated_funcs + unannotated_funcs

        imputed_annotated_lines = (physical_lines * annotated_funcs //
                                   total_funcs
                                   if total_funcs else physical_lines)

        self.counts[tree._fullname] = (imputed_annotated_lines, physical_lines,
                                       annotated_funcs, total_funcs)
Example #35
0
    def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type],
                options: Options) -> None:
        self.last_xml = None
        path = os.path.relpath(tree.path)
        if stats.is_special_module(path):
            return
        if path.startswith('..'):
            return
        if 'stubs' in path.split('/'):
            return

        visitor = stats.StatisticsVisitor(inferred=True,
                                          filename=tree.fullname(),
                                          typemap=type_map,
                                          all_nodes=True)
        tree.accept(visitor)

        root = etree.Element('mypy-report-file',
                             name=path,
                             module=tree._fullname)
        doc = etree.ElementTree(root)
        file_info = FileInfo(path, tree._fullname)

        with tokenize.open(path) as input_file:
            for lineno, line_text in enumerate(input_file, 1):
                status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
                file_info.counts[status] += 1
                etree.SubElement(root,
                                 'line',
                                 number=str(lineno),
                                 precision=stats.precision_names[status],
                                 content=line_text.rstrip('\n').translate(
                                     self.control_fixer),
                                 any_info=self._get_any_info_for_line(
                                     visitor, lineno))
        # Assumes a layout similar to what XmlReporter uses.
        xslt_path = os.path.relpath('mypy-html.xslt', path)
        transform_pi = etree.ProcessingInstruction(
            'xml-stylesheet',
            'type="text/xsl" href="%s"' % pathname2url(xslt_path))
        root.addprevious(transform_pi)
        self.schema.assertValid(doc)

        self.last_xml = doc
        self.files.append(file_info)
Example #36
0
    def on_file(self, tree: MypyFile, type_map: Dict[Node, Type]) -> None:
        import lxml.etree as etree

        self.last_xml = None
        path = os.path.relpath(tree.path)
        if stats.is_special_module(path):
            return
        if path.startswith('..'):
            return
        if 'stubs' in path.split('/'):
            return

        visitor = stats.StatisticsVisitor(inferred=True,
                                          typemap=type_map,
                                          all_nodes=True)
        tree.accept(visitor)

        root = etree.Element('mypy-report-file',
                             name=path,
                             module=tree._fullname)
        doc = etree.ElementTree(root)
        file_info = FileInfo(path, tree._fullname)

        with open(path) as input_file:
            for lineno, line_text in enumerate(input_file, 1):
                status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
                file_info.counts[status] += 1
                etree.SubElement(root,
                                 'line',
                                 number=str(lineno),
                                 precision=stats.precision_names[status],
                                 content=line_text[:-1])
        # Assumes a layout similar to what XmlReporter uses.
        xslt_path = os.path.relpath('mypy-html.xslt', path)
        xml_pi = etree.ProcessingInstruction('xml',
                                             'version="1.0" encoding="utf-8"')
        transform_pi = etree.ProcessingInstruction(
            'xml-stylesheet',
            'type="text/xsl" href="%s"' % cgi.escape(xslt_path, True))
        root.addprevious(xml_pi)
        root.addprevious(transform_pi)
        self.schema.assertValid(doc)

        self.last_xml = doc
        self.files.append(file_info)
Example #37
0
    def on_file(self, tree: MypyFile, modules: Dict[str, MypyFile],
                type_map: Dict[Expression, Type], options: Options) -> None:
        self.last_xml = None

        try:
            path = os.path.relpath(tree.path)
        except ValueError:
            return

        if should_skip_path(path):
            return

        visitor = stats.StatisticsVisitor(inferred=True,
                                          filename=tree.fullname,
                                          modules=modules,
                                          typemap=type_map,
                                          all_nodes=True)
        tree.accept(visitor)

        root = etree.Element('mypy-report-file',
                             name=path,
                             module=tree._fullname)
        doc = etree.ElementTree(root)
        file_info = FileInfo(path, tree._fullname)

        for lineno, line_text in iterate_python_lines(path):
            status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
            file_info.counts[status] += 1
            etree.SubElement(
                root,
                'line',
                any_info=self._get_any_info_for_line(visitor, lineno),
                content=line_text.rstrip('\n').translate(self.control_fixer),
                number=str(lineno),
                precision=stats.precision_names[status])
        # Assumes a layout similar to what XmlReporter uses.
        xslt_path = os.path.relpath('mypy-html.xslt', path)
        transform_pi = etree.ProcessingInstruction(
            'xml-stylesheet',
            'type="text/xsl" href="%s"' % pathname2url(xslt_path))
        root.addprevious(transform_pi)
        self.schema.assertValid(doc)

        self.last_xml = doc
        self.files.append(file_info)
Example #38
0
 def visit_Module(self, mod: ast27.Module) -> MypyFile:
     self.type_ignores = {ti.lineno: parse_type_ignore_tag(ti.tag)  # type: ignore
                          for ti in mod.type_ignores}
     body = self.fix_function_overloads(self.translate_stmt_list(mod.body))
     return MypyFile(body,
                     self.imports,
                     False,
                     self.type_ignores,
                     )
Example #39
0
 def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
                patches: List[Tuple[int, Callable[[], None]]]) -> None:
     self.recurse_into_functions = True
     self.errors.set_file(fnam, file_node.fullname(), scope=self.scope)
     self.options = options
     self.sem.options = options
     self.patches = patches
     self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
     self.sem.cur_mod_id = file_node.fullname()
     self.cur_mod_node = file_node
     self.sem.globals = file_node.names
     with experiments.strict_optional_set(options.strict_optional):
         self.scope.enter_file(file_node.fullname())
         self.accept(file_node)
         self.analyze_symbol_table(file_node.names)
         self.scope.leave()
     del self.cur_mod_node
     self.patches = []
Example #40
0
 def visit_Module(self, mod: ast27.Module) -> MypyFile:
     body = self.fix_function_overloads(self.translate_stmt_list(mod.body))
     ignores = [ti.lineno for ti in mod.type_ignores]
     ignores.extend(self.extra_type_ignores)
     return MypyFile(body,
                     self.imports,
                     False,
                     set(ignores),
                     )
Example #41
0
 def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
                patches: List[Callable[[], None]]) -> None:
     self.errors.set_file(fnam, file_node.fullname())
     self.options = options
     self.sem.options = options
     self.patches = patches
     self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
     self.sem.globals = file_node.names
     with experiments.strict_optional_set(options.strict_optional):
         self.accept(file_node)
Example #42
0
    def on_file(self, tree: MypyFile, modules: Dict[str, MypyFile],
                type_map: Dict[Expression, Type], options: Options) -> None:
        path = os.path.relpath(tree.path)
        if should_skip_path(path):
            return

        visitor = stats.StatisticsVisitor(inferred=True,
                                          filename=tree.fullname(),
                                          modules=modules,
                                          typemap=type_map,
                                          all_nodes=True)
        tree.accept(visitor)

        file_info = FileInfo(path, tree._fullname)
        for lineno, _ in iterate_python_lines(path):
            status = visitor.line_map.get(lineno, stats.TYPE_EMPTY)
            file_info.counts[status] += 1

        self.files.append(file_info)
Example #43
0
def create_indirect_imported_name(file_node: MypyFile,
                                  module: str,
                                  relative: int,
                                  imported_name: str) -> Optional[SymbolTableNode]:
    """Create symbol table entry for a name imported from another module.

    These entries act as indirect references.
    """
    target_module, ok = correct_relative_import(
        file_node.fullname(),
        relative,
        module,
        file_node.is_package_init_file())
    if not ok:
        return None
    target_name = '%s.%s' % (target_module, imported_name)
    link = ImportedName(target_name)
    # Use GDEF since this refers to a module-level definition.
    return SymbolTableNode(GDEF, link)
Example #44
0
 def visit_file(self, file_node: MypyFile, fnam: str, options: Options,
                patches: List[Tuple[int, Callable[[], None]]]) -> None:
     self.recurse_into_functions = True
     self.options = options
     self.sem.options = options
     self.patches = patches
     self.is_typeshed_file = self.errors.is_typeshed_file(fnam)
     self.sem.cur_mod_id = file_node.fullname()
     self.cur_mod_node = file_node
     self.sem.globals = file_node.names
Example #45
0
def merge_asts(old: MypyFile, old_symbols: SymbolTable,
               new: MypyFile, new_symbols: SymbolTable) -> None:
    """Merge a new version of a module AST to a previous version.

    The main idea is to preserve the identities of externally visible
    nodes in the old AST (that have a corresponding node in the new AST).
    All old node state (outside identity) will come from the new AST.

    When this returns, 'old' will refer to the merged AST, but 'new_symbols'
    will be the new symbol table. 'new' and 'old_symbols' will no longer be
    valid.
    """
    assert new.fullname() == old.fullname()
    replacement_map = replacement_map_from_symbol_table(
        old_symbols, new_symbols, prefix=old.fullname())
    replacement_map[new] = old
    node = replace_nodes_in_ast(new, replacement_map)
    assert node is old
    replace_nodes_in_symbol_table(new_symbols, replacement_map)
Example #46
0
def merge_asts(old: MypyFile, old_symbols: SymbolTable,
               new: MypyFile, new_symbols: SymbolTable) -> None:
    """Merge a new version of a module AST to a previous version.

    The main idea is to preserve the identities of externally visible
    nodes in the old AST (that have a corresponding node in the new AST).
    All old node state (outside identity) will come from the new AST.

    When this returns, 'old' will refer to the merged AST, but 'new_symbols'
    will be the new symbol table. 'new' and 'old_symbols' will no longer be
    valid.
    """
    assert new.fullname() == old.fullname()
    replacement_map = replacement_map_from_symbol_table(
        old_symbols, new_symbols, prefix=old.fullname())
    replacement_map[new] = old
    node = replace_nodes_in_ast(new, replacement_map)
    assert node is old
    replace_nodes_in_symbol_table(new_symbols, replacement_map)
Example #47
0
def dump_type_stats(tree: MypyFile, path: str, inferred: bool = False,
                    typemap: Dict[Node, Type] = None) -> None:
    if is_special_module(path):
        return
    print(path)
    visitor = StatisticsVisitor(inferred, typemap)
    tree.accept(visitor)
    for line in visitor.output:
        print(line)
    print('  ** precision **')
    print('  precise  ', visitor.num_precise)
    print('  imprecise', visitor.num_imprecise)
    print('  any      ', visitor.num_any)
    print('  ** kinds **')
    print('  simple   ', visitor.num_simple)
    print('  generic  ', visitor.num_generic)
    print('  function ', visitor.num_function)
    print('  tuple    ', visitor.num_tuple)
    print('  TypeVar  ', visitor.num_typevar)
    print('  complex  ', visitor.num_complex)
    print('  any      ', visitor.num_any)
Example #48
0
 def visit_mypy_file(self, o: MypyFile) -> None:
     """Transform an file."""
     res = []  # type: List[Node]
     for d in o.defs:
         if isinstance(d, ClassDef):
             self._type_context = d.info
             res.extend(self.type_tf.transform_class_def(d))
             self._type_context = None
         else:
             d.accept(self)
             res.append(d)
     o.defs = res
Example #49
0
File: stats.py Project: python/mypy
def dump_type_stats(tree: MypyFile, path: str, inferred: bool = False,
                    typemap: Optional[Dict[Expression, Type]] = None) -> None:
    if is_special_module(path):
        return
    print(path)
    visitor = StatisticsVisitor(inferred, filename=tree.fullname(), typemap=typemap)
    tree.accept(visitor)
    for line in visitor.output:
        print(line)
    print('  ** precision **')
    print('  precise  ', visitor.num_precise_exprs)
    print('  imprecise', visitor.num_imprecise_exprs)
    print('  any      ', visitor.num_any_exprs)
    print('  ** kinds **')
    print('  simple   ', visitor.num_simple_types)
    print('  generic  ', visitor.num_generic_types)
    print('  function ', visitor.num_function_types)
    print('  tuple    ', visitor.num_tuple_types)
    print('  TypeVar  ', visitor.num_typevar_types)
    print('  complex  ', visitor.num_complex_types)
    print('  any      ', visitor.num_any_types)
Example #50
0
def dump_type_stats(tree: MypyFile, path: str, inferred: bool = False,
                    typemap: Optional[Dict[Expression, Type]] = None) -> None:
    if is_special_module(path):
        return
    print(path)
    visitor = StatisticsVisitor(inferred, filename=tree.fullname(), typemap=typemap)
    tree.accept(visitor)
    for line in visitor.output:
        print(line)
    print('  ** precision **')
    print('  precise  ', visitor.num_precise_exprs)
    print('  imprecise', visitor.num_imprecise_exprs)
    print('  any      ', visitor.num_any_exprs)
    print('  ** kinds **')
    print('  simple   ', visitor.num_simple_types)
    print('  generic  ', visitor.num_generic_types)
    print('  function ', visitor.num_function_types)
    print('  tuple    ', visitor.num_tuple_types)
    print('  TypeVar  ', visitor.num_typevar_types)
    print('  complex  ', visitor.num_complex_types)
    print('  any      ', visitor.num_any_types)
Example #51
0
 def visit_mypy_file(self, o: MypyFile) -> None:
     """Transform an file."""
     res = []  # type: List[Node]
     for d in o.defs:
         if isinstance(d, ClassDef):
             self._type_context = d.info
             res.extend(self.type_tf.transform_class_def(d))
             self._type_context = None
         else:
             d.accept(self)
             res.append(d)
     o.defs = res
Example #52
0
    def get_additional_deps(self,
                            file: MypyFile) -> List[Tuple[int, str, int]]:
        # for settings
        if file.fullname(
        ) == 'django.conf' and self.django_context.django_settings_module:
            return [
                self._new_dependency(
                    self.django_context.django_settings_module)
            ]

        # for values / values_list
        if file.fullname() == 'django.db.models':
            return [
                self._new_dependency('mypy_extensions'),
                self._new_dependency('typing')
            ]

        # for `get_user_model()`
        if self.django_context.settings:
            if (file.fullname() == 'django.contrib.auth' or file.fullname()
                    in {'django.http', 'django.http.request'}):
                auth_user_model_name = self.django_context.settings.AUTH_USER_MODEL
                try:
                    auth_user_module = self.django_context.apps_registry.get_model(
                        auth_user_model_name).__module__
                except LookupError:
                    # get_user_model() model app is not installed
                    return []
                return [self._new_dependency(auth_user_module)]

        # ensure that all mentioned to='someapp.SomeModel' are loaded with corresponding related Fields
        defined_model_classes = self.django_context.model_modules.get(
            file.fullname())
        if not defined_model_classes:
            return []
        deps = set()
        for model_class in defined_model_classes:
            # forward relations
            for field in self.django_context.get_model_fields(model_class):
                if isinstance(field, RelatedField):
                    related_model_cls = self.django_context.get_field_related_model_cls(
                        field)
                    if related_model_cls is None:
                        continue
                    related_model_module = related_model_cls.__module__
                    if related_model_module != file.fullname():
                        deps.add(self._new_dependency(related_model_module))
            # reverse relations
            for relation in model_class._meta.related_objects:
                related_model_cls = self.django_context.get_field_related_model_cls(
                    relation)
                related_model_module = related_model_cls.__module__
                if related_model_module != file.fullname():
                    deps.add(self._new_dependency(related_model_module))
        return list(deps)
Example #53
0
    def on_file(self, tree: MypyFile, type_map: Dict[Expression, Type],
                options: Options) -> None:
        # Count physical lines.  This assumes the file's encoding is a
        # superset of ASCII (or at least uses \n in its line endings).
        with open(tree.path, 'rb') as f:
            physical_lines = len(f.readlines())

        func_counter = FuncCounterVisitor()
        tree.accept(func_counter)
        unannotated_funcs, annotated_funcs = func_counter.counts
        total_funcs = annotated_funcs + unannotated_funcs

        # Don't count lines or functions as annotated if they have their errors ignored.
        if options.ignore_errors:
            annotated_funcs = 0

        imputed_annotated_lines = (physical_lines * annotated_funcs //
                                   total_funcs
                                   if total_funcs else physical_lines)

        self.counts[tree._fullname] = (imputed_annotated_lines, physical_lines,
                                       annotated_funcs, total_funcs)
Example #54
0
 def visit_mypy_file(self, node: MypyFile) -> Node:
     # NOTE: The 'names' and 'imports' instance variables will be empty!
     new = MypyFile(self.nodes(node.defs), [], node.is_bom)
     new._name = node._name
     new._fullname = node._fullname
     new.path = node.path
     new.names = SymbolTable()
     return new
Example #55
0
    def on_file(self,
                tree: MypyFile,
                type_map: Dict[Expression, Type],
                options: Options) -> None:
        # Count physical lines.  This assumes the file's encoding is a
        # superset of ASCII (or at least uses \n in its line endings).
        with open(tree.path, 'rb') as f:
            physical_lines = len(f.readlines())

        func_counter = FuncCounterVisitor()
        tree.accept(func_counter)
        unannotated_funcs, annotated_funcs = func_counter.counts
        total_funcs = annotated_funcs + unannotated_funcs

        # Don't count lines or functions as annotated if they have their errors ignored.
        if options.ignore_errors:
            annotated_funcs = 0

        imputed_annotated_lines = (physical_lines * annotated_funcs // total_funcs
                                   if total_funcs else physical_lines)

        self.counts[tree._fullname] = (imputed_annotated_lines, physical_lines,
                                       annotated_funcs, total_funcs)
Example #56
0
 def mypyfile(self, node: MypyFile) -> MypyFile:
     new = node.accept(self)
     assert isinstance(new, MypyFile)
     new.set_line(node.line)
     return new
Example #57
0
 def get_additional_deps(self, file: MypyFile) -> List[Tuple[int, str, int]]:
     if file.fullname() == '__main__':
         return [(10, 'err', -1)]
     return []
Example #58
0
 def visit_mypy_file(self, f: MypyFile) -> None:
     self.is_typing = f.fullname() == 'typing' or f.fullname() == 'builtins'
     super().visit_mypy_file(f)
Example #59
0
File: deps.py Project: sixolet/mypy
 def visit_mypy_file(self, o: MypyFile) -> None:
     self.scope.enter_file(o.fullname())
     self.is_package_init_file = o.is_package_init_file()
     self.add_type_alias_deps(self.scope.current_target())
     super().visit_mypy_file(o)
     self.scope.leave()