def _LookupClassReferences(serializable_ast, module_map, self_name): """Fills .cls references in serializable_ast.ast with ones from module_map. Already filled references are not changed. References to the module self._name are not filled. Setting self_name=None will fill all references. Args: serializable_ast: A SerializableAst instance. module_map: Used to resolve ClassType.cls links to already loaded modules. The loaded module will be added to the dict. self_name: A string representation of a module which should not be resolved, for example: "foo.bar.module1" or None to resolve all modules. Returns: A SerializableAst with an updated .ast. .class_type_nodes is set to None if any of the Nodes needed to be regenerated. """ class_lookup = visitors.LookupExternalTypes(module_map, self_name=self_name) raw_ast = serializable_ast.ast for node in (serializable_ast.class_type_nodes or ()): try: if node is not class_lookup.VisitClassType(node): serializable_ast = serializable_ast.Replace( class_type_nodes=None) break except KeyError as e: raise UnrestorableDependencyError("Unresolved class: %r." % utils.message(e)) for node in (serializable_ast.function_type_nodes or ()): try: # Use VisitNamedType, even though this is a FunctionType. We want to # do a name lookup, to make sure this is still a function. if not isinstance(class_lookup.VisitNamedType(node), pytd.FunctionType): serializable_ast = serializable_ast.Replace( function_type_nodes=None) break except KeyError as e: raise UnrestorableDependencyError("Unresolved class: %r." % utils.message(e)) if (serializable_ast.class_type_nodes is None or serializable_ast.function_type_nodes is None): try: raw_ast = raw_ast.Visit(class_lookup) except KeyError as e: raise UnrestorableDependencyError("Unresolved class: %r." % utils.message(e)) serializable_ast = serializable_ast.Replace(ast=raw_ast) return serializable_ast
def _verify_pyi(self, pyval, ast_name=None): try: pyval.Visit(visitors.VerifyLookup(ignore_late_types=True)) except ValueError as e: raise BadDependencyError(utils.message(e), ast_name or pyval.name) from e pyval.Visit(visitors.VerifyContainers())
def apply_type_comment(self, state, op, name, value): """If there is a type comment for the op, return its value.""" assert op is self.vm.frame.current_opcode if op.code.co_filename != self.vm.filename: return value if not op.type_comment: return value comment = op.type_comment try: var = self._eval_expr(state.node, self.vm.frame.f_globals, self.vm.frame.f_locals, comment) except EvaluationError as e: self.vm.errorlog.invalid_type_comment(self.vm.frames, comment, details=utils.message(e)) value = self.vm.new_unsolvable(state.node) else: try: typ = abstract_utils.get_atomic_value(var) except abstract_utils.ConversionError: self.vm.errorlog.invalid_type_comment( self.vm.frames, comment, details="Must be constant.") value = self.vm.new_unsolvable(state.node) else: if self.get_type_parameters(typ): self.vm.errorlog.not_supported_yet( self.vm.frames, "using type parameter in type comment") try: value = self.init_annotation(typ, name, self.vm.frames, state.node) except self.LateAnnotationError: value = LateAnnotation(typ, name, self.vm.simple_stack()) return value
def load_file(self, module_name, filename, ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" if not is_pickle(filename): return super(PickledPyiLoader, self).load_file(module_name, filename, ast) existing = self._get_existing_ast(module_name) if existing: # TODO(kramm): When does this happen? return existing loaded_ast = pytd_utils.LoadPickle(filename) # At this point ast.name and module_name could be different. # They are later synced in ProcessAst. dependencies = {d: names for d, names in loaded_ast.dependencies if d != loaded_ast.ast.name} loaded_ast = serialize_ast.EnsureAstName(loaded_ast, module_name, fix=True) self._modules[module_name] = Module(module_name, filename, loaded_ast.ast) self._load_ast_dependencies(dependencies, ast, module_name) try: ast = serialize_ast.ProcessAst(loaded_ast, self._get_module_map()) except serialize_ast.UnrestorableDependencyError as e: del self._modules[module_name] raise BadDependencyError(utils.message(e), module_name) # Mark all the module's late dependencies as explicitly imported. for d, _ in loaded_ast.late_dependencies: if d != loaded_ast.ast.name: self.add_module_prefixes(d) self._modules[module_name].ast = ast self._modules[module_name].pickle = None self._modules[module_name].dirty = False return ast
def _check_arg(self, condition, name, value): if condition: try: condition.check(value) except PreconditionError as e: raise PreconditionError("argument=%s: %s." % (name, utils.message(e)))
def call(self, node, _, args): try: name_var, field_names, field_types = self._getargs(node, args) except abstract_utils.ConversionError: return node, self.vm.convert.unsolvable.to_variable(node) try: name = abstract_utils.get_atomic_python_constant(name_var) except abstract_utils.ConversionError: return node, self.vm.convert.unsolvable.to_variable(node) try: field_names = self._validate_and_rename_args( name, field_names, False) except ValueError as e: self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames, utils.message(e)) return node, self.vm.convert.unsolvable.to_variable(node) annots, late_annots = self.vm.annotations_util.convert_annotations_list( moves.zip(field_names, field_types)) if late_annots: # We currently don't support forward references. Report if we find any, # then continue by using Unsolvable instead. self.vm.errorlog.not_supported_yet( self.vm.frames, "Forward references in typing.NamedTuple") field_types = [ annots.get(field_name, self.vm.convert.unsolvable) for field_name in field_names ] cls_var = self._build_namedtuple(name, field_names, field_types, node) self.vm.trace_classdef(cls_var) return node, cls_var
def check_error(self, src, expected_line, message): """Check that parsing the src raises the expected error.""" with self.assertRaises(parser.ParseError) as e: parser.parse_string(textwrap.dedent(src).lstrip(), options=self.options) self.assertRegex(utils.message(e.exception), re.escape(message)) self.assertEqual(expected_line, e.exception.line)
def load_file(self, module_name, filename, ast=None): """Load (or retrieve from cache) a module and resolve its dependencies.""" if not os.path.splitext(filename)[1].startswith(".pickled"): return super(PickledPyiLoader, self).load_file(module_name, filename, ast) existing = self._get_existing_ast(module_name) if existing: # TODO(kramm): When does this happen? return existing loaded_ast = pytd_utils.LoadPickle(filename) # At this point ast.name and module_name could be different. # They are later synced in ProcessAst. dependencies = [d for d in loaded_ast.dependencies if d != loaded_ast.ast.name] loaded_ast = serialize_ast.EnsureAstName(loaded_ast, module_name, fix=True) self._modules[module_name] = Module(module_name, filename, loaded_ast.ast) self._load_ast_dependencies(dependencies, ast, module_name) try: ast = serialize_ast.ProcessAst(loaded_ast, self._get_module_map()) except serialize_ast.UnrestorableDependencyError as e: del self._modules[module_name] raise BadDependencyError(utils.message(e), module_name) self._modules[module_name].ast = ast self._modules[module_name].pickle = None self._modules[module_name].dirty = False return ast
def _resolve_external_types(self, ast): try: ast = ast.Visit(visitors.LookupExternalTypes( self._get_module_map(), self_name=ast.name)) except KeyError as e: raise BadDependencyError(utils.message(e), ast.name) return ast
def call(self, node, _, args): try: name_var, field_names, field_types = self._getargs(node, args) except abstract_utils.ConversionError: return node, self.vm.new_unsolvable(node) try: name = abstract_utils.get_atomic_python_constant(name_var) except abstract_utils.ConversionError: return node, self.vm.new_unsolvable(node) try: field_names = self._validate_and_rename_args( name, field_names, False) except ValueError as e: self.vm.errorlog.invalid_namedtuple_arg(self.vm.frames, utils.message(e)) return node, self.vm.new_unsolvable(node) annots, late_annots = self.vm.annotations_util.convert_annotations_list( moves.zip(field_names, field_types)) field_types = [ annots.get(field_name, self.vm.convert.unsolvable) for field_name in field_names ] node, cls_var = self._build_namedtuple(name, field_names, field_types, late_annots, node) self.vm.trace_classdef(cls_var) return node, cls_var
def verify(self, mod_ast, *, mod_name=None): try: mod_ast.Visit(visitors.VerifyLookup(ignore_late_types=True)) except ValueError as e: name = mod_name or mod_ast.name raise BadDependencyError(utils.message(e), name) from e mod_ast.Visit(visitors.VerifyContainers())
def main(): argument_parser = make_parser() opts = argument_parser.parse_args() python_version = utils.version_from_string(opts.python_version) try: utils.validate_version(python_version) except utils.UsageError as e: sys.stderr.write("Usage error: %s\n" % utils.message(e)) sys.exit(1) with open(opts.input) as fi: sourcecode = fi.read() try: parsed = parser.parse_string(sourcecode, filename=opts.input, python_version=python_version) except parser.ParseError as e: sys.stderr.write(str(e)) sys.exit(1) if opts.optimize: parsed = optimize.Optimize(parsed, builtins.GetBuiltinsPyTD(python_version), lossy=opts.lossy, use_abcs=opts.use_abcs, max_union=opts.max_union, remove_mutable=opts.remove_mutable, can_do_lookup=False) if opts.output is not None: out_text = pytd_utils.Print(parsed, opts.multiline_args) if opts.output == "-": sys.stdout.write(out_text) else: with open(opts.output, "w") as out: out.write(out_text)
def from_function(cls, function: ast3.AST, is_async: bool) -> "NameAndSig": """Constructor from an ast.FunctionDef node.""" name = function.name # decorators decorators = set(function.decorator_list) abstracts = {"abstractmethod", "abc.abstractmethod"} coroutines = {"coroutine", "asyncio.coroutine", "coroutines.coroutine"} overload = {"overload"} ignored = {"type_check_only"} is_abstract = bool(decorators & abstracts) is_coroutine = bool(decorators & coroutines) is_overload = bool(decorators & overload) decorators -= abstracts decorators -= coroutines decorators -= overload decorators -= ignored # TODO(mdemello): do we need this limitation? if len(decorators) > 1: raise ParseError("Too many decorators for %s" % name) decorator, = decorators if decorators else (None, ) exceptions = [] mutators = [] for i, x in enumerate(function.body): if isinstance(x, types.Raise): exceptions.append(x.exception) elif isinstance(x, Mutator): mutators.append(x) elif isinstance(x, types.Ellipsis): pass elif (isinstance(x, ast3.Expr) and isinstance(x.value, ast3.Str) and i == 0): # docstring pass else: msg = textwrap.dedent(""" Unexpected statement in function body. Only `raise` statements and type mutations are valid """).lstrip() if isinstance(x, ast3.AST): raise ParseError(msg).at(x) else: raise ParseError(msg) # exceptions sig = _pytd_signature(function, is_async, exceptions=exceptions) # mutators for mutator in mutators: try: sig = sig.Visit(mutator) except NotImplementedError as e: raise ParseError(utils.message(e)) from e if not mutator.successful: raise ParseError("No parameter named %s" % mutator.name) return cls(name, sig, decorator, is_abstract, is_coroutine, is_overload)
def parse(self, src, name, filename): """Parse a PYI file and return the corresponding AST. Note that parse() should be called exactly once per _Parser instance. It holds aggregated state during parsing and is not designed to be reused. Args: src: The source text to parse. name: The name of the module to be created. filename: The name of the source file. Returns: A pytd.TypeDeclUnit() representing the parsed pyi. Raises: ParseError: If the PYI source could not be parsed. """ # Ensure instances do not get reused. assert not self._used self._used = True self._filename = filename self._ast_name = name self._type_map = {} try: defs = parser_ext.parse(self, src) ast = self._build_type_decl_unit(defs) except ParseError as e: if self._error_location: line = e.line or self._error_location[0] try: text = src.splitlines()[line-1] except IndexError: text = None raise ParseError(utils.message(e), line=line, filename=self._filename, column=self._error_location[1], text=text) else: raise e ast = ast.Visit(_PropertyToConstant()) ast = ast.Visit(_InsertTypeParameters()) # TODO(kramm): This is in the wrong place- it should happen after resolving # local names, in load_pytd. ast = ast.Visit(pep484.ConvertTypingToNative(name)) if name: ast = ast.Replace(name=name) ast = ast.Visit(visitors.AddNamePrefix()) else: # If there's no unique name, hash the sourcecode. ast = ast.Replace(name=hashlib.md5(src.encode("utf-8")).hexdigest()) ast = ast.Visit(visitors.StripExternalNamePrefix()) # Typeshed files that explicitly import and refer to "builtins" need to have # that rewritten to __builtin__ ast = ast.Visit(visitors.RenameBuiltinsPrefix()) return ast.Replace(is_package=utils.is_pyi_directory_init(filename))
def _resolve_external_types(self, pyval, ast_name=None): name = ast_name or pyval.name try: pyval = pyval.Visit(visitors.LookupExternalTypes( self._get_module_map(), self_name=name, module_alias_map=self._aliases)) except KeyError as e: raise BadDependencyError(utils.message(e), name) return pyval
def resolve_external_types(self, mod_ast, module_map, aliases, *, mod_name=None): name = mod_name or mod_ast.name try: mod_ast = mod_ast.Visit(visitors.LookupExternalTypes( module_map, self_name=name, module_alias_map=aliases)) except KeyError as e: raise BadDependencyError(utils.message(e), name) from e return mod_ast
def call(self, node, _, args): """Call typing.TypeVar().""" try: param = self._get_typeparam(node, args) except TypeVarError as e: self.vm.errorlog.invalid_typevar(self.vm.frames, utils.message(e), e.bad_call) return node, self.vm.convert.unsolvable.to_variable(node) return node, param.to_variable(node)
def check(self, value): errors = [] for c in self._choices: try: c.check(value) return except PreconditionError as e: errors.append(e) raise PreconditionError(" or ".join("(%s)" % utils.message(e) for e in errors))
def call(self, node, _, args, alias_map=None): sig = None if isinstance(self.func.__self__, _classes.CallableClass): sig = function.Signature.from_callable(self.func.__self__) args = args.simplify(node, self.ctx, match_signature=sig) posargs = [u.AssignToNewVariable(node) for u in args.posargs] namedargs = { k: u.AssignToNewVariable(node) for k, u in args.namedargs.items() } try: inspect.signature(self.func).bind(node, *posargs, **namedargs) except ValueError as e: # Happens for, e.g., # def f((x, y)): pass # f((42,)) raise NotImplementedError( "Wrong number of values to unpack") from e except TypeError as e: # The possible errors here are: # (1) wrong arg count # (2) duplicate keyword # (3) unexpected keyword # The way we constructed namedargs rules out (2). if "keyword" in utils.message(e): # Happens for, e.g., # def f(*args): pass # f(x=42) raise NotImplementedError("Unexpected keyword") from e # The function was passed the wrong number of arguments. The signature is # ([self, ]node, ...). The length of "..." tells us how many variables # are expected. expected_argcount = len(inspect.getfullargspec(self.func).args) - 1 if inspect.ismethod(self.func) and self.func.__self__ is not None: expected_argcount -= 1 actual_argcount = len(posargs) + len(namedargs) if (actual_argcount > expected_argcount or (not args.starargs and not args.starstarargs)): # If we have too many arguments, or starargs and starstarargs are both # empty, then we can be certain of a WrongArgCount error. argnames = tuple("_" + str(i) for i in range(expected_argcount)) sig = function.Signature(self.name, argnames, 0, None, set(), None, {}, {}, {}) raise function.WrongArgCount(sig, args, self.ctx) assert actual_argcount < expected_argcount # Assume that starargs or starstarargs fills in the missing arguments. # Instead of guessing where these arguments should go, overwrite all of # the arguments with a list of unsolvables of the correct length, which # is guaranteed to give us a correct (but imprecise) analysis. posargs = [ self.ctx.new_unsolvable(node) for _ in range(expected_argcount) ] namedargs = {} return self.func(node, *posargs, **namedargs)
def call(self, node, _, args): """Call typing.TypeVar().""" try: param = self._get_typeparam(node, args) except TypeVarError as e: self.vm.errorlog.invalid_typevar(self.vm.frames, utils.message(e), e.bad_call) return node, self.vm.new_unsolvable(node) if param.has_late_types(): self.vm.params_with_late_types.append( (param, self.vm.simple_stack())) return node, param.to_variable(node)
def process_file(options, source_text=None, kythe_args=None): """Process a single file and return cross references. Args: options: A dictionary of pytype options. source_text: Optional text of the file; will be read from the file pointed to by options.input if not supplied. kythe_args: Extra args for generating the kythe index Returns: An Indexer object with the indexed code, or None if pytype fails. """ # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = source_text or io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) try: analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return None major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. a = ast27.parse(src, options.input) ast = ast27 else: a = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer(source, vm.loader.get_resolved_modules(), module_name, kythe_args) ix.index(a) ix.finalize() return ix
def __str__(self): lines = [] if self._filename or self._line is not None: lines.append(' File: "%s", line %s' % (self._filename, self._line)) if self._column and self._text: indent = 4 stripped = self._text.lstrip() lines.append("%*s%s" % (indent, "", stripped)) # Output a pointer below the error column, adjusting for stripped spaces. pos = indent + (self._column - 1) - (len(self._text) - len(stripped)) lines.append("%*s^" % (pos, "")) lines.append("%s: %s" % (type(self).__name__, utils.message(self))) return "\n".join(lines)
def check_or_generate_pyi(options, errorlog, loader): """Returns generated errors and result pyi or None if it's only check. Args: options: config.Options object. errorlog: errors.ErrorLog object. loader: load_pytd.Loader object. Returns: A tuple, (PYI Ast as string, AST) or None. """ result = pytd_builtins.DEFAULT_SRC ast = pytd_builtins.GetDefaultAst(options.python_version) try: if options.check: check_py(input_filename=options.input, errorlog=errorlog, options=options, loader=loader) return None else: result, ast = generate_pyi(input_filename=options.input, errorlog=errorlog, options=options, loader=loader) except utils.UsageError as e: raise except pyc.CompileError as e: errorlog.python_compiler_error(options.input, e.lineno, e.error) except IndentationError as e: errorlog.python_compiler_error(options.input, e.lineno, e.msg) except tokenize.TokenError as e: msg, (lineno, unused_column) = e.args # pylint: disable=unbalanced-tuple-unpacking errorlog.python_compiler_error(options.input, lineno, msg) except directors.SkipFile: result += "# skip-file found, file not analyzed" except Exception as e: # pylint: disable=broad-except if options.nofail: log.warn("***Caught exception: %s", str(e), exc_info=True) if not options.check: result += ( # pytype: disable=name-error "# Caught error in pytype: " + str(e).replace("\n", "\n#") + "\n# " + "\n# ".join(traceback.format_exc().splitlines())) else: e.args = (str(utils.message(e)) + "\nFile: %s" % options.input, ) + e.args[1:] raise return (result, ast)
def _parse_source(self, src): """Parse a source file, extracting directives from comments.""" f = moves.StringIO(src) defs_start = None closing_bracket_lines = set() whitespace_lines = set() for tok, _, start, _, line in tokenize.generate_tokens(f.readline): lineno, col = start if defs_start is None and _CLASS_OR_FUNC_RE.match(line): defs_start = lineno if _CLOSING_BRACKETS_RE.match(line): closing_bracket_lines.add(lineno) elif _WHITESPACE_RE.match(line): whitespace_lines.add(lineno) elif _DOCSTRING_RE.match(line): self._docstrings.add(lineno) else: if closing_bracket_lines: self._adjust_type_comments(closing_bracket_lines, whitespace_lines) closing_bracket_lines.clear() whitespace_lines.clear() if tok == tokenize.COMMENT: matches = list(_DIRECTIVE_RE.finditer(line[col:])) is_nested = bool(matches) and matches[0].start(0) > 0 for m in matches: code = line[:col].strip() tool, data = m.groups() open_ended = not code data = data.strip() if tool == "type": self._process_type(lineno, code, data, is_nested) elif tool == "pytype": try: self._process_pytype(lineno, data, open_ended) except _DirectiveError as e: self._errorlog.invalid_directive( self._filename, lineno, utils.message(e)) else: pass # ignore comments for other tools if closing_bracket_lines: self._adjust_type_comments(closing_bracket_lines, whitespace_lines) if defs_start is not None: disables = list(self._disables.items()) # Add "# type: ignore" to the list of disables that we check. disables.append(("Type checking", self._ignore)) for name, lineset in disables: lineno = lineset.get_disable_after(defs_start) if lineno is not None: self._errorlog.late_directive(self._filename, lineno, name)
def process_one_file(options): """Check a .py file or generate a .pyi for it, according to options. Args: options: config.Options object. Returns: An error code (0 means no error). """ log.info("Process %s => %s", options.input, options.output) errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) try: generated_values = check_or_generate_pyi(options, errorlog, loader) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 if not options.check: result, ast = generated_values if options.pickle_output: pyi_output = options.verify_pickle else: pyi_output = options.output if pyi_output == "-": sys.stdout.write(result) elif pyi_output: log.info("write pyi %r => %r", options.input, pyi_output) with open(pyi_output, "w") as fi: fi.write(result) if options.pickle_output: log.info("write pickle %r => %r", options.input, options.output) write_pickle(ast, loader, options) exit_status = handle_errors(errorlog, options) # If we have set return_success, set exit_status to 0 after the regular error # handler has been called. if options.return_success: exit_status = 0 # Touch output file upon success. if options.touch and not exit_status: with open(options.touch, "a"): os.utime(options.touch, None) return exit_status
def _eval_comparison(self, ident, op, value) -> bool: """Evaluate a comparison and return a bool. Args: ident: A tuple of a dotted name string and an optional __getitem__ key (int or slice). op: One of the comparison operator strings in cmp_slots.COMPARES. value: Either a string, an integer, or a tuple of integers. Returns: The boolean result of the comparison. Raises: ParseError: If the comparison cannot be evaluated. """ name, key = ident if name == "sys.version_info": if key is None: key = slice(None, None, None) assert isinstance(key, (int, slice)) if isinstance(key, int) and not isinstance(value, int): raise ParseError( "an element of sys.version_info must be compared to an integer" ) if isinstance(key, slice) and not _is_int_tuple(value): raise ParseError( "sys.version_info must be compared to a tuple of integers") try: actual = self._version[key] except IndexError as e: raise ParseError(utils.message(e)) from e if isinstance(key, slice): actual = _three_tuple(actual) value = _three_tuple(value) elif name == "sys.platform": if not isinstance(value, str): raise ParseError("sys.platform must be compared to a string") valid_cmps = (cmp_slots.EQ, cmp_slots.NE) if op not in valid_cmps: raise ParseError( "sys.platform must be compared using %s or %s" % valid_cmps) actual = self._platform else: raise ParseError("Unsupported condition: '%s'." % name) return cmp_slots.COMPARES[op](actual, value)
def __init__(self, argv): """Parse and encapsulate the command-line options. Also sets up some basic logger configuration. Args: argv: sys.argv[1:] (sys.argv[0] is the main script). Raises: sys.exit(2): bad option or input filenames. """ argument_parser = make_parser() options = argument_parser.parse_args(argv) names = set(vars(options)) try: Postprocessor(names, options, self).process() except PostprocessingError as e: argument_parser.error(utils.message(e))
def process_file(options): """Process a single file and return cross references.""" # We bind the global ast variable in this function. global ast errorlog = errors.ErrorLog() loader = load_pytd.create_loader(options) src = io.read_source_file(options.input) vm = analyze.CallTracer(errorlog=errorlog, options=options, generate_unknowns=options.protocols, store_all_calls=False, loader=loader) try: analyze.infer_types(src=src, filename=options.input, errorlog=errorlog, options=options, loader=loader, tracer_vm=vm) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 major, minor = options.python_version if major == 2: # python2.7 is the only supported py2 version. a = ast27.parse(src, options.input) ast = ast27 else: a = ast3.parse(src, options.input, feature_version=minor) ast = ast3 # TODO(mdemello): Get from args module_name = "module" source = SourceFile(src, vm.opcode_traces, filename=options.input) ix = Indexer(source, module_name) ix.index(a) ix.lookup_refs() ix.process_deflocs() ix.process_links() return ix
def process_one_file(options): """Check a .py file or generate a .pyi for it, according to options. Args: options: config.Options object. Returns: An error code (0 means no error). """ log.info("Process %s => %s", options.input, options.output) loader = load_pytd.create_loader(options) try: errorlog, result, ast = check_or_generate_pyi(options, loader) except utils.UsageError as e: logging.error("Usage error: %s\n", utils.message(e)) return 1 if not options.check: if options.pickle_output: pyi_output = options.verify_pickle else: pyi_output = options.output # Write out the pyi file. if pyi_output: _write_pyi_output(options, result, pyi_output) # Write out the pickle file. if options.pickle_output: log.info("write pickle %r => %r", options.input, options.output) write_pickle(ast, options, loader) exit_status = handle_errors(errorlog, options) # If we have set return_success, set exit_status to 0 after the regular error # handler has been called. if options.return_success: exit_status = 0 # Touch output file upon success. if options.touch and not exit_status: with options.open_function(options.touch, "a"): os.utime(options.touch, None) return exit_status
def _process_comment(self, line, lineno, col, type_comment_set): """Process a single comment.""" matches = list(_DIRECTIVE_RE.finditer(line[col:])) is_nested = bool(matches) and matches[0].start(0) > 0 for m in matches: code = line[:col].strip() tool, data = m.groups() open_ended = not code data = data.strip() if tool == "type": self._process_type(lineno, code, data, is_nested, type_comment_set) elif tool == "pytype": try: self._process_pytype(lineno, data, open_ended) except _DirectiveError as e: self._errorlog.invalid_directive(self._filename, lineno, utils.message(e)) else: pass # ignore comments for other tools