def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: # The file system cache we create gets passed off to # BuildManager, and thence to FineGrainedBuildManager, which # assumes responsibility for clearing it after updates. fscache = FileSystemCache(self.options.python_version) self.fswatcher = FileSystemWatcher(fscache) self.update_sources(sources) try: result = mypy.build.build(sources=sources, options=self.options, fscache=fscache, alt_lib_path=self.alt_lib_path) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) changed, removed = self.find_changed(sources) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) # Run an update messages = self.fine_grained_manager.update(changed, removed) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() fscache.flush() status = 1 if messages else 0 return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
def create_source_list(paths: Sequence[str], options: Options, fscache: Optional[FileSystemCache] = None, allow_empty_dir: bool = False) -> List[BuildSource]: """From a list of source files/directories, makes a list of BuildSources. Raises InvalidSourceList on errors. """ fscache = fscache or FileSystemCache() finder = SourceFinder(fscache, options) sources = [] for path in paths: path = os.path.normpath(path) if path.endswith(PY_EXTENSIONS): # Can raise InvalidSourceList if a directory doesn't have a valid module name. name, base_dir = finder.crawl_up(path) sources.append(BuildSource(path, name, None, base_dir)) elif fscache.isdir(path): sub_sources = finder.find_sources_in_dir(path) if not sub_sources and not allow_empty_dir: raise InvalidSourceList( "There are no .py[i] files in directory '{}'".format(path)) sources.extend(sub_sources) else: mod = os.path.basename( path) if options.scripts_are_modules else None sources.append(BuildSource(path, mod, None)) return sources
def create_source_list(files: Sequence[str], options: Options, fscache: Optional[FileSystemCache] = None, allow_empty_dir: bool = False) -> List[BuildSource]: """From a list of source files/directories, makes a list of BuildSources. Raises InvalidSourceList on errors. """ fscache = fscache or FileSystemCache() finder = SourceFinder(fscache) targets = [] for f in files: if f.endswith(PY_EXTENSIONS): # Can raise InvalidSourceList if a directory doesn't have a valid module name. targets.append(BuildSource(f, finder.crawl_up(f), None)) elif fscache.isdir(f): sub_targets = finder.expand_dir(f) if not sub_targets and not allow_empty_dir: raise InvalidSourceList("There are no .py[i] files in directory '{}'" .format(f)) targets.extend(sub_targets) else: mod = os.path.basename(f) if options.scripts_are_modules else None targets.append(BuildSource(f, mod, None)) return targets
def fine_grained_increment(self, sources: List[BuildSource]) -> Dict[str, Any]: assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() self.update_sources(sources) changed, removed = self.find_changed(sources) # TODO: Why create a new FileSystemCache rather than using self.fscache? manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir, FileSystemCache()) t1 = time.time() messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log( "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s". format(t1 - t0, t2 - t1)) status = 1 if messages else 0 self.previous_sources = sources return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status }
def __init__(self, options: Options, status_file: str, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(status_file): os.unlink(status_file) self.fscache = FileSystemCache() options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True self.status_file = status_file
def test_filter_out_missing_top_level_packages(self) -> None: with tempfile.TemporaryDirectory() as td: self.make_file(td, 'base/a/') self.make_file(td, 'base/b.py') self.make_file(td, 'base/c.pyi') self.make_file(td, 'base/missing.txt') self.make_file(td, 'typeshed/d.pyi') self.make_file(td, 'typeshed/@python2/e') self.make_file(td, 'pkg1/f-stubs') self.make_file(td, 'pkg2/g-python2-stubs') self.make_file(td, 'mpath/sub/long_name/') def makepath(p: str) -> str: return os.path.join(td, p) search = SearchPaths(python_path=(makepath('base'), ), mypy_path=(makepath('mpath/sub'), ), package_path=(makepath('pkg1'), makepath('pkg2')), typeshed_path=(makepath('typeshed'), )) fscache = FileSystemCache() res = filter_out_missing_top_level_packages( { 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'long_name', 'ff', 'missing' }, search, fscache) assert res == {'a', 'b', 'c', 'd', 'e', 'f', 'g', 'long_name'}
def get_sources(fscache: FileSystemCache, modules: Dict[str, str], changed_modules: List[Tuple[str, str]]) -> List[BuildSource]: sources = [] for id, path in changed_modules: if fscache.isfile(path): sources.append(BuildSource(path, id, None)) return sources
def get_site_packages_dirs( python_executable: Optional[str], fscache: FileSystemCache) -> Tuple[List[str], List[str]]: """Find package directories for given python. This runs a subprocess call, which generates a list of the egg directories, and the site package directories. To avoid repeatedly calling a subprocess (which can be slow!) we lru_cache the results.""" def make_abspath(path: str, root: str) -> str: """Take a path and make it absolute relative to root if not already absolute.""" if os.path.isabs(path): return os.path.normpath(path) else: return os.path.join(root, os.path.normpath(path)) if python_executable is None: return [], [] if python_executable == sys.executable: # Use running Python's package dirs site_packages = sitepkgs.getsitepackages() else: # Use subprocess to get the package directory of given Python # executable site_packages = ast.literal_eval( subprocess.check_output([python_executable, sitepkgs.__file__], stderr=subprocess.PIPE).decode()) egg_dirs = [] for dir in site_packages: pth = os.path.join(dir, 'easy-install.pth') if fscache.isfile(pth): with open(pth) as f: egg_dirs.extend( [make_abspath(d.rstrip(), dir) for d in f.readlines()]) return egg_dirs, site_packages
def initialize_fine_grained(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: self.fscache = FileSystemCache(self.options.python_version) self.fswatcher = FileSystemWatcher(self.fscache) self.update_sources(sources) if not self.options.use_fine_grained_cache: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() try: # TODO: alt_lib_path result = mypy.build.build(sources=sources, options=self.options) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors manager = result.manager graph = result.graph self.fine_grained_manager = mypy.server.update.FineGrainedBuildManager(manager, graph) self.fine_grained_initialized = True self.previous_sources = sources self.fscache.flush() # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if self.options.use_fine_grained_cache: # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for meta, mypyfile, type_map in manager.saved_cache.values(): if meta.mtime is None: continue self.fswatcher.set_file_data( mypyfile.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) # Run an update changed = self.find_changed(sources) if changed: messages = self.fine_grained_manager.update(changed) self.fscache.flush() status = 1 if messages else 0 self.previous_messages = messages[:] return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status}
def verify_module(fscache: FileSystemCache, id: str, path: str) -> bool: """Check that all packages containing id have a __init__ file.""" if path.endswith(('__init__.py', '__init__.pyi')): path = os.path.dirname(path) for i in range(id.count('.')): path = os.path.dirname(path) if not any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension))) for extension in PYTHON_EXTENSIONS): return False return True
def mypyc_build( paths: List[str], compiler_options: CompilerOptions, *, separate: Union[bool, List[Tuple[List[str], Optional[str]]]] = False, only_compile_paths: Optional[Iterable[str]] = None, skip_cgen_input: Optional[Any] = None, always_use_shared_lib: bool = False ) -> Tuple[emitmodule.Groups, List[Tuple[List[str], List[str]]]]: """Do the front and middle end of mypyc building, producing and writing out C source.""" fscache = FileSystemCache() mypyc_sources, all_sources, options = get_mypy_config( paths, only_compile_paths, compiler_options, fscache) # We generate a shared lib if there are multiple modules or if any # of the modules are in package. (Because I didn't want to fuss # around with making the single module code handle packages.) use_shared_lib = (len(mypyc_sources) > 1 or any('.' in x.module for x in mypyc_sources) or always_use_shared_lib) groups = construct_groups(mypyc_sources, separate, use_shared_lib) # We let the test harness just pass in the c file contents instead # so that it can do a corner-cutting version without full stubs. if not skip_cgen_input: group_cfiles, ops_text = generate_c(all_sources, options, groups, fscache, compiler_options=compiler_options) # TODO: unique names? write_file(os.path.join(compiler_options.target_dir, 'ops.txt'), ops_text) else: group_cfiles = skip_cgen_input # Write out the generated C and collect the files for each group # Should this be here?? group_cfilenames = [] # type: List[Tuple[List[str], List[str]]] for cfiles in group_cfiles: cfilenames = [] for cfile, ctext in cfiles: cfile = os.path.join(compiler_options.target_dir, cfile) write_file(cfile, ctext) if os.path.splitext(cfile)[1] == '.c': cfilenames.append(cfile) deps = [ os.path.join(compiler_options.target_dir, dep) for dep in get_header_deps(cfiles) ] group_cfilenames.append((cfilenames, deps)) return groups, group_cfilenames
def highest_init_level(fscache: FileSystemCache, id: str, path: str) -> int: """Compute the highest level where an __init__ file is found.""" if path.endswith(('__init__.py', '__init__.pyi')): path = os.path.dirname(path) level = 0 for i in range(id.count('.')): path = os.path.dirname(path) if any(fscache.isfile_case(os.path.join(path, '__init__{}'.format(extension))) for extension in PYTHON_EXTENSIONS): level = i + 1 return level
def __init__(self, search_paths: SearchPaths, fscache: Optional[FileSystemCache] = None, options: Optional[Options] = None) -> None: self.search_paths = search_paths self.fscache = fscache or FileSystemCache() # Cache find_lib_path_dirs: (dir_chain, search_paths) -> list(package_dirs, should_verify) self.dirs = {} # type: Dict[Tuple[str, Tuple[str, ...]], PackageDirs] # Cache find_module: id -> result self.results = {} # type: Dict[str, Optional[str]] self.options = options
def matches_exclude(subpath: str, exclude: str, fscache: FileSystemCache, verbose: bool) -> bool: if not exclude: return False subpath_str = os.path.relpath(subpath).replace(os.sep, "/") if fscache.isdir(subpath): subpath_str += "/" if re.search(exclude, subpath_str): if verbose: print("TRACE: Excluding {}".format(subpath_str), file=sys.stderr) return True return False
def __init__(self, search_paths: SearchPaths, fscache: Optional[FileSystemCache], options: Optional[Options]) -> None: self.search_paths = search_paths self.fscache = fscache or FileSystemCache() # Cache for get_toplevel_possibilities: # search_paths -> (toplevel_id -> list(package_dirs)) self.initial_components = { } # type: Dict[Tuple[str, ...], Dict[str, List[str]]] # Cache find_module: id -> result self.results = {} # type: Dict[str, ModuleSearchResult] self.ns_ancestors = {} # type: Dict[str, str] self.options = options
def __init__(self, options: Options, timeout: Optional[int] = None, alt_lib_path: Optional[str] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options self.timeout = timeout self.alt_lib_path = alt_lib_path self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(STATUS_FILE): os.unlink(STATUS_FILE) self.fscache = FileSystemCache(self.options.python_version) options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: options.cache_fine_grained = True # set this so that cache options match else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True
def matches_exclude(subpath: str, excludes: List[str], fscache: FileSystemCache, verbose: bool) -> bool: if not excludes: return False subpath_str = os.path.relpath(subpath).replace(os.sep, "/") if fscache.isdir(subpath): subpath_str += "/" for exclude in excludes: if re.search(exclude, subpath_str): if verbose: print(f"TRACE: Excluding {subpath_str} (matches pattern {exclude})", file=sys.stderr) return True return False
def refresh_suppressed_submodules(module: str, path: Optional[str], deps: Dict[str, Set[str]], graph: Graph, fscache: FileSystemCache) -> None: """Look for submodules that are now suppressed in target package. If a submodule a.b gets added, we need to mark it as suppressed in modules that contain "from a import b". Previously we assumed that 'a.b' is not a module but a regular name. This is only relevant when following imports normally. Args: module: target package in which to look for submodules path: path of the module """ if path is None or not path.endswith(INIT_SUFFIXES): # Only packages have submodules. return # Find any submodules present in the directory. pkgdir = os.path.dirname(path) for fnam in fscache.listdir(pkgdir): if (not fnam.endswith(('.py', '.pyi')) or fnam.startswith("__init__.") or fnam.count('.') != 1): continue shortname = fnam.split('.')[0] submodule = module + '.' + shortname trigger = make_trigger(submodule) if trigger in deps: for dep in deps[trigger]: # TODO: <...> deps, etc. state = graph.get(dep) if not state: # Maybe it's a non-top-level target. We only care about the module. dep_module = module_prefix(graph, dep) if dep_module is not None: state = graph.get(dep_module) if state: tree = state.tree assert tree # TODO: What if doesn't exist? for imp in tree.imports: if isinstance(imp, ImportFrom): if (imp.id == module and any(name == shortname for name, _ in imp.names)): # TODO: Only if does not exist already state.suppressed.append(submodule) state.suppressed_set.add(submodule)
def __init__(self, search_paths: SearchPaths, fscache: Optional[FileSystemCache], options: Optional[Options]) -> None: self.search_paths = search_paths self.fscache = fscache or FileSystemCache() # Cache for get_toplevel_possibilities: # search_paths -> (toplevel_id -> list(package_dirs)) self.initial_components = {} # type: Dict[Tuple[str, ...], Dict[str, List[str]]] # Cache find_module: id -> result self.results = {} # type: Dict[str, ModuleSearchResult] self.ns_ancestors = {} # type: Dict[str, str] self.options = options custom_typeshed_dir = None if options: custom_typeshed_dir = options.custom_typeshed_dir self.stdlib_py_versions = load_stdlib_py_versions(custom_typeshed_dir) self.python2 = options and options.python_version[0] == 2
def _make_manager(self) -> BuildManager: errors = Errors() options = Options() fscache = FileSystemCache() manager = BuildManager( data_dir='', lib_path=[], ignore_prefix='', source_set=BuildSourceSet([]), reports=Reports('', {}), options=options, version_id=__version__, plugin=Plugin(options), errors=errors, flush_errors=lambda msgs, serious: None, fscache=fscache, ) return manager
def _make_manager(self) -> BuildManager: errors = Errors() options = Options() fscache = FileSystemCache() search_paths = SearchPaths((), (), (), ()) manager = BuildManager( data_dir='', search_paths=search_paths, ignore_prefix='', source_set=BuildSourceSet([]), reports=Reports('', {}), options=options, version_id=__version__, plugin=Plugin(options), plugins_snapshot={}, errors=errors, flush_errors=lambda msgs, serious: None, fscache=fscache, stdout=sys.stdout, stderr=sys.stderr, ) return manager
def mypy_test(self): """ Make sure that the generated python typechecks successfully """ messages = [] def flush_errors(new_messages, serious): messages.extend(new_messages) options = mypy.main.Options() options.allow_untyped_globals = True mypy.main.build.build( [mypy.main.BuildSource(path=self.test_file, module="")], options, None, flush_errors, FileSystemCache(), sys.stdout, sys.stderr, ) for m in messages: print(m) self.assertFalse(messages)
def __init__(self, search_paths: SearchPaths, fscache: Optional[FileSystemCache], options: Optional[Options], stdlib_py_versions: Optional[StdlibVersions] = None, source_set: Optional[BuildSourceSet] = None) -> None: self.search_paths = search_paths self.source_set = source_set self.fscache = fscache or FileSystemCache() # Cache for get_toplevel_possibilities: # search_paths -> (toplevel_id -> list(package_dirs)) self.initial_components: Dict[Tuple[str, ...], Dict[str, List[str]]] = {} # Cache find_module: id -> result self.results: Dict[str, ModuleSearchResult] = {} self.ns_ancestors: Dict[str, str] = {} self.options = options custom_typeshed_dir = None if options: custom_typeshed_dir = options.custom_typeshed_dir self.stdlib_py_versions = ( stdlib_py_versions or load_stdlib_py_versions(custom_typeshed_dir) ) self.python_major_ver = 3 if options is None else options.python_version[0]
def filter_out_missing_top_level_packages(packages: Set[str], search_paths: SearchPaths, fscache: FileSystemCache) -> Set[str]: """Quickly filter out obviously missing top-level packages. Return packages with entries that can't be found removed. This is approximate: some packages that aren't actually valid may be included. However, all potentially valid packages must be returned. """ # Start with a empty set and add all potential top-level packages. found = set() paths = ( search_paths.python_path + search_paths.mypy_path + search_paths.package_path + search_paths.typeshed_path ) paths += tuple(os.path.join(p, '@python2') for p in search_paths.typeshed_path) for p in paths: try: entries = fscache.listdir(p) except Exception: entries = [] for entry in entries: # The code is hand-optimized for mypyc since this may be somewhat # performance-critical. if entry.endswith('.py'): entry = entry[:-3] elif entry.endswith('.pyi'): entry = entry[:-4] elif entry.endswith('-stubs'): # Possible PEP 561 stub package entry = entry[:-6] if entry.endswith('-python2'): entry = entry[:-8] if entry in packages: found.add(entry) return found
def initialize_fine_grained( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: # The file system cache we create gets passed off to # BuildManager, and thence to FineGrainedBuildManager, which # assumes responsibility for clearing it after updates. fscache = FileSystemCache(self.options.python_version) self.fswatcher = FileSystemWatcher(fscache) self.update_sources(sources) try: result = mypy.build.build(sources=sources, options=self.options, fscache=fscache, alt_lib_path=self.alt_lib_path) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) changed, removed = self.find_changed(sources) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) # Run an update messages = self.fine_grained_manager.update(changed, removed) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() fscache.flush() status = 1 if messages else 0 return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status }
class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, options: Options, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(STATUS_FILE): os.unlink(STATUS_FILE) self.fscache = FileSystemCache() options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" try: sock = self.create_listening_socket() if self.timeout is not None: sock.settimeout(self.timeout) try: with open(STATUS_FILE, 'w') as f: json.dump( { 'pid': os.getpid(), 'sockname': sock.getsockname() }, f) f.write('\n') # I like my JSON with trailing newline while True: try: conn, addr = sock.accept() except socket.timeout: print("Exiting due to inactivity.") reset_global_state() sys.exit(0) try: data = receive(conn) except OSError as err: conn.close() # Maybe the client hung up continue resp = {} # type: Dict[str, Any] if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') try: resp = self.run_command(command, data) except Exception: # If we are crashing, report the crash to the client tb = traceback.format_exception( *sys.exc_info()) resp = { 'error': "Daemon crashed!\n" + "".join(tb) } conn.sendall(json.dumps(resp).encode('utf8')) raise try: conn.sendall(json.dumps(resp).encode('utf8')) except OSError as err: pass # Maybe the client hung up conn.close() if command == 'stop': sock.close() reset_global_state() sys.exit(0) finally: os.unlink(STATUS_FILE) finally: shutil.rmtree(self.sock_directory) exc_info = sys.exc_info() if exc_info[0] and exc_info[0] is not SystemExit: traceback.print_exception(*exc_info) def create_listening_socket(self) -> socket.socket: """Create the socket and set it up for listening.""" self.sock_directory = tempfile.mkdtemp() sockname = os.path.join(self.sock_directory, SOCKET_NAME) sock = socket.socket(socket.AF_UNIX) sock.bind(sockname) sock.listen(1) return sock def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': "Unrecognized command '%s'" % command} else: return method(self, **data) # Command functions (run in the server via RPC). def cmd_status(self) -> Dict[str, object]: """Return daemon status.""" res = {} # type: Dict[str, object] res.update(get_meminfo()) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" return {} last_sources = None # type: List[mypy.build.BuildSource] def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" try: self.last_sources, options = mypy.main.process_options( ['-i'] + list(args), require_targets=True, server_options=True, fscache=self.fscache) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} if __version__ != version: return {'restart': 'mypy version changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(self.last_sources) def cmd_check(self, files: Sequence[str]) -> Dict[str, object]: """Check a list of files.""" try: self.last_sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(self.last_sources) def cmd_recheck(self) -> Dict[str, object]: """Check the same list of files we checked most recently.""" if not self.last_sources: return { 'error': "Command 'recheck' is only valid after a 'check' command" } return self.check(self.last_sources) def check(self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: """Check using fine-grained incremental mode.""" if not self.fine_grained_manager: res = self.initialize_fine_grained(sources) else: res = self.fine_grained_increment(sources) self.fscache.flush() return res def initialize_fine_grained( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) self.update_sources(sources) try: result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) changed, removed = self.find_changed(sources) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) # Run an update messages = self.fine_grained_manager.update(changed, removed) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile(run_gc=False) status = 1 if messages else 0 return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def fine_grained_increment( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() self.update_sources(sources) changed, removed = self.find_changed(sources) manager.lib_path = tuple( mypy.build.compute_lib_path(sources, manager.options, manager.data_dir)) t1 = time.time() messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log( "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s". format(t1 - t0, t2 - t1)) status = 1 if messages else 0 self.previous_sources = sources return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def update_sources(self, sources: List[mypy.build.BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] self.fswatcher.add_watched_paths(paths) def find_changed( self, sources: List[mypy.build.BuildSource] ) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]: changed_paths = self.fswatcher.find_changed() # Find anything that has been added or modified changed = [(source.module, source.path) for source in sources if source.path in changed_paths] # Now find anything that has been removed from the build modules = {source.module for source in sources} omitted = [ source for source in self.previous_sources if source.module not in modules ] removed = [] for source in omitted: path = source.path assert path removed.append((source.module, path)) # Find anything that has had its module path change because of added or removed __init__s last = {s.path: s.module for s in self.previous_sources} for s in sources: assert s.path if s.path in last and last[s.path] != s.module: # Mark it as removed from its old name and changed at its new name removed.append((last[s.path], s.path)) changed.append((s.module, s.path)) return changed, removed def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {}
class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, options: Options, status_file: str, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(status_file): os.unlink(status_file) self.fscache = FileSystemCache() options.raise_exceptions = True options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True self.status_file = status_file # Since the object is created in the parent process we can check # the output terminal options here. self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.show_error_codes) def _response_metadata(self) -> Dict[str, str]: py_version = '{}_{}'.format(self.options.python_version[0], self.options.python_version[1]) return { 'platform': self.options.platform, 'python_version': py_version, } def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" command = None try: server = IPCServer(CONNECTION_NAME, self.timeout) with open(self.status_file, 'w') as f: json.dump( { 'pid': os.getpid(), 'connection_name': server.connection_name }, f) f.write('\n') # I like my JSON with a trailing newline while True: with server: data = receive(server) resp = {} # type: Dict[str, Any] if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') try: resp = self.run_command(command, data) except Exception: # If we are crashing, report the crash to the client tb = traceback.format_exception( *sys.exc_info()) resp = { 'error': "Daemon crashed!\n" + "".join(tb) } resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) raise try: resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) except OSError: pass # Maybe the client hung up if command == 'stop': reset_global_state() sys.exit(0) finally: # If the final command is something other than a clean # stop, remove the status file. (We can't just # simplify the logic and always remove the file, since # that could cause us to remove a future server's # status file.) if command != 'stop': os.unlink(self.status_file) try: server.cleanup() # try to remove the socket dir on Linux except OSError: pass exc_info = sys.exc_info() if exc_info[0] and exc_info[0] is not SystemExit: traceback.print_exception(*exc_info) def run_command(self, command: str, data: Dict[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': "Unrecognized command '%s'" % command} else: if command not in {'check', 'recheck', 'run'}: # Only the above commands use some error formatting. del data['is_tty'] del data['terminal_width'] return method(self, **data) # Command functions (run in the server via RPC). def cmd_status( self, fswatcher_dump_file: Optional[str] = None) -> Dict[str, object]: """Return daemon status.""" res = {} # type: Dict[str, object] res.update(get_meminfo()) if fswatcher_dump_file: data = self.fswatcher.dump_file_data() if hasattr( self, 'fswatcher') else {} # Using .dumps and then writing was noticeably faster than using dump s = json.dumps(data) with open(fswatcher_dump_file, 'w') as f: f.write(s) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" # We need to remove the status file *before* we complete the # RPC. Otherwise a race condition exists where a subsequent # command can see a status file from a dying server and think # it is a live one. os.unlink(self.status_file) return {} def cmd_run(self, version: str, args: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" try: # Process options can exit on improper arguments, so we need to catch that and # capture stderr so the client can report it stderr = io.StringIO() stdout = io.StringIO() with redirect_stderr(stderr): with redirect_stdout(stdout): sources, options = mypy.main.process_options( ['-i'] + list(args), require_targets=True, server_options=True, fscache=self.fscache, program='mypy-daemon', header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} if __version__ != version: return {'restart': 'mypy version changed'} if self.fine_grained_manager: manager = self.fine_grained_manager.manager start_plugins_snapshot = manager.plugins_snapshot _, current_plugins_snapshot = mypy.build.load_plugins( options, manager.errors, sys.stdout, extra_plugins=()) if current_plugins_snapshot != start_plugins_snapshot: return {'restart': 'plugins changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} except SystemExit as e: return { 'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': e.code } return self.check(sources, is_tty, terminal_width) def cmd_check(self, files: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files.""" try: sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(sources, is_tty, terminal_width) def cmd_recheck(self, is_tty: bool, terminal_width: int, remove: Optional[List[str]] = None, update: Optional[List[str]] = None) -> Dict[str, object]: """Check the same list of files we checked most recently. If remove/update is given, they modify the previous list; if all are None, stat() is called for each file in the previous list. """ t0 = time.time() if not self.fine_grained_manager: return { 'error': "Command 'recheck' is only valid after a 'check' command" } sources = self.previous_sources if remove: removals = set(remove) sources = [s for s in sources if s.path and s.path not in removals] if update: known = {s.path for s in sources if s.path} added = [p for p in update if p not in known] try: added_sources = create_source_list(added, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} sources = sources + added_sources # Make a copy! t1 = time.time() manager = self.fine_grained_manager.manager manager.log("fine-grained increment: cmd_recheck: {:.3f}s".format(t1 - t0)) if not self.following_imports(): messages = self.fine_grained_increment(sources, remove, update) else: assert remove is None and update is None messages = self.fine_grained_increment_follow_imports(sources) res = self.increment_output(messages, sources, is_tty, terminal_width) self.fscache.flush() self.update_stats(res) return res def check(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: """Check using fine-grained incremental mode. If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: if not self.following_imports(): messages = self.fine_grained_increment(sources) else: messages = self.fine_grained_increment_follow_imports(sources) res = self.increment_output(messages, sources, is_tty, terminal_width) self.fscache.flush() self.update_stats(res) return res def update_stats(self, res: Dict[str, Any]) -> None: if self.fine_grained_manager: manager = self.fine_grained_manager.manager manager.dump_stats() res['stats'] = manager.stats manager.stats = {} def following_imports(self) -> bool: """Are we following imports?""" # TODO: What about silent? return self.options.follow_imports == 'normal' def initialize_fine_grained(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) t0 = time.time() self.update_sources(sources) t1 = time.time() try: result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) if self.following_imports(): sources = find_all_sources_in_build( self.fine_grained_manager.graph, sources) self.update_sources(sources) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: t2 = time.time() # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, hash=meta.hash)) changed, removed = self.find_changed(sources) changed += self.find_added_suppressed( self.fine_grained_manager.graph, set(), self.fine_grained_manager.manager.search_paths) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) t3 = time.time() # Run an update messages = self.fine_grained_manager.update(changed, removed) if self.following_imports(): # We need to do another update to any new files found by following imports. messages = self.fine_grained_increment_follow_imports(sources) t4 = time.time() self.fine_grained_manager.manager.add_stats( update_sources_time=t1 - t0, build_time=t2 - t1, find_changes_time=t3 - t2, fg_update_time=t4 - t3, files_changed=len(removed) + len(changed)) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile(run_gc=False) status = 1 if messages else 0 messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def fine_grained_increment( self, sources: List[BuildSource], remove: Optional[List[str]] = None, update: Optional[List[str]] = None, ) -> List[str]: """Perform a fine-grained type checking increment. If remove and update are None, determine changed paths by using fswatcher. Otherwise, assume that only these files have changes. Args: sources: sources passed on the command line remove: paths of files that have been removed update: paths of files that have been changed or created """ assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() if remove is None and update is None: # Use the fswatcher to determine which files were changed # (updated or added) or removed. self.update_sources(sources) changed, removed = self.find_changed(sources) else: # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) changed += self.find_added_suppressed(self.fine_grained_manager.graph, set(), manager.search_paths) manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1)) manager.add_stats(find_changes_time=t1 - t0, fg_update_time=t2 - t1, files_changed=len(removed) + len(changed)) self.previous_sources = sources return messages def fine_grained_increment_follow_imports( self, sources: List[BuildSource]) -> List[str]: """Like fine_grained_increment, but follow imports.""" t0 = time.time() # TODO: Support file events assert self.fine_grained_manager is not None fine_grained_manager = self.fine_grained_manager graph = fine_grained_manager.graph manager = fine_grained_manager.manager orig_modules = list(graph.keys()) self.update_sources(sources) changed_paths = self.fswatcher.find_changed() manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) seen = {source.module for source in sources} # Find changed modules reachable from roots (or in roots) already in graph. changed, new_files = self.find_reachable_changed_modules( sources, graph, seen, changed_paths) sources.extend(new_files) # Process changes directly reachable from roots. messages = fine_grained_manager.update(changed, []) # Follow deps from changed modules (still within graph). worklist = changed[:] while worklist: module = worklist.pop() if module[0] not in graph: continue sources2 = self.direct_imports(module, graph) # Filter anything already seen before. This prevents # infinite looping if there are any self edges. (Self # edges are maybe a bug, but...) sources2 = [ source for source in sources2 if source.module not in seen ] changed, new_files = self.find_reachable_changed_modules( sources2, graph, seen, changed_paths) self.update_sources(new_files) messages = fine_grained_manager.update(changed, []) worklist.extend(changed) t2 = time.time() def refresh_file(module: str, path: str) -> List[str]: return fine_grained_manager.update([(module, path)], []) for module_id, state in list(graph.items()): new_messages = refresh_suppressed_submodules( module_id, state.path, fine_grained_manager.deps, graph, self.fscache, refresh_file) if new_messages is not None: messages = new_messages t3 = time.time() # There may be new files that became available, currently treated as # suppressed imports. Process them. while True: new_unsuppressed = self.find_added_suppressed( graph, seen, manager.search_paths) if not new_unsuppressed: break new_files = [ BuildSource(mod[1], mod[0]) for mod in new_unsuppressed ] sources.extend(new_files) self.update_sources(new_files) messages = fine_grained_manager.update(new_unsuppressed, []) for module_id, path in new_unsuppressed: new_messages = refresh_suppressed_submodules( module_id, path, fine_grained_manager.deps, graph, self.fscache, refresh_file) if new_messages is not None: messages = new_messages t4 = time.time() # Find all original modules in graph that were not reached -- they are deleted. to_delete = [] for module_id in orig_modules: if module_id not in graph: continue if module_id not in seen: module_path = graph[module_id].path assert module_path is not None to_delete.append((module_id, module_path)) if to_delete: messages = fine_grained_manager.update([], to_delete) fix_module_deps(graph) self.previous_sources = find_all_sources_in_build(graph) self.update_sources(self.previous_sources) # Store current file state as side effect self.fswatcher.find_changed() t5 = time.time() manager.log("fine-grained increment: update: {:.3f}s".format(t5 - t1)) manager.add_stats(find_changes_time=t1 - t0, fg_update_time=t2 - t1, refresh_suppressed_time=t3 - t2, find_added_supressed_time=t4 - t3, cleanup_time=t5 - t4) return messages def find_reachable_changed_modules( self, roots: List[BuildSource], graph: mypy.build.Graph, seen: Set[str], changed_paths: AbstractSet[str] ) -> Tuple[List[Tuple[str, str]], List[BuildSource]]: """Follow imports within graph from given sources until hitting changed modules. If we find a changed module, we can't continue following imports as the imports may have changed. Args: roots: modules where to start search from graph: module graph to use for the search seen: modules we've seen before that won't be visited (mutated here!!) changed_paths: which paths have changed (stop search here and return any found) Return (encountered reachable changed modules, unchanged files not in sources_set traversed). """ changed = [] new_files = [] worklist = roots[:] seen.update(source.module for source in worklist) while worklist: nxt = worklist.pop() if nxt.module not in seen: seen.add(nxt.module) new_files.append(nxt) if nxt.path in changed_paths: assert nxt.path is not None # TODO changed.append((nxt.module, nxt.path)) elif nxt.module in graph: state = graph[nxt.module] for dep in state.dependencies: if dep not in seen: seen.add(dep) worklist.append( BuildSource(graph[dep].path, graph[dep].id)) return changed, new_files def direct_imports(self, module: Tuple[str, str], graph: mypy.build.Graph) -> List[BuildSource]: """Return the direct imports of module not included in seen.""" state = graph[module[0]] return [ BuildSource(graph[dep].path, dep) for dep in state.dependencies ] def find_added_suppressed( self, graph: mypy.build.Graph, seen: Set[str], search_paths: SearchPaths) -> List[Tuple[str, str]]: """Find suppressed modules that have been added (and not included in seen). Args: seen: reachable modules we've seen before (mutated here!!) Return suppressed, added modules. """ all_suppressed = set() for state in graph.values(): all_suppressed |= state.suppressed_set # Filter out things that shouldn't actually be considered suppressed. # # TODO: Figure out why these are treated as suppressed all_suppressed = { module for module in all_suppressed if module not in graph and not ignore_suppressed_imports(module) } # Optimization: skip top-level packages that are obviously not # there, to avoid calling the relatively slow find_module() # below too many times. packages = {module.split('.', 1)[0] for module in all_suppressed} packages = filter_out_missing_top_level_packages( packages, search_paths, self.fscache) # TODO: Namespace packages finder = FindModuleCache(search_paths, self.fscache, self.options) found = [] for module in all_suppressed: top_level_pkg = module.split('.', 1)[0] if top_level_pkg not in packages: # Fast path: non-existent top-level package continue result = finder.find_module(module, fast_path=True) if isinstance(result, str) and module not in seen: # When not following imports, we only follow imports to .pyi files. if not self.following_imports() and not result.endswith( '.pyi'): continue found.append((module, result)) seen.add(module) return found def increment_output(self, messages: List[str], sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: status = 1 if messages else 0 messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def pretty_messages(self, messages: List[str], n_sources: int, is_tty: bool = False, terminal_width: Optional[int] = None) -> List[str]: use_color = self.options.color_output and is_tty fit_width = self.options.pretty and is_tty if fit_width: messages = self.formatter.fit_in_terminal( messages, fixed_terminal_width=terminal_width) if self.options.error_summary: summary = None # type: Optional[str] if messages: n_errors, n_files = count_stats(messages) if n_errors: summary = self.formatter.format_error(n_errors, n_files, n_sources, use_color=use_color) else: summary = self.formatter.format_success(n_sources, use_color) if summary: # Create new list to avoid appending multiple summaries on successive runs. messages = messages + [summary] if use_color: messages = [self.formatter.colorize(m) for m in messages] return messages def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] if self.following_imports(): # Filter out directories (used for namespace packages). paths = [path for path in paths if self.fscache.isfile(path)] self.fswatcher.add_watched_paths(paths) def update_changed( self, sources: List[BuildSource], remove: List[str], update: List[str], ) -> ChangesAndRemovals: changed_paths = self.fswatcher.update_changed(remove, update) return self._find_changed(sources, changed_paths) def find_changed(self, sources: List[BuildSource]) -> ChangesAndRemovals: changed_paths = self.fswatcher.find_changed() return self._find_changed(sources, changed_paths) def _find_changed(self, sources: List[BuildSource], changed_paths: AbstractSet[str]) -> ChangesAndRemovals: # Find anything that has been added or modified changed = [(source.module, source.path) for source in sources if source.path and source.path in changed_paths] # Now find anything that has been removed from the build modules = {source.module for source in sources} omitted = [ source for source in self.previous_sources if source.module not in modules ] removed = [] for source in omitted: path = source.path assert path removed.append((source.module, path)) # Find anything that has had its module path change because of added or removed __init__s last = {s.path: s.module for s in self.previous_sources} for s in sources: assert s.path if s.path in last and last[s.path] != s.module: # Mark it as removed from its old name and changed at its new name removed.append((last[s.path], s.path)) changed.append((s.module, s.path)) return changed, removed def cmd_suggest(self, function: str, callsites: bool, **kwargs: Any) -> Dict[str, object]: """Suggest a signature for a function.""" if not self.fine_grained_manager: return { 'error': "Command 'suggest' is only valid after a 'check' command" " (that produces no parse errors)" } engine = SuggestionEngine(self.fine_grained_manager, **kwargs) try: if callsites: out = engine.suggest_callsites(function) else: out = engine.suggest(function) except SuggestionFailure as err: return {'error': str(err)} else: if not out: out = "No suggestions\n" elif not out.endswith("\n"): out += "\n" return {'out': out, 'err': "", 'status': 0} finally: self.fscache.flush() def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {}
class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, options: Options, status_file: str, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(status_file): os.unlink(status_file) self.fscache = FileSystemCache() options.raise_exceptions = True options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True self.status_file = status_file # Since the object is created in the parent process we can check # the output terminal options here. self.formatter = FancyFormatter(sys.stdout, sys.stderr, options.show_error_codes) def _response_metadata(self) -> Dict[str, str]: py_version = '{}_{}'.format(self.options.python_version[0], self.options.python_version[1]) return { 'platform': self.options.platform, 'python_version': py_version, } def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" command = None try: server = IPCServer(CONNECTION_NAME, self.timeout) with open(self.status_file, 'w') as f: json.dump( { 'pid': os.getpid(), 'connection_name': server.connection_name }, f) f.write('\n') # I like my JSON with a trailing newline while True: with server: data = receive(server) resp = {} # type: Dict[str, Any] if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') try: resp = self.run_command(command, data) except Exception: # If we are crashing, report the crash to the client tb = traceback.format_exception( *sys.exc_info()) resp = { 'error': "Daemon crashed!\n" + "".join(tb) } resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) raise try: resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) except OSError: pass # Maybe the client hung up if command == 'stop': reset_global_state() sys.exit(0) finally: # If the final command is something other than a clean # stop, remove the status file. (We can't just # simplify the logic and always remove the file, since # that could cause us to remove a future server's # status file.) if command != 'stop': os.unlink(self.status_file) try: server.cleanup() # try to remove the socket dir on Linux except OSError: pass exc_info = sys.exc_info() if exc_info[0] and exc_info[0] is not SystemExit: traceback.print_exception(*exc_info) def run_command(self, command: str, data: Dict[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': "Unrecognized command '%s'" % command} else: if command not in {'check', 'recheck', 'run'}: # Only the above commands use some error formatting. del data['is_tty'] del data['terminal_width'] return method(self, **data) # Command functions (run in the server via RPC). def cmd_status( self, fswatcher_dump_file: Optional[str] = None) -> Dict[str, object]: """Return daemon status.""" res = {} # type: Dict[str, object] res.update(get_meminfo()) if fswatcher_dump_file: data = self.fswatcher.dump_file_data() if hasattr( self, 'fswatcher') else {} # Using .dumps and then writing was noticably faster than using dump s = json.dumps(data) with open(fswatcher_dump_file, 'w') as f: f.write(s) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" # We need to remove the status file *before* we complete the # RPC. Otherwise a race condition exists where a subsequent # command can see a status file from a dying server and think # it is a live one. os.unlink(self.status_file) return {} def cmd_run(self, version: str, args: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" try: # Process options can exit on improper arguments, so we need to catch that and # capture stderr so the client can report it stderr = io.StringIO() stdout = io.StringIO() with redirect_stderr(stderr): with redirect_stdout(stdout): sources, options = mypy.main.process_options( ['-i'] + list(args), require_targets=True, server_options=True, fscache=self.fscache, program='mypy-daemon', header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} if __version__ != version: return {'restart': 'mypy version changed'} if self.fine_grained_manager: manager = self.fine_grained_manager.manager start_plugins_snapshot = manager.plugins_snapshot _, current_plugins_snapshot = mypy.build.load_plugins( options, manager.errors, sys.stdout, extra_plugins=()) if current_plugins_snapshot != start_plugins_snapshot: return {'restart': 'plugins changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} except SystemExit as e: return { 'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': e.code } return self.check(sources, is_tty, terminal_width) def cmd_check(self, files: Sequence[str], is_tty: bool, terminal_width: int) -> Dict[str, object]: """Check a list of files.""" try: sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(sources, is_tty, terminal_width) def cmd_recheck(self, is_tty: bool, terminal_width: int, remove: Optional[List[str]] = None, update: Optional[List[str]] = None) -> Dict[str, object]: """Check the same list of files we checked most recently. If remove/update is given, they modify the previous list; if all are None, stat() is called for each file in the previous list. """ t0 = time.time() if not self.fine_grained_manager: return { 'error': "Command 'recheck' is only valid after a 'check' command" } sources = self.previous_sources if remove: removals = set(remove) sources = [s for s in sources if s.path and s.path not in removals] if update: known = {s.path for s in sources if s.path} added = [p for p in update if p not in known] try: added_sources = create_source_list(added, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} sources = sources + added_sources # Make a copy! t1 = time.time() manager = self.fine_grained_manager.manager manager.log("fine-grained increment: cmd_recheck: {:.3f}s".format(t1 - t0)) res = self.fine_grained_increment(sources, is_tty, terminal_width, remove, update) self.fscache.flush() self.update_stats(res) return res def check(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: """Check using fine-grained incremental mode. If is_tty is True format the output nicely with colors and summary line (unless disabled in self.options). Also pass the terminal_width to formatter. """ if not self.fine_grained_manager: res = self.initialize_fine_grained(sources, is_tty, terminal_width) else: res = self.fine_grained_increment(sources, is_tty, terminal_width) self.fscache.flush() self.update_stats(res) return res def update_stats(self, res: Dict[str, Any]) -> None: if self.fine_grained_manager: manager = self.fine_grained_manager.manager manager.dump_stats() res['stats'] = manager.stats manager.stats = {} def initialize_fine_grained(self, sources: List[BuildSource], is_tty: bool, terminal_width: int) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) t0 = time.time() self.update_sources(sources) t1 = time.time() try: result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: t2 = time.time() # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) changed, removed = self.find_changed(sources) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) t3 = time.time() # Run an update messages = self.fine_grained_manager.update(changed, removed) t4 = time.time() self.fine_grained_manager.manager.add_stats( update_sources_time=t1 - t0, build_time=t2 - t1, find_changes_time=t3 - t2, fg_update_time=t4 - t3, files_changed=len(removed) + len(changed)) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile(run_gc=False) status = 1 if messages else 0 messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def fine_grained_increment( self, sources: List[BuildSource], is_tty: bool, terminal_width: int, remove: Optional[List[str]] = None, update: Optional[List[str]] = None, ) -> Dict[str, Any]: assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() if remove is None and update is None: # Use the fswatcher to determine which files were changed # (updated or added) or removed. self.update_sources(sources) changed, removed = self.find_changed(sources) else: # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1)) manager.add_stats(find_changes_time=t1 - t0, fg_update_time=t2 - t1, files_changed=len(removed) + len(changed)) status = 1 if messages else 0 self.previous_sources = sources messages = self.pretty_messages(messages, len(sources), is_tty, terminal_width) return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def pretty_messages(self, messages: List[str], n_sources: int, is_tty: bool = False, terminal_width: Optional[int] = None) -> List[str]: use_color = self.options.color_output and is_tty fit_width = self.options.pretty and is_tty if fit_width: messages = self.formatter.fit_in_terminal( messages, fixed_terminal_width=terminal_width) if self.options.error_summary: summary = None # type: Optional[str] if messages: n_errors, n_files = count_stats(messages) if n_errors: summary = self.formatter.format_error( n_errors, n_files, n_sources, use_color) else: summary = self.formatter.format_success(n_sources, use_color) if summary: # Create new list to avoid appending multiple summaries on successive runs. messages = messages + [summary] if use_color: messages = [self.formatter.colorize(m) for m in messages] return messages def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] self.fswatcher.add_watched_paths(paths) def update_changed( self, sources: List[BuildSource], remove: List[str], update: List[str], ) -> ChangesAndRemovals: changed_paths = self.fswatcher.update_changed(remove, update) return self._find_changed(sources, changed_paths) def find_changed(self, sources: List[BuildSource]) -> ChangesAndRemovals: changed_paths = self.fswatcher.find_changed() return self._find_changed(sources, changed_paths) def _find_changed(self, sources: List[BuildSource], changed_paths: AbstractSet[str]) -> ChangesAndRemovals: # Find anything that has been added or modified changed = [(source.module, source.path) for source in sources if source.path and source.path in changed_paths] # Now find anything that has been removed from the build modules = {source.module for source in sources} omitted = [ source for source in self.previous_sources if source.module not in modules ] removed = [] for source in omitted: path = source.path assert path removed.append((source.module, path)) # Find anything that has had its module path change because of added or removed __init__s last = {s.path: s.module for s in self.previous_sources} for s in sources: assert s.path if s.path in last and last[s.path] != s.module: # Mark it as removed from its old name and changed at its new name removed.append((last[s.path], s.path)) changed.append((s.module, s.path)) return changed, removed def cmd_suggest(self, function: str, callsites: bool, **kwargs: Any) -> Dict[str, object]: """Suggest a signature for a function.""" if not self.fine_grained_manager: return { 'error': "Command 'suggest' is only valid after a 'check' command" " (that produces no parse errors)" } engine = SuggestionEngine(self.fine_grained_manager, **kwargs) try: if callsites: out = engine.suggest_callsites(function) else: out = engine.suggest(function) except SuggestionFailure as err: return {'error': str(err)} else: if not out: out = "No suggestions\n" elif not out.endswith("\n"): out += "\n" return {'out': out, 'err': "", 'status': 0} finally: self.fscache.flush() def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {}
def main( script_path: Optional[str], stdout: TextIO, stderr: TextIO, args: Optional[List[str]] = None, ) -> None: """Main entry point to the type checker. Args: script_path: Path to the 'mypy' script (used for finding data files). args: Custom command-line arguments. If not given, sys.argv[1:] will be used. """ util.check_python_version('mypy') t0 = time.time() # To log stat() calls: os.stat = stat_proxy sys.setrecursionlimit(2**14) if args is None: args = sys.argv[1:] fscache = FileSystemCache() sources, options = process_options(args, stdout=stdout, stderr=stderr, fscache=fscache) messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: messages.extend(new_messages) f = stderr if serious else stdout try: for msg in new_messages: f.write(msg + '\n') f.flush() except BrokenPipeError: sys.exit(2) serious = False blockers = False res = None try: # Keep a dummy reference (res) for memory profiling below, as otherwise # the result could be freed. res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) except CompileError as e: blockers = True if not e.use_stdout: serious = True if options.warn_unused_configs and options.unused_configs and not options.incremental: print("Warning: unused section(s) in %s: %s" % (options.config_file, ", ".join( "[mypy-%s]" % glob for glob in options.per_module_options.keys() if glob in options.unused_configs)), file=stderr) if options.junit_xml: t1 = time.time() py_version = '{}_{}'.format(options.python_version[0], options.python_version[1]) util.write_junit_xml(t1 - t0, serious, messages, options.junit_xml, py_version, options.platform) if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile() del res # Now it's safe to delete code = 0 if messages: code = 2 if blockers else 1 if options.fast_exit: # Exit without freeing objects -- it's faster. # # NOTE: We don't flush all open files on exit (or run other destructors)! util.hard_exit(code) elif code: sys.exit(code)
class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, flags: List[str]) -> None: """Initialize the server with the desired mypy flags.""" self.saved_cache = {} # type: mypy.build.SavedCache self.fine_grained_initialized = False sources, options = mypy.main.process_options(['-i'] + flags, require_targets=False, server_options=True) self.fine_grained = options.fine_grained_incremental if sources: sys.exit("dmypy: start/restart does not accept sources") if options.report_dirs: sys.exit("dmypy: start/restart cannot generate reports") if not options.incremental: sys.exit( "dmypy: start/restart should not disable incremental mode") if options.quick_and_dirty: sys.exit( "dmypy: start/restart should not specify quick_and_dirty mode") if options.use_fine_grained_cache and not options.fine_grained_incremental: sys.exit( "dmypy: fine-grained cache can only be used in experimental mode" ) self.options = options if os.path.isfile(STATUS_FILE): os.unlink(STATUS_FILE) if self.fine_grained: options.incremental = True options.show_traceback = True if options.use_fine_grained_cache: options.cache_fine_grained = True # set this so that cache options match else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" try: sock = self.create_listening_socket() try: with open(STATUS_FILE, 'w') as f: json.dump( { 'pid': os.getpid(), 'sockname': sock.getsockname() }, f) f.write('\n') # I like my JSON with trailing newline while True: conn, addr = sock.accept() data = receive(conn) resp = {} # type: Dict[str, Any] if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') resp = self.run_command(command, data) try: conn.sendall(json.dumps(resp).encode('utf8')) except OSError as err: pass # Maybe the client hung up conn.close() if command == 'stop': sock.close() sys.exit(0) finally: os.unlink(STATUS_FILE) finally: os.unlink(self.sockname) exc_info = sys.exc_info() if exc_info[0]: traceback.print_exception(*exc_info) # type: ignore def create_listening_socket(self) -> socket.socket: """Create the socket and set it up for listening.""" self.sockname = os.path.abspath(SOCKET_NAME) if os.path.exists(self.sockname): os.unlink(self.sockname) sock = socket.socket(socket.AF_UNIX) sock.bind(self.sockname) sock.listen(1) return sock def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': "Unrecognized command '%s'" % command} else: return method(self, **data) # Command functions (run in the server via RPC). def cmd_status(self) -> Dict[str, object]: """Return daemon status.""" res = {} # type: Dict[str, object] res.update(get_meminfo()) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" return {} last_sources = None # type: List[mypy.build.BuildSource] def cmd_check(self, files: Sequence[str]) -> Dict[str, object]: """Check a list of files.""" # TODO: Move this into check(), in case one of the args is a directory. # Capture stdout/stderr and catch SystemExit while processing the source list. save_stdout = sys.stdout save_stderr = sys.stderr try: sys.stdout = stdout = io.StringIO() sys.stderr = stderr = io.StringIO() self.last_sources = mypy.main.create_source_list( files, self.options) except SystemExit as err: return { 'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': err.code } finally: sys.stdout = save_stdout sys.stderr = save_stderr return self.check(self.last_sources) def cmd_recheck(self) -> Dict[str, object]: """Check the same list of files we checked most recently.""" if not self.last_sources: return { 'error': "Command 'recheck' is only valid after a 'check' command" } return self.check(self.last_sources) # Needed by tests. last_manager = None # type: Optional[mypy.build.BuildManager] def check(self, sources: List[mypy.build.BuildSource], alt_lib_path: Optional[str] = None) -> Dict[str, Any]: if self.fine_grained: return self.check_fine_grained(sources) else: return self.check_default(sources, alt_lib_path) def check_default(self, sources: List[mypy.build.BuildSource], alt_lib_path: Optional[str] = None) -> Dict[str, Any]: """Check using the default (per-file) incremental mode.""" self.last_manager = None with GcLogger() as gc_result: try: # saved_cache is mutated in place. res = mypy.build.build(sources, self.options, saved_cache=self.saved_cache, alt_lib_path=alt_lib_path) msgs = res.errors self.last_manager = res.manager # type: Optional[mypy.build.BuildManager] except mypy.errors.CompileError as err: msgs = err.messages if msgs: msgs.append("") response = {'out': "\n".join(msgs), 'err': "", 'status': 1} else: response = {'out': "", 'err': "", 'status': 0} response.update(gc_result.get_stats()) response.update(get_meminfo()) if self.last_manager is not None: response.update(self.last_manager.stats_summary()) return response def check_fine_grained( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: """Check using fine-grained incremental mode.""" if not self.fine_grained_initialized: return self.initialize_fine_grained(sources) else: return self.fine_grained_increment(sources) def initialize_fine_grained( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: self.fscache = FileSystemCache(self.options.python_version) self.fswatcher = FileSystemWatcher(self.fscache) self.update_sources(sources) if not self.options.use_fine_grained_cache: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() try: # TODO: alt_lib_path result = mypy.build.build(sources=sources, options=self.options) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors manager = result.manager graph = result.graph self.fine_grained_manager = mypy.server.update.FineGrainedBuildManager( manager, graph) self.fine_grained_initialized = True self.previous_sources = sources self.fscache.flush() # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if self.options.use_fine_grained_cache: # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for meta, mypyfile, type_map in manager.saved_cache.values(): if meta.mtime is None: continue self.fswatcher.set_file_data( mypyfile.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) # Run an update changed = self.find_changed(sources) if changed: messages = self.fine_grained_manager.update(changed) self.fscache.flush() status = 1 if messages else 0 self.previous_messages = messages[:] return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def fine_grained_increment( self, sources: List[mypy.build.BuildSource]) -> Dict[str, Any]: t0 = time.time() self.update_sources(sources) changed = self.find_changed(sources) t1 = time.time() if not changed: # Nothing changed -- just produce the same result as before. messages = self.previous_messages else: messages = self.fine_grained_manager.update(changed) t2 = time.time() self.fine_grained_manager.manager.log( "fine-grained increment: find_changed: {:.3f}s, update: {:.3f}s". format(t1 - t0, t2 - t1)) status = 1 if messages else 0 self.previous_messages = messages[:] self.previous_sources = sources self.fscache.flush() return { 'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status } def update_sources(self, sources: List[mypy.build.BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] self.fswatcher.add_watched_paths(paths) def find_changed( self, sources: List[mypy.build.BuildSource]) -> List[Tuple[str, str]]: changed_paths = self.fswatcher.find_changed() changed = [(source.module, source.path) for source in sources if source.path in changed_paths] modules = {source.module for source in sources} omitted = [ source for source in self.previous_sources if source.module not in modules ] for source in omitted: path = source.path assert path # Note that a file could be removed from the list of root sources but have no changes. if path in changed_paths: changed.append((source.module, path)) return changed def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {}
class Server: # NOTE: the instance is constructed in the parent process but # serve() is called in the grandchild (by daemonize()). def __init__(self, options: Options, status_file: str, timeout: Optional[int] = None) -> None: """Initialize the server with the desired mypy flags.""" self.options = options # Snapshot the options info before we muck with it, to detect changes self.options_snapshot = options.snapshot() self.timeout = timeout self.fine_grained_manager = None # type: Optional[FineGrainedBuildManager] if os.path.isfile(status_file): os.unlink(status_file) self.fscache = FileSystemCache() options.incremental = True options.fine_grained_incremental = True options.show_traceback = True if options.use_fine_grained_cache: # Using fine_grained_cache implies generating and caring # about the fine grained cache options.cache_fine_grained = True else: options.cache_dir = os.devnull # Fine-grained incremental doesn't support general partial types # (details in https://github.com/python/mypy/issues/4492) options.local_partial_types = True self.status_file = status_file def _response_metadata(self) -> Dict[str, str]: py_version = '{}_{}'.format(self.options.python_version[0], self.options.python_version[1]) return { 'platform': self.options.platform, 'python_version': py_version, } def serve(self) -> None: """Serve requests, synchronously (no thread or fork).""" command = None try: server = IPCServer(CONNECTION_NAME, self.timeout) with open(self.status_file, 'w') as f: json.dump({'pid': os.getpid(), 'connection_name': server.connection_name}, f) f.write('\n') # I like my JSON with a trailing newline while True: with server: data = receive(server) resp = {} # type: Dict[str, Any] if 'command' not in data: resp = {'error': "No command found in request"} else: command = data['command'] if not isinstance(command, str): resp = {'error': "Command is not a string"} else: command = data.pop('command') try: resp = self.run_command(command, data) except Exception: # If we are crashing, report the crash to the client tb = traceback.format_exception(*sys.exc_info()) resp = {'error': "Daemon crashed!\n" + "".join(tb)} resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) raise try: resp.update(self._response_metadata()) server.write(json.dumps(resp).encode('utf8')) except OSError: pass # Maybe the client hung up if command == 'stop': reset_global_state() sys.exit(0) finally: # If the final command is something other than a clean # stop, remove the status file. (We can't just # simplify the logic and always remove the file, since # that could cause us to remove a future server's # status file.) if command != 'stop': os.unlink(self.status_file) try: server.cleanup() # try to remove the socket dir on Linux except OSError: pass exc_info = sys.exc_info() if exc_info[0] and exc_info[0] is not SystemExit: traceback.print_exception(*exc_info) def run_command(self, command: str, data: Mapping[str, object]) -> Dict[str, object]: """Run a specific command from the registry.""" key = 'cmd_' + command method = getattr(self.__class__, key, None) if method is None: return {'error': "Unrecognized command '%s'" % command} else: return method(self, **data) # Command functions (run in the server via RPC). def cmd_status(self, fswatcher_dump_file: Optional[str] = None) -> Dict[str, object]: """Return daemon status.""" res = {} # type: Dict[str, object] res.update(get_meminfo()) if fswatcher_dump_file: data = self.fswatcher.dump_file_data() if hasattr(self, 'fswatcher') else {} # Using .dumps and then writing was noticably faster than using dump s = json.dumps(data) with open(fswatcher_dump_file, 'w') as f: f.write(s) return res def cmd_stop(self) -> Dict[str, object]: """Stop daemon.""" # We need to remove the status file *before* we complete the # RPC. Otherwise a race condition exists where a subsequent # command can see a status file from a dying server and think # it is a live one. os.unlink(self.status_file) return {} def cmd_run(self, version: str, args: Sequence[str]) -> Dict[str, object]: """Check a list of files, triggering a restart if needed.""" try: # Process options can exit on improper arguments, so we need to catch that and # capture stderr so the client can report it stderr = io.StringIO() stdout = io.StringIO() with redirect_stderr(stderr): with redirect_stdout(stdout): sources, options = mypy.main.process_options( ['-i'] + list(args), require_targets=True, server_options=True, fscache=self.fscache, program='mypy-daemon', header=argparse.SUPPRESS) # Signal that we need to restart if the options have changed if self.options_snapshot != options.snapshot(): return {'restart': 'configuration changed'} if __version__ != version: return {'restart': 'mypy version changed'} if self.fine_grained_manager: manager = self.fine_grained_manager.manager start_plugins_snapshot = manager.plugins_snapshot _, current_plugins_snapshot = mypy.build.load_plugins(options, manager.errors) if current_plugins_snapshot != start_plugins_snapshot: return {'restart': 'plugins changed'} except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} except SystemExit as e: return {'out': stdout.getvalue(), 'err': stderr.getvalue(), 'status': e.code} return self.check(sources) def cmd_check(self, files: Sequence[str]) -> Dict[str, object]: """Check a list of files.""" try: sources = create_source_list(files, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} return self.check(sources) def cmd_recheck(self, remove: Optional[List[str]] = None, update: Optional[List[str]] = None) -> Dict[str, object]: """Check the same list of files we checked most recently. If remove/update is given, they modify the previous list; if all are None, stat() is called for each file in the previous list. """ t0 = time.time() if not self.fine_grained_manager: return {'error': "Command 'recheck' is only valid after a 'check' command"} sources = self.previous_sources if remove: removals = set(remove) sources = [s for s in sources if s.path and s.path not in removals] if update: known = {s.path for s in sources if s.path} added = [p for p in update if p not in known] try: added_sources = create_source_list(added, self.options, self.fscache) except InvalidSourceList as err: return {'out': '', 'err': str(err), 'status': 2} sources = sources + added_sources # Make a copy! t1 = time.time() manager = self.fine_grained_manager.manager manager.log("fine-grained increment: cmd_recheck: {:.3f}s".format(t1 - t0)) res = self.fine_grained_increment(sources, remove, update) self.fscache.flush() self.update_stats(res) return res def check(self, sources: List[BuildSource]) -> Dict[str, Any]: """Check using fine-grained incremental mode.""" if not self.fine_grained_manager: res = self.initialize_fine_grained(sources) else: res = self.fine_grained_increment(sources) self.fscache.flush() self.update_stats(res) return res def update_stats(self, res: Dict[str, Any]) -> None: if self.fine_grained_manager: manager = self.fine_grained_manager.manager manager.dump_stats() res['stats'] = manager.stats manager.stats = {} def initialize_fine_grained(self, sources: List[BuildSource]) -> Dict[str, Any]: self.fswatcher = FileSystemWatcher(self.fscache) t0 = time.time() self.update_sources(sources) t1 = time.time() try: result = mypy.build.build(sources=sources, options=self.options, fscache=self.fscache) except mypy.errors.CompileError as e: output = ''.join(s + '\n' for s in e.messages) if e.use_stdout: out, err = output, '' else: out, err = '', output return {'out': out, 'err': err, 'status': 2} messages = result.errors self.fine_grained_manager = FineGrainedBuildManager(result) self.previous_sources = sources # If we are using the fine-grained cache, build hasn't actually done # the typechecking on the updated files yet. # Run a fine-grained update starting from the cached data if result.used_cache: t2 = time.time() # Pull times and hashes out of the saved_cache and stick them into # the fswatcher, so we pick up the changes. for state in self.fine_grained_manager.graph.values(): meta = state.meta if meta is None: continue assert state.path is not None self.fswatcher.set_file_data( state.path, FileData(st_mtime=float(meta.mtime), st_size=meta.size, md5=meta.hash)) changed, removed = self.find_changed(sources) # Find anything that has had its dependency list change for state in self.fine_grained_manager.graph.values(): if not state.is_fresh(): assert state.path is not None changed.append((state.id, state.path)) t3 = time.time() # Run an update messages = self.fine_grained_manager.update(changed, removed) t4 = time.time() self.fine_grained_manager.manager.add_stats( update_sources_time=t1 - t0, build_time=t2 - t1, find_changes_time=t3 - t2, fg_update_time=t4 - t3, files_changed=len(removed) + len(changed)) else: # Stores the initial state of sources as a side effect. self.fswatcher.find_changed() if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile(run_gc=False) status = 1 if messages else 0 return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} def fine_grained_increment(self, sources: List[BuildSource], remove: Optional[List[str]] = None, update: Optional[List[str]] = None, ) -> Dict[str, Any]: assert self.fine_grained_manager is not None manager = self.fine_grained_manager.manager t0 = time.time() if remove is None and update is None: # Use the fswatcher to determine which files were changed # (updated or added) or removed. self.update_sources(sources) changed, removed = self.find_changed(sources) else: # Use the remove/update lists to update fswatcher. # This avoids calling stat() for unchanged files. changed, removed = self.update_changed(sources, remove or [], update or []) manager.search_paths = compute_search_paths(sources, manager.options, manager.data_dir) t1 = time.time() manager.log("fine-grained increment: find_changed: {:.3f}s".format(t1 - t0)) messages = self.fine_grained_manager.update(changed, removed) t2 = time.time() manager.log("fine-grained increment: update: {:.3f}s".format(t2 - t1)) manager.add_stats( find_changes_time=t1 - t0, fg_update_time=t2 - t1, files_changed=len(removed) + len(changed)) status = 1 if messages else 0 self.previous_sources = sources return {'out': ''.join(s + '\n' for s in messages), 'err': '', 'status': status} def update_sources(self, sources: List[BuildSource]) -> None: paths = [source.path for source in sources if source.path is not None] self.fswatcher.add_watched_paths(paths) def update_changed(self, sources: List[BuildSource], remove: List[str], update: List[str], ) -> ChangesAndRemovals: changed_paths = self.fswatcher.update_changed(remove, update) return self._find_changed(sources, changed_paths) def find_changed(self, sources: List[BuildSource]) -> ChangesAndRemovals: changed_paths = self.fswatcher.find_changed() return self._find_changed(sources, changed_paths) def _find_changed(self, sources: List[BuildSource], changed_paths: AbstractSet[str]) -> ChangesAndRemovals: # Find anything that has been added or modified changed = [(source.module, source.path) for source in sources if source.path and source.path in changed_paths] # Now find anything that has been removed from the build modules = {source.module for source in sources} omitted = [source for source in self.previous_sources if source.module not in modules] removed = [] for source in omitted: path = source.path assert path removed.append((source.module, path)) # Find anything that has had its module path change because of added or removed __init__s last = {s.path: s.module for s in self.previous_sources} for s in sources: assert s.path if s.path in last and last[s.path] != s.module: # Mark it as removed from its old name and changed at its new name removed.append((last[s.path], s.path)) changed.append((s.module, s.path)) return changed, removed def cmd_suggest(self, function: str, callsites: bool, # We'd like to just use **kwargs here and save some duplication but # mypyc doesn't support it yet... json: bool, no_errors: bool, no_any: bool) -> Dict[str, object]: """Suggest a signature for a function.""" if not self.fine_grained_manager: return {'error': "Command 'suggest' is only valid after a 'check' command"} engine = SuggestionEngine(self.fine_grained_manager, json, no_errors, no_any) try: if callsites: out = engine.suggest_callsites(function) else: out = engine.suggest(function) except SuggestionFailure as err: return {'error': str(err)} else: if not out: out = "No suggestions\n" elif not out.endswith("\n"): out += "\n" return {'out': out, 'err': "", 'status': 0} finally: self.fscache.flush() def cmd_hang(self) -> Dict[str, object]: """Hang for 100 seconds, as a debug hack.""" time.sleep(100) return {}
def main(script_path: Optional[str], stdout: TextIO, stderr: TextIO, args: Optional[List[str]] = None, ) -> None: """Main entry point to the type checker. Args: script_path: Path to the 'mypy' script (used for finding data files). args: Custom command-line arguments. If not given, sys.argv[1:] will be used. """ util.check_python_version('mypy') t0 = time.time() # To log stat() calls: os.stat = stat_proxy sys.setrecursionlimit(2 ** 14) if args is None: args = sys.argv[1:] fscache = FileSystemCache() sources, options = process_options(args, stdout=stdout, stderr=stderr, fscache=fscache) messages = [] formatter = util.FancyFormatter(stdout, stderr, options.show_error_codes) def flush_errors(new_messages: List[str], serious: bool) -> None: if options.pretty: new_messages = formatter.fit_in_terminal(new_messages) messages.extend(new_messages) f = stderr if serious else stdout try: for msg in new_messages: if options.color_output: msg = formatter.colorize(msg) f.write(msg + '\n') f.flush() except BrokenPipeError: sys.exit(2) serious = False blockers = False res = None try: # Keep a dummy reference (res) for memory profiling below, as otherwise # the result could be freed. res = build.build(sources, options, None, flush_errors, fscache, stdout, stderr) except CompileError as e: blockers = True if not e.use_stdout: serious = True if options.warn_unused_configs and options.unused_configs and not options.incremental: print("Warning: unused section(s) in %s: %s" % (options.config_file, ", ".join("[mypy-%s]" % glob for glob in options.per_module_options.keys() if glob in options.unused_configs)), file=stderr) maybe_write_junit_xml(time.time() - t0, serious, messages, options) if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile() code = 0 if messages: code = 2 if blockers else 1 if options.error_summary: if messages: n_errors, n_files = util.count_stats(messages) if n_errors: stdout.write(formatter.format_error(n_errors, n_files, len(sources), options.color_output) + '\n') else: stdout.write(formatter.format_success(len(sources), options.color_output) + '\n') stdout.flush() if options.fast_exit: # Exit without freeing objects -- it's faster. # # NOTE: We don't flush all open files on exit (or run other destructors)! util.hard_exit(code) elif code: sys.exit(code) # HACK: keep res alive so that mypyc won't free it before the hard_exit list([res])
def main(script_path: Optional[str], args: Optional[List[str]] = None) -> None: """Main entry point to the type checker. Args: script_path: Path to the 'mypy' script (used for finding data files). args: Custom command-line arguments. If not given, sys.argv[1:] will be used. """ # Check for known bad Python versions. if sys.version_info[:2] < (3, 4): sys.exit("Running mypy with Python 3.3 or lower is not supported; " "please upgrade to 3.4 or newer") if sys.version_info[:3] == (3, 5, 0): sys.exit("Running mypy with Python 3.5.0 is not supported; " "please upgrade to 3.5.1 or newer") t0 = time.time() # To log stat() calls: os.stat = stat_proxy if script_path: bin_dir = find_bin_directory(script_path) # type: Optional[str] else: bin_dir = None sys.setrecursionlimit(2**14) if args is None: args = sys.argv[1:] fscache = FileSystemCache() sources, options = process_options(args, fscache=fscache) messages = [] def flush_errors(new_messages: List[str], serious: bool) -> None: messages.extend(new_messages) f = sys.stderr if serious else sys.stdout try: for msg in new_messages: f.write(msg + '\n') f.flush() except BrokenPipeError: sys.exit(2) serious = False blockers = False try: # Keep a dummy reference (res) for memory profiling below, as otherwise # the result could be freed. res = type_check_only(sources, bin_dir, options, flush_errors, fscache) # noqa except CompileError as e: blockers = True if not e.use_stdout: serious = True if options.warn_unused_configs and options.unused_configs: print("Warning: unused section(s) in %s: %s" % (options.config_file, ", ".join( "[mypy-%s]" % glob for glob in options.per_module_options.keys() if glob in options.unused_configs)), file=sys.stderr) if options.junit_xml: t1 = time.time() util.write_junit_xml(t1 - t0, serious, messages, options.junit_xml) if MEM_PROFILE: from mypy.memprofile import print_memory_profile print_memory_profile() if messages: code = 2 if blockers else 1 sys.exit(code)