class BuildManager: """This is the central class for building a mypy program. It coordinates parsing, import processing, semantic analysis and type checking. It manages state objects that actually perform the build steps. Attributes: data_dir: Mypy data directory (contains stubs) target: Build target; selects which passes to perform lib_path: Library path for looking up modules semantic_analyzer: Semantic analyzer, pass 2 semantic_analyzer_pass3: Semantic analyzer, pass 3 type_checker: Type checker errors: Used for reporting all errors pyversion: Python version (major, minor) flags: Build options states: States of all individual files that are being processed. Each file in a build is always represented by a single state object (after it has been encountered for the first time). This is the only place where states are stored. module_files: Map from module name to source file path. There is a 1:1 mapping between modules and source files. module_deps: Cache for module dependencies (direct or indirect). Item (m, n) indicates whether m depends on n (directly or indirectly). missing_modules: Set of modules that could not be imported encountered so far """ def __init__(self, data_dir: str, lib_path: List[str], target: int, pyversion: Tuple[int, int], flags: List[str], ignore_prefix: str, custom_typing_module: str, reports: Reports) -> None: self.data_dir = data_dir self.errors = Errors() self.errors.set_ignore_prefix(ignore_prefix) self.lib_path = lib_path self.target = target self.pyversion = pyversion self.flags = flags self.custom_typing_module = custom_typing_module self.reports = reports self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors, pyversion=pyversion) modules = self.semantic_analyzer.modules self.semantic_analyzer_pass3 = ThirdPass(modules, self.errors) self.type_checker = TypeChecker(self.errors, modules, self.pyversion) self.states = [] # type: List[State] self.module_files = {} # type: Dict[str, str] self.module_deps = {} # type: Dict[Tuple[str, str], bool] self.missing_modules = set() # type: Set[str] def process(self, initial_states: List['UnprocessedFile']) -> BuildResult: """Perform a build. The argument is a state that represents the main program file. This method should only be called once per a build manager object. The return values are identical to the return values of the build function. """ self.states += initial_states for initial_state in initial_states: self.module_files[initial_state.id] = initial_state.path for initial_state in initial_states: initial_state.load_dependencies() # Process states in a loop until all files (states) have been # semantically analyzed or type checked (depending on target). # # We type check all files before the rest of the passes so that we can # report errors and fail as quickly as possible. while True: # Find the next state that has all its dependencies met. next = self.next_available_state() if not next: trace('done') break # Potentially output some debug information. trace('next {} ({})'.format(next.path, next.state())) # Set the import context for reporting error messages correctly. self.errors.set_import_context(next.import_context) # Process the state. The process method is reponsible for adding a # new state object representing the new state of the file. next.process() # Raise exception if the build failed. The build can fail for # various reasons, such as parse error, semantic analysis error, # etc. if self.errors.is_blockers(): self.errors.raise_error() # If there were no errors, all files should have been fully processed. for s in self.states: assert s.state() == final_state, ( '{} still unprocessed in state {}'.format(s.path, s.state())) if self.errors.is_errors(): self.errors.raise_error() # Collect a list of all files. trees = [] # type: List[MypyFile] for state in self.states: trees.append(cast(ParsedFile, state).tree) # Perform any additional passes after type checking for all the files. self.final_passes(trees, self.type_checker.type_map) return BuildResult(self.semantic_analyzer.modules, self.type_checker.type_map) def next_available_state(self) -> 'State': """Find a ready state (one that has all its dependencies met).""" i = len(self.states) - 1 while i >= 0: if self.states[i].is_ready(): num_incomplete = self.states[i].num_incomplete_deps() if num_incomplete == 0: # This is perfect; no need to look for the best match. return self.states[i] i -= 1 return None def has_module(self, name: str) -> bool: """Have we seen a module yet?""" return name in self.module_files def file_state(self, path: str) -> int: """Return the state of a source file. In particular, return UNSEEN_STATE if the file has no associated state. This function does not consider any dependencies. """ for s in self.states: if s.path == path: return s.state() return UNSEEN_STATE def module_state(self, name: str) -> int: """Return the state of a module. In particular, return UNSEEN_STATE if the file has no associated state. This considers also module dependencies. """ if not self.has_module(name): return UNSEEN_STATE state = final_state fs = self.file_state(self.module_files[name]) if earlier_state(fs, state): state = fs return state def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool: """Does m1 import m2 directly or indirectly?""" # Have we computed this previously? dep = self.module_deps.get((m1, m2)) if dep is not None: return dep if not done: done = set([m1]) # m1 depends on m2 iff one of the deps of m1 depends on m2. st = self.lookup_state(m1) for m in st.dependencies: if m in done: continue done.add(m) # Cache this dependency. self.module_deps[m1, m] = True # Search recursively. if m == m2 or self.is_dep(m, m2, done): # Yes! Mark it in the cache. self.module_deps[m1, m2] = True return True # No dependency. Mark it in the cache. self.module_deps[m1, m2] = False return False def lookup_state(self, module: str) -> 'State': for state in self.states: if state.id == module: return state raise RuntimeError('%s not found' % module) def all_imported_modules_in_file(self, file: MypyFile) -> List[Tuple[str, int]]: """Find all reachable import statements in a file. Return list of tuples (module id, import line number) for all modules imported in file. """ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: """Function to correct for relative imports.""" file_id = file.fullname() rel = imp.relative if rel == 0: return imp.id if os.path.basename(file.path).startswith('__init__.'): rel -= 1 if rel != 0: file_id = ".".join(file_id.split(".")[:-rel]) new_id = file_id + "." + imp.id if imp.id else file_id return new_id res = [] # type: List[Tuple[str, int]] for imp in file.imports: if not imp.is_unreachable: if isinstance(imp, Import): for id, _ in imp.ids: res.append((id, imp.line)) elif isinstance(imp, ImportFrom): cur_id = correct_rel_imp(imp) res.append((cur_id, imp.line)) # Also add any imported names that are submodules. for name, __ in imp.names: sub_id = cur_id + '.' + name if self.is_module(sub_id): res.append((sub_id, imp.line)) elif isinstance(imp, ImportAll): res.append((correct_rel_imp(imp), imp.line)) return res def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" return find_module(id, self.lib_path) is not None def final_passes(self, files: List[MypyFile], types: Dict[Node, Type]) -> None: """Perform the code generation passes for type checked files.""" if self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]: pass # Nothing to do. else: raise RuntimeError('Unsupported target %d' % self.target) def log(self, message: str) -> None: if VERBOSE in self.flags: print('LOG: %s' % message)
class BuildManager: """This is the central class for building a mypy program. It coordinates parsing, import processing, semantic analysis and type checking. It manages state objects that actually perform the build steps. Attributes: data_dir: Mypy data directory (contains stubs) target: Build target; selects which passes to perform lib_path: Library path for looking up modules semantic_analyzer: Semantic analyzer, pass 2 semantic_analyzer_pass3: Semantic analyzer, pass 3 type_checker: Type checker errors: Used for reporting all errors output_dir: Store output files here (Python) pyversion: Python version (2 or 3) flags: Build options states: States of all individual files that are being processed. Each file in a build is always represented by a single state object (after it has been encountered for the first time). This is the only place where states are stored. module_files: Map from module name to source file path. There is a 1:1 mapping between modules and source files. icode: Generated icode (when compiling via C) binary_path: Path of the generated binary (or None) module_deps: Cache for module dependencies (direct or indirect). Item (m, n) indicates whether m depends on n (directly or indirectly). TODO Refactor code related to transformation, icode generation etc. to external objects. This module should not directly depend on them. """ def __init__(self, data_dir: str, lib_path: List[str], target: int, output_dir: str, pyversion: int, flags: List[str], ignore_prefix: str) -> None: self.data_dir = data_dir self.errors = Errors() self.errors.set_ignore_prefix(ignore_prefix) self.lib_path = lib_path self.target = target self.output_dir = output_dir self.pyversion = pyversion self.flags = flags self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors) self.semantic_analyzer_pass3 = ThirdPass(self.errors) self.type_checker = TypeChecker(self.errors, self.semantic_analyzer.modules, self.pyversion) self.states = List[State]() self.module_files = Dict[str, str]() self.icode = Dict[str, FuncIcode]() self.binary_path = None # type: str self.module_deps = Dict[Tuple[str, str], bool]() def process(self, initial_state: 'UnprocessedFile') -> BuildResult: """Perform a build. The argument is a state that represents the main program file. This method should only be called once per a build manager object. The return values are identical to the return values of the build function. """ self.states.append(initial_state) # Process states in a loop until all files (states) have been # semantically analyzed or type checked (depending on target). # # We type check all files before the rest of the passes so that we can # report errors and fail as quickly as possible. while True: # Find the next state that has all its dependencies met. next = self.next_available_state() if not next: trace('done') break # Potentially output some debug information. trace('next {} ({})'.format(next.path, next.state())) # Set the import context for reporting error messages correctly. self.errors.set_import_context(next.import_context) # Process the state. The process method is reponsible for adding a # new state object representing the new state of the file. next.process() # Raise exception if the build failed. The build can fail for # various reasons, such as parse error, semantic analysis error, # etc. if self.errors.is_errors(): self.errors.raise_error() # If there were no errors, all files should have been fully processed. for s in self.states: assert s.state() == final_state, ( '{} still unprocessed'.format(s.path)) # Collect a list of all files. trees = List[MypyFile]() for state in self.states: trees.append((cast('ParsedFile', state)).tree) # Perform any additional passes after type checking for all the files. self.final_passes(trees, self.type_checker.type_map) return BuildResult(self.semantic_analyzer.modules, self.type_checker.type_map, self.icode, self.binary_path) def next_available_state(self) -> 'State': """Find a ready state (one that has all its dependencies met).""" i = len(self.states) - 1 while i >= 0: if self.states[i].is_ready(): num_incomplete = self.states[i].num_incomplete_deps() if num_incomplete == 0: # This is perfect; no need to look for the best match. return self.states[i] i -= 1 return None def has_module(self, name: str) -> bool: """Have we seen a module yet?""" return name in self.module_files def file_state(self, path: str) -> int: """Return the state of a source file. In particular, return UNSEEN_STATE if the file has no associated state. This function does not consider any dependencies. """ for s in self.states: if s.path == path: return s.state() return UNSEEN_STATE def module_state(self, name: str) -> int: """Return the state of a module. In particular, return UNSEEN_STATE if the file has no associated state. This considers also module dependencies. """ if not self.has_module(name): return UNSEEN_STATE state = final_state fs = self.file_state(self.module_files[name]) if earlier_state(fs, state): state = fs return state def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool: """Does m1 import m2 directly or indirectly?""" # Have we computed this previously? dep = self.module_deps.get((m1, m2)) if dep is not None: return dep if not done: done = set([m1]) # m1 depends on m2 iff one of the deps of m1 depends on m2. st = self.lookup_state(m1) for m in st.dependencies: if m in done: continue done.add(m) # Cache this dependency. self.module_deps[m1, m] = True # Search recursively. if m == m2 or self.is_dep(m, m2, done): # Yes! Mark it in the cache. self.module_deps[m1, m2] = True return True # No dependency. Mark it in the cache. self.module_deps[m1, m2] = False return False def lookup_state(self, module: str) -> 'State': for state in self.states: if state.id == module: return state raise RuntimeError('%s not found' % str) def all_imported_modules_in_file(self, file: MypyFile) -> List[Tuple[str, int]]: """Find all import statements in a file. Return list of tuples (module id, import line number) for all modules imported in file. """ # TODO also find imports not at the top level of the file res = List[Tuple[str, int]]() for imp in file.imports: if isinstance(imp, Import): for id, _ in imp.ids: res.append((id, imp.line)) elif isinstance(imp, ImportFrom): res.append((imp.id, imp.line)) # Also add any imported names that are submodules. for name, __ in imp.names: sub_id = imp.id + '.' + name if self.is_module(sub_id): res.append((sub_id, imp.line)) elif isinstance(imp, ImportAll): res.append((imp.id, imp.line)) return res def is_module(self, id: str) -> bool: """Is there a file in the file system corresponding to module id?""" return find_module(id, self.lib_path) is not None def final_passes(self, files: List[MypyFile], types: Dict[Node, Type]) -> None: """Perform the code generation passes for type checked files.""" if self.target == TRANSFORM: self.transform(files) elif self.target == ICODE: self.transform(files) self.generate_icode(files, types) elif self.target == C: self.transform(files) self.generate_icode(files, types) self.generate_c_and_compile(files) elif self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]: pass # Nothing to do. else: raise RuntimeError('Unsupported target %d' % self.target) def get_python_out_path(self, f: MypyFile) -> str: if f.fullname() == '__main__': return os.path.join(self.output_dir, basename(f.path)) else: components = f.fullname().split('.') if os.path.basename(f.path) == '__init__.py': components.append('__init__.py') else: components[-1] += '.py' return os.path.join(self.output_dir, *components) def transform(self, files: List[MypyFile]) -> None: for f in files: if f.fullname() == 'typing': # The typing module is special and is currently not # transformed. continue # Transform parse tree and produce pretty-printed output. v = transform.DyncheckTransformVisitor( self.type_checker.type_map, self.semantic_analyzer.modules, is_pretty=True) f.accept(v) def generate_icode(self, files: List[MypyFile], types: Dict[Node, Type]) -> None: builder = icode.IcodeBuilder(types) for f in files: # TODO remove ugly builtins hack if not f.path.endswith('/builtins.py'): f.accept(builder) self.icode = builder.generated def generate_c_and_compile(self, files: List[MypyFile]) -> None: gen = cgen.CGenerator() for fn, icode in self.icode.items(): gen.generate_function('M' + fn, icode) program_name = os.path.splitext(basename(files[0].path))[0] c_file = '%s.c' % program_name # Write C file. self.log('writing %s' % c_file) out = open(c_file, 'w') out.writelines(gen.output()) out.close() if COMPILE_ONLY not in self.flags: # Generate binary file. data_dir = self.data_dir vm_dir = os.path.join(data_dir, 'vm') cc = os.getenv('CC', 'gcc') cflags = shlex.split(os.getenv('CFLAGS', '-O2')) cmdline = [cc] + cflags +['-I%s' % vm_dir, '-o%s' % program_name, c_file, os.path.join(vm_dir, 'runtime.c')] self.log(' '.join(cmdline)) status = subprocess.call(cmdline) # TODO check status self.log('removing %s' % c_file) os.remove(c_file) self.binary_path = os.path.join('.', program_name) def log(self, message: str) -> None: if VERBOSE in self.flags: print('LOG: %s' % message)