class BuildManager:
"""This is the central class for building a mypy program.
It coordinates parsing, import processing, semantic analysis and
type checking. It manages state objects that actually perform the
build steps.
Attributes:
data_dir: Mypy data directory (contains stubs)
target: Build target; selects which passes to perform
lib_path: Library path for looking up modules
semantic_analyzer:
Semantic analyzer, pass 2
semantic_analyzer_pass3:
Semantic analyzer, pass 3
type_checker: Type checker
errors: Used for reporting all errors
pyversion: Python version (major, minor)
flags: Build options
states: States of all individual files that are being
processed. Each file in a build is always represented
by a single state object (after it has been encountered
for the first time). This is the only place where
states are stored.
module_files: Map from module name to source file path. There is a
1:1 mapping between modules and source files.
module_deps: Cache for module dependencies (direct or indirect).
Item (m, n) indicates whether m depends on n (directly
or indirectly).
missing_modules: Set of modules that could not be imported encountered so far
"""
def __init__(self, data_dir: str,
lib_path: List[str],
target: int,
pyversion: Tuple[int, int],
flags: List[str],
ignore_prefix: str,
custom_typing_module: str,
reports: Reports) -> None:
self.data_dir = data_dir
self.errors = Errors()
self.errors.set_ignore_prefix(ignore_prefix)
self.lib_path = lib_path
self.target = target
self.pyversion = pyversion
self.flags = flags
self.custom_typing_module = custom_typing_module
self.reports = reports
self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors,
pyversion=pyversion)
modules = self.semantic_analyzer.modules
self.semantic_analyzer_pass3 = ThirdPass(modules, self.errors)
self.type_checker = TypeChecker(self.errors, modules, self.pyversion)
self.states = [] # type: List[State]
self.module_files = {} # type: Dict[str, str]
self.module_deps = {} # type: Dict[Tuple[str, str], bool]
self.missing_modules = set() # type: Set[str]
def process(self, initial_states: List['UnprocessedFile']) -> BuildResult:
"""Perform a build.
The argument is a state that represents the main program
file. This method should only be called once per a build
manager object. The return values are identical to the return
values of the build function.
"""
self.states += initial_states
for initial_state in initial_states:
self.module_files[initial_state.id] = initial_state.path
for initial_state in initial_states:
initial_state.load_dependencies()
# Process states in a loop until all files (states) have been
# semantically analyzed or type checked (depending on target).
#
# We type check all files before the rest of the passes so that we can
# report errors and fail as quickly as possible.
while True:
# Find the next state that has all its dependencies met.
next = self.next_available_state()
if not next:
trace('done')
break
# Potentially output some debug information.
trace('next {} ({})'.format(next.path, next.state()))
# Set the import context for reporting error messages correctly.
self.errors.set_import_context(next.import_context)
# Process the state. The process method is reponsible for adding a
# new state object representing the new state of the file.
next.process()
# Raise exception if the build failed. The build can fail for
# various reasons, such as parse error, semantic analysis error,
# etc.
if self.errors.is_blockers():
self.errors.raise_error()
# If there were no errors, all files should have been fully processed.
for s in self.states:
assert s.state() == final_state, (
'{} still unprocessed in state {}'.format(s.path, s.state()))
if self.errors.is_errors():
self.errors.raise_error()
# Collect a list of all files.
trees = [] # type: List[MypyFile]
for state in self.states:
trees.append(cast(ParsedFile, state).tree)
# Perform any additional passes after type checking for all the files.
self.final_passes(trees, self.type_checker.type_map)
return BuildResult(self.semantic_analyzer.modules,
self.type_checker.type_map)
def next_available_state(self) -> 'State':
"""Find a ready state (one that has all its dependencies met)."""
i = len(self.states) - 1
while i >= 0:
if self.states[i].is_ready():
num_incomplete = self.states[i].num_incomplete_deps()
if num_incomplete == 0:
# This is perfect; no need to look for the best match.
return self.states[i]
i -= 1
return None
def has_module(self, name: str) -> bool:
"""Have we seen a module yet?"""
return name in self.module_files
def file_state(self, path: str) -> int:
"""Return the state of a source file.
In particular, return UNSEEN_STATE if the file has no associated
state.
This function does not consider any dependencies.
"""
for s in self.states:
if s.path == path:
return s.state()
return UNSEEN_STATE
def module_state(self, name: str) -> int:
"""Return the state of a module.
In particular, return UNSEEN_STATE if the file has no associated
state.
This considers also module dependencies.
"""
if not self.has_module(name):
return UNSEEN_STATE
state = final_state
fs = self.file_state(self.module_files[name])
if earlier_state(fs, state):
state = fs
return state
def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool:
"""Does m1 import m2 directly or indirectly?"""
# Have we computed this previously?
dep = self.module_deps.get((m1, m2))
if dep is not None:
return dep
if not done:
done = set([m1])
# m1 depends on m2 iff one of the deps of m1 depends on m2.
st = self.lookup_state(m1)
for m in st.dependencies:
if m in done:
continue
done.add(m)
# Cache this dependency.
self.module_deps[m1, m] = True
# Search recursively.
if m == m2 or self.is_dep(m, m2, done):
# Yes! Mark it in the cache.
self.module_deps[m1, m2] = True
return True
# No dependency. Mark it in the cache.
self.module_deps[m1, m2] = False
return False
def lookup_state(self, module: str) -> 'State':
for state in self.states:
if state.id == module:
return state
raise RuntimeError('%s not found' % module)
def all_imported_modules_in_file(self,
file: MypyFile) -> List[Tuple[str, int]]:
"""Find all reachable import statements in a file.
Return list of tuples (module id, import line number) for all modules
imported in file.
"""
def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
"""Function to correct for relative imports."""
file_id = file.fullname()
rel = imp.relative
if rel == 0:
return imp.id
if os.path.basename(file.path).startswith('__init__.'):
rel -= 1
if rel != 0:
file_id = ".".join(file_id.split(".")[:-rel])
new_id = file_id + "." + imp.id if imp.id else file_id
return new_id
res = [] # type: List[Tuple[str, int]]
for imp in file.imports:
if not imp.is_unreachable:
if isinstance(imp, Import):
for id, _ in imp.ids:
res.append((id, imp.line))
elif isinstance(imp, ImportFrom):
cur_id = correct_rel_imp(imp)
res.append((cur_id, imp.line))
# Also add any imported names that are submodules.
for name, __ in imp.names:
sub_id = cur_id + '.' + name
if self.is_module(sub_id):
res.append((sub_id, imp.line))
elif isinstance(imp, ImportAll):
res.append((correct_rel_imp(imp), imp.line))
return res
def is_module(self, id: str) -> bool:
"""Is there a file in the file system corresponding to module id?"""
return find_module(id, self.lib_path) is not None
def final_passes(self, files: List[MypyFile],
types: Dict[Node, Type]) -> None:
"""Perform the code generation passes for type checked files."""
if self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]:
pass # Nothing to do.
else:
raise RuntimeError('Unsupported target %d' % self.target)
def log(self, message: str) -> None:
if VERBOSE in self.flags:
print('LOG: %s' % message)
class BuildManager:
"""This is the central class for building a mypy program.
It coordinates parsing, import processing, semantic analysis and
type checking. It manages state objects that actually perform the
build steps.
Attributes:
data_dir: Mypy data directory (contains stubs)
target: Build target; selects which passes to perform
lib_path: Library path for looking up modules
semantic_analyzer:
Semantic analyzer, pass 2
semantic_analyzer_pass3:
Semantic analyzer, pass 3
type_checker: Type checker
errors: Used for reporting all errors
pyversion: Python version (major, minor)
flags: Build options
states: States of all individual files that are being
processed. Each file in a build is always represented
by a single state object (after it has been encountered
for the first time). This is the only place where
states are stored.
module_files: Map from module name to source file path. There is a
1:1 mapping between modules and source files.
module_deps: Cache for module dependencies (direct or indirect).
Item (m, n) indicates whether m depends on n (directly
or indirectly).
missing_modules: Set of modules that could not be imported encountered so far
"""
def __init__(self, data_dir: str,
lib_path: List[str],
target: int,
pyversion: Tuple[int, int],
flags: List[str],
ignore_prefix: str,
custom_typing_module: str,
reports: Reports) -> None:
self.data_dir = data_dir
self.errors = Errors()
self.errors.set_ignore_prefix(ignore_prefix)
self.lib_path = lib_path
self.target = target
self.pyversion = pyversion
self.flags = flags
self.custom_typing_module = custom_typing_module
self.reports = reports
self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors,
pyversion=pyversion)
modules = self.semantic_analyzer.modules
self.semantic_analyzer_pass3 = ThirdPass(modules, self.errors)
self.type_checker = TypeChecker(self.errors, modules, self.pyversion)
self.states = [] # type: List[State]
self.module_files = {} # type: Dict[str, str]
self.module_deps = {} # type: Dict[Tuple[str, str], bool]
self.missing_modules = set() # type: Set[str]
def process(self, initial_states: List['UnprocessedFile']) -> BuildResult:
"""Perform a build.
The argument is a state that represents the main program
file. This method should only be called once per a build
manager object. The return values are identical to the return
values of the build function.
"""
self.states += initial_states
for initial_state in initial_states:
self.module_files[initial_state.id] = initial_state.path
for initial_state in initial_states:
initial_state.load_dependencies()
# Process states in a loop until all files (states) have been
# semantically analyzed or type checked (depending on target).
#
# We type check all files before the rest of the passes so that we can
# report errors and fail as quickly as possible.
while True:
# Find the next state that has all its dependencies met.
next = self.next_available_state()
if not next:
trace('done')
break
# Potentially output some debug information.
trace('next {} ({})'.format(next.path, next.state()))
# Set the import context for reporting error messages correctly.
self.errors.set_import_context(next.import_context)
# Process the state. The process method is reponsible for adding a
# new state object representing the new state of the file.
next.process()
# Raise exception if the build failed. The build can fail for
# various reasons, such as parse error, semantic analysis error,
# etc.
if self.errors.is_blockers():
self.errors.raise_error()
# If there were no errors, all files should have been fully processed.
for s in self.states:
assert s.state() == final_state, (
'{} still unprocessed in state {}'.format(s.path, s.state()))
if self.errors.is_errors():
self.errors.raise_error()
# Collect a list of all files.
trees = [] # type: List[MypyFile]
for state in self.states:
trees.append(cast(ParsedFile, state).tree)
# Perform any additional passes after type checking for all the files.
self.final_passes(trees, self.type_checker.type_map)
return BuildResult(self.semantic_analyzer.modules,
self.type_checker.type_map)
def next_available_state(self) -> 'State':
"""Find a ready state (one that has all its dependencies met)."""
i = len(self.states) - 1
while i >= 0:
if self.states[i].is_ready():
num_incomplete = self.states[i].num_incomplete_deps()
if num_incomplete == 0:
# This is perfect; no need to look for the best match.
return self.states[i]
i -= 1
return None
def has_module(self, name: str) -> bool:
"""Have we seen a module yet?"""
return name in self.module_files
def file_state(self, path: str) -> int:
"""Return the state of a source file.
In particular, return UNSEEN_STATE if the file has no associated
state.
This function does not consider any dependencies.
"""
for s in self.states:
if s.path == path:
return s.state()
return UNSEEN_STATE
def module_state(self, name: str) -> int:
"""Return the state of a module.
In particular, return UNSEEN_STATE if the file has no associated
state.
This considers also module dependencies.
"""
if not self.has_module(name):
return UNSEEN_STATE
state = final_state
fs = self.file_state(self.module_files[name])
if earlier_state(fs, state):
state = fs
return state
def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool:
"""Does m1 import m2 directly or indirectly?"""
# Have we computed this previously?
dep = self.module_deps.get((m1, m2))
if dep is not None:
return dep
if not done:
done = set([m1])
# m1 depends on m2 iff one of the deps of m1 depends on m2.
st = self.lookup_state(m1)
for m in st.dependencies:
if m in done:
continue
done.add(m)
# Cache this dependency.
self.module_deps[m1, m] = True
# Search recursively.
if m == m2 or self.is_dep(m, m2, done):
# Yes! Mark it in the cache.
self.module_deps[m1, m2] = True
return True
# No dependency. Mark it in the cache.
self.module_deps[m1, m2] = False
return False
def lookup_state(self, module: str) -> 'State':
for state in self.states:
if state.id == module:
return state
raise RuntimeError('%s not found' % module)
def all_imported_modules_in_file(self,
file: MypyFile) -> List[Tuple[str, int]]:
"""Find all reachable import statements in a file.
Return list of tuples (module id, import line number) for all modules
imported in file.
"""
def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str:
"""Function to correct for relative imports."""
file_id = file.fullname()
rel = imp.relative
if rel == 0:
return imp.id
if os.path.basename(file.path).startswith('__init__.'):
rel -= 1
if rel != 0:
file_id = ".".join(file_id.split(".")[:-rel])
new_id = file_id + "." + imp.id if imp.id else file_id
return new_id
res = [] # type: List[Tuple[str, int]]
for imp in file.imports:
if not imp.is_unreachable:
if isinstance(imp, Import):
for id, _ in imp.ids:
res.append((id, imp.line))
elif isinstance(imp, ImportFrom):
cur_id = correct_rel_imp(imp)
res.append((cur_id, imp.line))
# Also add any imported names that are submodules.
for name, __ in imp.names:
sub_id = cur_id + '.' + name
if self.is_module(sub_id):
res.append((sub_id, imp.line))
elif isinstance(imp, ImportAll):
res.append((correct_rel_imp(imp), imp.line))
return res
def is_module(self, id: str) -> bool:
"""Is there a file in the file system corresponding to module id?"""
return find_module(id, self.lib_path) is not None
def final_passes(self, files: List[MypyFile],
types: Dict[Node, Type]) -> None:
"""Perform the code generation passes for type checked files."""
if self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]:
pass # Nothing to do.
else:
raise RuntimeError('Unsupported target %d' % self.target)
def log(self, message: str) -> None:
if VERBOSE in self.flags:
print('LOG: %s' % message)
class BuildManager:
"""This is the central class for building a mypy program.
It coordinates parsing, import processing, semantic analysis and
type checking. It manages state objects that actually perform the
build steps.
Attributes:
data_dir: Mypy data directory (contains stubs)
target: Build target; selects which passes to perform
lib_path: Library path for looking up modules
semantic_analyzer:
Semantic analyzer, pass 2
semantic_analyzer_pass3:
Semantic analyzer, pass 3
type_checker: Type checker
errors: Used for reporting all errors
output_dir: Store output files here (Python)
pyversion: Python version (2 or 3)
flags: Build options
states: States of all individual files that are being
processed. Each file in a build is always represented
by a single state object (after it has been encountered
for the first time). This is the only place where
states are stored.
module_files: Map from module name to source file path. There is a
1:1 mapping between modules and source files.
icode: Generated icode (when compiling via C)
binary_path: Path of the generated binary (or None)
module_deps: Cache for module dependencies (direct or indirect).
Item (m, n) indicates whether m depends on n (directly
or indirectly).
TODO Refactor code related to transformation, icode generation etc. to
external objects. This module should not directly depend on them.
"""
def __init__(self, data_dir: str,
lib_path: List[str],
target: int,
output_dir: str,
pyversion: int,
flags: List[str],
ignore_prefix: str) -> None:
self.data_dir = data_dir
self.errors = Errors()
self.errors.set_ignore_prefix(ignore_prefix)
self.lib_path = lib_path
self.target = target
self.output_dir = output_dir
self.pyversion = pyversion
self.flags = flags
self.semantic_analyzer = SemanticAnalyzer(lib_path, self.errors)
self.semantic_analyzer_pass3 = ThirdPass(self.errors)
self.type_checker = TypeChecker(self.errors,
self.semantic_analyzer.modules,
self.pyversion)
self.states = List[State]()
self.module_files = Dict[str, str]()
self.icode = Dict[str, FuncIcode]()
self.binary_path = None # type: str
self.module_deps = Dict[Tuple[str, str], bool]()
def process(self, initial_state: 'UnprocessedFile') -> BuildResult:
"""Perform a build.
The argument is a state that represents the main program
file. This method should only be called once per a build
manager object. The return values are identical to the return
values of the build function.
"""
self.states.append(initial_state)
# Process states in a loop until all files (states) have been
# semantically analyzed or type checked (depending on target).
#
# We type check all files before the rest of the passes so that we can
# report errors and fail as quickly as possible.
while True:
# Find the next state that has all its dependencies met.
next = self.next_available_state()
if not next:
trace('done')
break
# Potentially output some debug information.
trace('next {} ({})'.format(next.path, next.state()))
# Set the import context for reporting error messages correctly.
self.errors.set_import_context(next.import_context)
# Process the state. The process method is reponsible for adding a
# new state object representing the new state of the file.
next.process()
# Raise exception if the build failed. The build can fail for
# various reasons, such as parse error, semantic analysis error,
# etc.
if self.errors.is_errors():
self.errors.raise_error()
# If there were no errors, all files should have been fully processed.
for s in self.states:
assert s.state() == final_state, (
'{} still unprocessed'.format(s.path))
# Collect a list of all files.
trees = List[MypyFile]()
for state in self.states:
trees.append((cast('ParsedFile', state)).tree)
# Perform any additional passes after type checking for all the files.
self.final_passes(trees, self.type_checker.type_map)
return BuildResult(self.semantic_analyzer.modules,
self.type_checker.type_map,
self.icode, self.binary_path)
def next_available_state(self) -> 'State':
"""Find a ready state (one that has all its dependencies met)."""
i = len(self.states) - 1
while i >= 0:
if self.states[i].is_ready():
num_incomplete = self.states[i].num_incomplete_deps()
if num_incomplete == 0:
# This is perfect; no need to look for the best match.
return self.states[i]
i -= 1
return None
def has_module(self, name: str) -> bool:
"""Have we seen a module yet?"""
return name in self.module_files
def file_state(self, path: str) -> int:
"""Return the state of a source file.
In particular, return UNSEEN_STATE if the file has no associated
state.
This function does not consider any dependencies.
"""
for s in self.states:
if s.path == path:
return s.state()
return UNSEEN_STATE
def module_state(self, name: str) -> int:
"""Return the state of a module.
In particular, return UNSEEN_STATE if the file has no associated
state.
This considers also module dependencies.
"""
if not self.has_module(name):
return UNSEEN_STATE
state = final_state
fs = self.file_state(self.module_files[name])
if earlier_state(fs, state):
state = fs
return state
def is_dep(self, m1: str, m2: str, done: Set[str] = None) -> bool:
"""Does m1 import m2 directly or indirectly?"""
# Have we computed this previously?
dep = self.module_deps.get((m1, m2))
if dep is not None:
return dep
if not done:
done = set([m1])
# m1 depends on m2 iff one of the deps of m1 depends on m2.
st = self.lookup_state(m1)
for m in st.dependencies:
if m in done:
continue
done.add(m)
# Cache this dependency.
self.module_deps[m1, m] = True
# Search recursively.
if m == m2 or self.is_dep(m, m2, done):
# Yes! Mark it in the cache.
self.module_deps[m1, m2] = True
return True
# No dependency. Mark it in the cache.
self.module_deps[m1, m2] = False
return False
def lookup_state(self, module: str) -> 'State':
for state in self.states:
if state.id == module:
return state
raise RuntimeError('%s not found' % str)
def all_imported_modules_in_file(self,
file: MypyFile) -> List[Tuple[str, int]]:
"""Find all import statements in a file.
Return list of tuples (module id, import line number) for all modules
imported in file.
"""
# TODO also find imports not at the top level of the file
res = List[Tuple[str, int]]()
for imp in file.imports:
if isinstance(imp, Import):
for id, _ in imp.ids:
res.append((id, imp.line))
elif isinstance(imp, ImportFrom):
res.append((imp.id, imp.line))
# Also add any imported names that are submodules.
for name, __ in imp.names:
sub_id = imp.id + '.' + name
if self.is_module(sub_id):
res.append((sub_id, imp.line))
elif isinstance(imp, ImportAll):
res.append((imp.id, imp.line))
return res
def is_module(self, id: str) -> bool:
"""Is there a file in the file system corresponding to module id?"""
return find_module(id, self.lib_path) is not None
def final_passes(self, files: List[MypyFile],
types: Dict[Node, Type]) -> None:
"""Perform the code generation passes for type checked files."""
if self.target == TRANSFORM:
self.transform(files)
elif self.target == ICODE:
self.transform(files)
self.generate_icode(files, types)
elif self.target == C:
self.transform(files)
self.generate_icode(files, types)
self.generate_c_and_compile(files)
elif self.target in [SEMANTIC_ANALYSIS, TYPE_CHECK]:
pass # Nothing to do.
else:
raise RuntimeError('Unsupported target %d' % self.target)
def get_python_out_path(self, f: MypyFile) -> str:
if f.fullname() == '__main__':
return os.path.join(self.output_dir, basename(f.path))
else:
components = f.fullname().split('.')
if os.path.basename(f.path) == '__init__.py':
components.append('__init__.py')
else:
components[-1] += '.py'
return os.path.join(self.output_dir, *components)
def transform(self, files: List[MypyFile]) -> None:
for f in files:
if f.fullname() == 'typing':
# The typing module is special and is currently not
# transformed.
continue
# Transform parse tree and produce pretty-printed output.
v = transform.DyncheckTransformVisitor(
self.type_checker.type_map,
self.semantic_analyzer.modules,
is_pretty=True)
f.accept(v)
def generate_icode(self, files: List[MypyFile],
types: Dict[Node, Type]) -> None:
builder = icode.IcodeBuilder(types)
for f in files:
# TODO remove ugly builtins hack
if not f.path.endswith('/builtins.py'):
f.accept(builder)
self.icode = builder.generated
def generate_c_and_compile(self, files: List[MypyFile]) -> None:
gen = cgen.CGenerator()
for fn, icode in self.icode.items():
gen.generate_function('M' + fn, icode)
program_name = os.path.splitext(basename(files[0].path))[0]
c_file = '%s.c' % program_name
# Write C file.
self.log('writing %s' % c_file)
out = open(c_file, 'w')
out.writelines(gen.output())
out.close()
if COMPILE_ONLY not in self.flags:
# Generate binary file.
data_dir = self.data_dir
vm_dir = os.path.join(data_dir, 'vm')
cc = os.getenv('CC', 'gcc')
cflags = shlex.split(os.getenv('CFLAGS', '-O2'))
cmdline = [cc] + cflags +['-I%s' % vm_dir,
'-o%s' % program_name,
c_file,
os.path.join(vm_dir, 'runtime.c')]
self.log(' '.join(cmdline))
status = subprocess.call(cmdline)
# TODO check status
self.log('removing %s' % c_file)
os.remove(c_file)
self.binary_path = os.path.join('.', program_name)
def log(self, message: str) -> None:
if VERBOSE in self.flags:
print('LOG: %s' % message)