def compile_modules_to_c(result: BuildResult,
module_names: List[str],
use_shared_lib: bool,
ops: Optional[List[str]] = None) -> str:
"""Compile Python module(s) to C that can be used from Python C extension modules."""
# Generate basic IR, with missing exception and refcount handling.
file_nodes = [result.files[name] for name in module_names]
literals, modules = genops.build_ir(file_nodes, result.graph, result.types)
# Insert exception handling.
for _, module in modules:
for fn in module.functions:
insert_exception_handling(fn)
# Insert refcount handling.
for _, module in modules:
for fn in module.functions:
insert_ref_count_opcodes(fn)
# Format ops for debugging
if ops is not None:
for _, module in modules:
for fn in module.functions:
ops.extend(format_func(fn))
ops.append('')
# Generate C code.
source_paths = {
module_name: result.files[module_name].path
for module_name in module_names
}
generator = ModuleGenerator(literals, modules, source_paths,
use_shared_lib)
return generator.generate_c_for_modules()
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a data-flow analysis test case."""
with use_custom_builtins(
os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
try:
ir = build_ir_for_single_file(testcase.input)
except CompileError as e:
actual = e.messages
else:
actual = []
for fn in ir:
if (fn.name == TOP_LEVEL_NAME
and not testcase.name.endswith('_toplevel')):
continue
exceptions.insert_exception_handling(fn)
actual.extend(format_func(fn))
cfg = analysis.get_cfg(fn.blocks)
args = set(reg for reg, i in fn.env.indexes.items()
if i < len(fn.args))
name = testcase.name
if name.endswith('_MaybeDefined'):
# Forward, maybe
analysis_result = analysis.analyze_maybe_defined_regs(
fn.blocks, cfg, args)
elif name.endswith('_Liveness'):
# Backward, maybe
analysis_result = analysis.analyze_live_regs(
fn.blocks, cfg)
elif name.endswith('_MustDefined'):
# Forward, must
analysis_result = analysis.analyze_must_defined_regs(
fn.blocks, cfg, args, regs=fn.env.regs())
elif name.endswith('_BorrowedArgument'):
# Forward, must
analysis_result = analysis.analyze_borrowed_arguments(
fn.blocks, cfg, args)
else:
assert False, 'No recognized _AnalysisName suffix in test case'
for key in sorted(analysis_result.before.keys(),
key=lambda x: (x[0].label, x[1])):
pre = ', '.join(
sorted(reg.name
for reg in analysis_result.before[key]))
post = ', '.join(
sorted(reg.name
for reg in analysis_result.after[key]))
actual.append('%-8s %-23s %s' %
((key[0].label, key[1]), '{%s}' % pre,
'{%s}' % post))
assert_test_output(testcase, actual, 'Invalid source code output')
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a runtime checking transformation test case."""
with use_custom_builtins(
os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
expected_output = remove_comment_lines(testcase.output)
try:
ir = build_ir_for_single_file(testcase.input)
except CompileError as e:
actual = e.messages
else:
actual = []
for fn in ir:
if (fn.name == TOP_LEVEL_NAME
and not testcase.name.endswith('_toplevel')):
continue
actual.extend(format_func(fn))
assert_test_output(testcase, actual, 'Invalid source code output',
expected_output)
def compile_modules_to_c(
result: BuildResult,
module_names: List[str],
shared_lib_name: Optional[str],
compiler_options: CompilerOptions,
errors: Errors,
ops: Optional[List[str]] = None) -> List[Tuple[str, str]]:
"""Compile Python module(s) to C that can be used from Python C extension modules."""
# Generate basic IR, with missing exception and refcount handling.
file_nodes = [result.files[name] for name in module_names]
literals, modules = genops.build_ir(file_nodes, result.graph, result.types,
compiler_options, errors)
if errors.num_errors > 0:
return []
# Insert uninit checks.
for _, module in modules:
for fn in module.functions:
insert_uninit_checks(fn)
# Insert exception handling.
for _, module in modules:
for fn in module.functions:
insert_exception_handling(fn)
# Insert refcount handling.
for _, module in modules:
for fn in module.functions:
insert_ref_count_opcodes(fn)
# Format ops for debugging
if ops is not None:
for _, module in modules:
for fn in module.functions:
ops.extend(format_func(fn))
ops.append('')
# Generate C code.
source_paths = {
module_name: result.files[module_name].path
for module_name in module_names
}
generator = ModuleGenerator(literals, modules, source_paths,
shared_lib_name, compiler_options.multi_file)
return generator.generate_c_for_modules()
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a reference count opcode insertion transform test case."""
with use_custom_builtins(
os.path.join(test_data_prefix, ICODE_GEN_BUILTINS), testcase):
try:
ir = build_ir_for_single_file(testcase.input)
except CompileError as e:
actual = e.messages
else:
# Expect one function + module top level function.
assert len(
ir
) == 2, "Only 1 function definition expected per test case"
fn = ir[0]
insert_exception_handling(fn)
insert_ref_count_opcodes(fn)
actual = format_func(fn)
actual = actual[actual.index('L0:'):]
assert_test_output(testcase, actual, 'Invalid source code output')
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a runtime checking transformation test case."""
with use_custom_builtins(
os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
expected_output = remove_comment_lines(testcase.output)
program_text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
source = build.BuildSource('main', '__main__', program_text)
try:
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
actual = e.messages
else:
if result.errors:
actual = result.errors
else:
modules = genops.build_ir([result.files['__main__']],
result.types)
module = modules[0][1]
actual = []
for fn in module.functions:
if is_empty_module_top_level(fn):
# Skip trivial module top levels that only return.
continue
actual.extend(format_func(fn))
assert_test_output(testcase, actual, 'Invalid source code output',
expected_output)
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a reference count opcode insertion transform test case."""
with use_custom_builtins(
os.path.join(test_data_prefix, ICODE_GEN_BUILTINS), testcase):
try:
ir = build_ir_for_single_file(testcase.input)
except CompileError as e:
actual = e.messages
else:
assert len(
ir
) == 1, "Only 1 function definition expected per test case"
fn = ir[0]
insert_ref_count_opcodes(fn)
actual = format_func(fn)
actual = actual[actual.index('L0:'):]
expected_output = testcase.output
assert_string_arrays_equal_wildcards(
expected_output, actual,
'Invalid source code output ({}, line {})'.format(
testcase.file, testcase.line))
def compile_modules_to_c(
result: BuildResult,
compiler_options: CompilerOptions,
errors: Errors,
groups: Groups,
ops: Optional[List[str]] = None,
) -> List[FileContents]:
"""Compile Python module(s) to the source of Python C extension modules.
This generates the source code for the "shared library" module
for each group. The shim modules are generated in mypyc.build.
Each shared library module provides, for each module in its group,
a PyCapsule containing an initialization function.
Additionally, it provides a capsule containing an export table of
pointers to all of the group's functions and static variables.
Arguments:
result: The BuildResult from the mypy front-end
compiler_options: The compilation options
errors: Where to report any errors encountered
groups: The groups that we are compiling. See documentation of Groups type above.
ops: Optionally, where to dump stringified ops for debugging.
Returns a list containing the generated files for each group.
"""
module_names = [source.module for group_sources, _ in groups for source in group_sources]
file_nodes = [result.files[name] for name in module_names]
# Construct a map from modules to what group they belong to
group_map = {}
for group, lib_name in groups:
for source in group:
group_map[source.module] = lib_name
# Generate basic IR, with missing exception and refcount handling.
mapper = genops.Mapper(group_map)
modules = genops.build_ir(file_nodes, result.graph, result.types,
mapper,
compiler_options, errors)
if errors.num_errors > 0:
return []
# Insert uninit checks.
for _, module in modules:
for fn in module.functions:
insert_uninit_checks(fn)
# Insert exception handling.
for _, module in modules:
for fn in module.functions:
insert_exception_handling(fn)
# Insert refcount handling.
for _, module in modules:
for fn in module.functions:
insert_ref_count_opcodes(fn)
# Format ops for debugging
if ops is not None:
for _, module in modules:
for fn in module.functions:
ops.extend(format_func(fn))
ops.append('')
source_paths = {module_name: result.files[module_name].path
for module_name in module_names}
names = NameGenerator([[source.module for source in sources] for sources, _ in groups])
# Generate C code for each compilation group. Each group will be
# compiled into a separate extension module.
ctext = []
module_dict = dict(modules)
for group_sources, group_name in groups:
modules = [(source.module, module_dict[source.module]) for source in group_sources]
literals = mapper.literals[group_name]
generator = GroupGenerator(
literals, modules, source_paths, group_name, group_map, names,
compiler_options.multi_file
)
ctext.append(generator.generate_c_for_modules())
return ctext
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a data-flow analysis test case."""
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
program_text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.python_version = (3, 6)
options.export_types = True
source = build.BuildSource('main', '__main__', program_text)
try:
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
actual = e.messages
else:
if result.errors:
actual = result.errors
else:
modules = genops.build_ir([result.files['__main__']], result.types)
module = modules[0][1]
assert len(module.functions) == 2, (
"Only 1 function definition expected per test case")
fn = module.functions[0]
actual = format_func(fn)
actual = actual[actual.index('L0:'):]
cfg = analysis.get_cfg(fn.blocks)
args = set(reg for reg, i in fn.env.indexes.items() if i < len(fn.args))
name = testcase.name
if name.endswith('_MaybeDefined'):
# Forward, maybe
analysis_result = analysis.analyze_maybe_defined_regs(fn.blocks, cfg, args)
elif name.endswith('_Liveness'):
# Backward, maybe
analysis_result = analysis.analyze_live_regs(fn.blocks, cfg)
elif name.endswith('_MustDefined'):
# Forward, must
analysis_result = analysis.analyze_must_defined_regs(
fn.blocks, cfg, args,
regs=fn.env.regs())
elif name.endswith('_BorrowedArgument'):
# Forward, must
analysis_result = analysis.analyze_borrowed_arguments(fn.blocks, cfg, args)
else:
assert False, 'No recognized _AnalysisName suffix in test case'
actual.append('')
for key in sorted(analysis_result.before.keys(),
key=lambda x: (x[0].label, x[1])):
pre = ', '.join(sorted(reg.name
for reg in analysis_result.before[key]))
post = ', '.join(sorted(reg.name
for reg in analysis_result.after[key]))
actual.append('%-8s %-23s %s' % ((key[0].label, key[1]),
'{%s}' % pre, '{%s}' % post))
assert_test_output(testcase, actual, 'Invalid source code output')
def run_case(self, testcase: DataDrivenTestCase) -> None:
"""Perform a data-flow analysis test case."""
with use_custom_builtins(
os.path.join(test_data_prefix, ICODE_GEN_BUILTINS), testcase):
expected_output = testcase.output
program_text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
source = build.BuildSource('main', '__main__', program_text)
try:
# Construct input as a single single.
# Parse and type check the input program.
result = build.build(sources=[source],
options=options,
alt_lib_path=test_temp_dir)
except CompileError as e:
actual = e.messages
else:
if result.errors:
actual = result.errors
else:
module = genops.build_ir(result.files['__main__'],
result.types)
assert len(
module.functions
) == 1, "Only 1 function definition expected per test case"
fn = module.functions[0]
actual = format_func(fn)
actual = actual[actual.index('L0:'):]
cfg = analysis.get_cfg(fn.blocks)
args = set([Register(i) for i in range(len(fn.args))])
name = testcase.name
if name.endswith('_MaybeDefined'):
# Forward, maybe
analysis_result = analysis.analyze_maybe_defined_regs(
fn.blocks, cfg, args)
elif name.endswith('_Liveness'):
# Backward, maybe
analysis_result = analysis.analyze_live_regs(
fn.blocks, cfg)
elif name.endswith('_MustDefined'):
# Forward, must
analysis_result = analysis.analyze_must_defined_regs(
fn.blocks, cfg, args, num_regs=fn.env.num_regs())
elif name.endswith('_BorrowedArgument'):
# Forward, must
analysis_result = analysis.analyze_borrowed_arguments(
fn.blocks, cfg, args)
else:
assert False, 'No recognized _AnalysisName suffix in test case'
actual.append('')
for key in sorted(analysis_result.before.keys()):
pre = ', '.join(fn.env.names[reg]
for reg in analysis_result.before[key])
post = ', '.join(fn.env.names[reg]
for reg in analysis_result.after[key])
actual.append('%-8s %-23s %s' %
(key, '{%s}' % pre, '{%s}' % post))
assert_string_arrays_equal_wildcards(
expected_output, actual,
'Invalid source code output ({}, line {})'.format(
testcase.file, testcase.line))
