def generate_c(sources: List[BuildSource], options: Options,
shared_lib_name: Optional[str],
compiler_options: Optional[CompilerOptions] = None
) -> Tuple[List[Tuple[str, str]], str]:
"""Drive the actual core compilation step.
Returns the C source code and (for debugging) the pretty printed IR.
"""
module_names = [source.module for source in sources]
compiler_options = compiler_options or CompilerOptions()
# Do the actual work now
t0 = time.time()
try:
result = emitmodule.parse_and_typecheck(sources, options)
except CompileError as e:
for line in e.messages:
print(line)
fail('Typechecking failure')
t1 = time.time()
if compiler_options.verbose:
print("Parsed and typechecked in {:.3f}s".format(t1 - t0))
ops = [] # type: List[str]
ctext = emitmodule.compile_modules_to_c(result, module_names, shared_lib_name,
compiler_options=compiler_options, ops=ops)
t2 = time.time()
if compiler_options.verbose:
print("Compiled to C in {:.3f}s".format(t2 - t1))
return ctext, '\n'.join(ops)
def run_case(self, testcase: DataDrivenTestCase) -> None:
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
# Build the program.
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
source = build.BuildSource('prog.py', 'prog', text)
try:
result = emitmodule.parse_and_typecheck(
sources=[source],
options=options,
alt_lib_path=test_temp_dir)
cfiles = emitmodule.compile_modules_to_c(
result, module_names=['prog'], multi_file=True, shared_lib_name=None)
out = []
for cfile, ctext in cfiles:
out.append('== {} =='.format(cfile))
out += ctext.splitlines()
except CompileError as e:
out = e.messages
# Verify output.
assert_test_output(testcase, out, 'Invalid output')
def generate_c(sources: List[BuildSource],
options: Options,
groups: emitmodule.Groups,
fscache: FileSystemCache,
compiler_options: CompilerOptions,
) -> Tuple[List[List[Tuple[str, str]]], str]:
"""Drive the actual core compilation step.
The groups argument describes how modules are assigned to C
extension modules. See the comments on the Groups type in
mypyc.emitmodule for details.
Returns the C source code and (for debugging) the pretty printed IR.
"""
t0 = time.time()
# Do the actual work now
serious = False
result = None
try:
result = emitmodule.parse_and_typecheck(
sources, options, compiler_options, groups, fscache)
messages = result.errors
except CompileError as e:
messages = e.messages
if not e.use_stdout:
serious = True
t1 = time.time()
if compiler_options.verbose:
print("Parsed and typechecked in {:.3f}s".format(t1 - t0))
if not messages and result:
errors = Errors()
modules, ctext = emitmodule.compile_modules_to_c(
result, compiler_options=compiler_options, errors=errors, groups=groups)
if errors.num_errors:
messages.extend(errors.new_messages())
t2 = time.time()
if compiler_options.verbose:
print("Compiled to C in {:.3f}s".format(t2 - t1))
# ... you know, just in case.
if options.junit_xml:
py_version = "{}_{}".format(
options.python_version[0], options.python_version[1]
)
write_junit_xml(
t2 - t0, serious, messages, options.junit_xml, py_version, options.platform
)
if messages:
print("\n".join(messages))
sys.exit(1)
return ctext, '\n'.join(format_modules(modules))
def generate_c(
sources: List[BuildSource],
options: Options,
groups: emitmodule.Groups,
compiler_options: Optional[CompilerOptions] = None
) -> Tuple[List[List[Tuple[str, str]]], str]:
"""Drive the actual core compilation step.
The groups argument describes how modules are assigned to C
extension modules. See the comments on the Groups type in
mypyc.emitmodule for details.
Returns the C source code and (for debugging) the pretty printed IR.
"""
compiler_options = compiler_options or CompilerOptions()
# Do the actual work now
t0 = time.time()
try:
result = emitmodule.parse_and_typecheck(sources, options)
except CompileError as e:
for line in e.messages:
print(line)
fail('Typechecking failure')
t1 = time.time()
if compiler_options.verbose:
print("Parsed and typechecked in {:.3f}s".format(t1 - t0))
all_module_names = []
for group_sources, _ in groups:
all_module_names.extend([source.module for source in group_sources])
errors = Errors()
ops = [] # type: List[str]
ctext = emitmodule.compile_modules_to_c(result,
compiler_options=compiler_options,
errors=errors,
ops=ops,
groups=groups)
if errors.num_errors:
errors.flush_errors()
sys.exit(1)
t2 = time.time()
if compiler_options.verbose:
print("Compiled to C in {:.3f}s".format(t2 - t1))
return ctext, '\n'.join(ops)
def run_case_step(self, testcase: DataDrivenTestCase,
incremental_step: int) -> None:
bench = testcase.config.getoption(
'--bench', False) and 'Benchmark' in testcase.name
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
# N.B: We try to (and ought to!) run with the current
# version of python, since we are going to link and run
# against the current version of python.
# But a lot of the tests use type annotations so we can't say it is 3.5.
options.python_version = max(sys.version_info[:2], (3, 6))
options.export_types = True
options.preserve_asts = True
options.incremental = False
# Avoid checking modules/packages named 'unchecked', to provide a way
# to test interacting with code we don't have types for.
options.per_module_options['unchecked.*'] = {'follow_imports': 'error'}
source = build.BuildSource('native.py', 'native', None)
sources = [source]
module_names = ['native']
module_paths = ['native.py']
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other') and fn.endswith('.py'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, None))
to_delete.append(fn)
module_paths.append(fn)
shutil.copyfile(
fn,
os.path.join(os.path.dirname(fn),
name + '_interpreted.py'))
for source in sources:
options.per_module_options.setdefault(source.module,
{})['mypyc'] = True
separate = (self.get_separate('\n'.join(
testcase.input), incremental_step) if self.separate else False)
groups = construct_groups(sources, separate, len(module_names) > 1)
try:
result = emitmodule.parse_and_typecheck(sources=sources,
options=options,
alt_lib_path='.')
errors = Errors()
compiler_options = CompilerOptions(multi_file=self.multi_file,
separate=self.separate)
ir, cfiles = emitmodule.compile_modules_to_c(
result,
compiler_options=compiler_options,
errors=errors,
groups=groups,
)
if errors.num_errors:
errors.flush_errors()
assert False, "Compile error"
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
# Check that serialization works on this IR
check_serialization_roundtrip(ir)
setup_file = os.path.abspath(os.path.join(WORKDIR, 'setup.py'))
# We pass the C file information to the build script via setup.py unfortunately
with open(setup_file, 'w', encoding='utf-8') as f:
f.write(
setup_format.format(module_paths, separate, cfiles,
self.multi_file))
if not run_setup(setup_file, ['build_ext', '--inplace']):
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
assert False, "Compilation failed"
# Assert that an output file got created
suffix = 'pyd' if sys.platform == 'win32' else 'so'
assert glob.glob('native.*.{}'.format(suffix))
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(
['lldb', '--', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(
['gdb', '--args', sys.executable, driver_path], env=env)
assert False, (
"Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen([sys.executable, driver_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
output = proc.communicate()[0].decode('utf8')
outlines = output.splitlines()
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
if incremental_step == 1:
msg = 'Invalid output'
expected = testcase.output
else:
msg = 'Invalid output (step {})'.format(incremental_step)
expected = testcase.output2.get(incremental_step, [])
assert_test_output(testcase, outlines, msg, expected)
if incremental_step > 1 and options.incremental:
suffix = '' if incremental_step == 2 else str(incremental_step - 1)
expected_rechecked = testcase.expected_rechecked_modules.get(
incremental_step - 1)
if expected_rechecked is not None:
assert_module_equivalence('rechecked' + suffix,
expected_rechecked,
result.manager.rechecked_modules)
expected_stale = testcase.expected_stale_modules.get(
incremental_step - 1)
if expected_stale is not None:
assert_module_equivalence('stale' + suffix, expected_stale,
result.manager.stale_modules)
assert proc.returncode == 0
def run_case(self, testcase: DataDrivenTestCase) -> None:
bench = testcase.config.getoption('--bench', False) and 'Benchmark' in testcase.name
# setup.py wants to be run from the root directory of the package, which we accommodate
# by chdiring into tmp/
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase), (
chdir_manager('tmp')):
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
# N.B: We try to (and ought to!) run with the current
# version of python, since we are going to link and run
# against the current version of python.
# But a lot of the tests use type annotations so we can't say it is 3.5.
options.python_version = max(sys.version_info[:2], (3, 6))
options.export_types = True
options.preserve_asts = True
# Avoid checking modules/packages named 'unchecked', to provide a way
# to test interacting with code we don't have types for.
options.per_module_options['unchecked.*'] = {'follow_imports': 'error'}
workdir = 'build'
os.mkdir(workdir)
source_path = 'native.py'
with open(source_path, 'w', encoding='utf-8') as f:
f.write(text)
with open('interpreted.py', 'w', encoding='utf-8') as f:
f.write(text)
shutil.copyfile(TESTUTIL_PATH, 'testutil.py')
source = build.BuildSource(source_path, 'native', text)
sources = [source]
module_names = ['native']
module_paths = [os.path.abspath('native.py')]
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, text))
to_delete.append(fn)
module_paths.append(os.path.abspath(fn))
shutil.copyfile(fn,
os.path.join(os.path.dirname(fn), name + '_interpreted.py'))
for source in sources:
options.per_module_options.setdefault(source.module, {})['mypyc'] = True
if len(module_names) == 1:
lib_name = None # type: Optional[str]
else:
lib_name = shared_lib_name([source.module for source in sources])
try:
result = emitmodule.parse_and_typecheck(
sources=sources,
options=options,
alt_lib_path='.')
errors = Errors()
compiler_options = CompilerOptions(multi_file=self.multi_file)
cfiles = emitmodule.compile_modules_to_c(
result,
module_names=module_names,
shared_lib_name=lib_name,
compiler_options=compiler_options,
errors=errors,
)
if errors.num_errors:
errors.flush_errors()
assert False, "Compile error"
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
for cfile, ctext in cfiles:
with open(os.path.join(workdir, cfile), 'w', encoding='utf-8') as f:
f.write(ctext)
setup_file = os.path.abspath(os.path.join(workdir, 'setup.py'))
with open(setup_file, 'w') as f:
f.write(setup_format.format(module_paths))
if not run_setup(setup_file, ['build_ext', '--inplace']):
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
assert False, "Compilation failed"
# Assert that an output file got created
suffix = 'pyd' if sys.platform == 'win32' else 'so'
assert glob.glob('native.*.{}'.format(suffix))
for p in to_delete:
os.remove(p)
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(['lldb', '--', sys.executable, driver_path], env=env)
assert False, ("Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(['gdb', '--args', sys.executable, driver_path], env=env)
assert False, ("Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen([sys.executable, driver_path], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=env)
output = proc.communicate()[0].decode('utf8')
outlines = output.splitlines()
if testcase.config.getoption('--mypyc-showc'):
show_c(cfiles)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
assert_test_output(testcase, outlines, 'Invalid output')
assert proc.returncode == 0
def run_case(self, testcase: DataDrivenTestCase) -> None:
bench = testcase.config.getoption('--bench', False) and 'Benchmark' in testcase.name
# setup.py wants to be run from the root directory of the package, which we accommodate
# by chdiring into tmp/
with use_custom_builtins(os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase), (
chdir_manager('tmp')):
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
workdir = 'build'
os.mkdir(workdir)
source_path = 'native.py'
with open(source_path, 'w') as f:
f.write(text)
with open('interpreted.py', 'w') as f:
f.write(text)
source = build.BuildSource(source_path, 'native', text)
sources = [source]
module_names = ['native']
module_paths = [os.path.abspath('native.py')]
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
fn = os.path.relpath(fn, test_temp_dir)
if os.path.basename(fn).startswith('other'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, text))
to_delete.append(fn)
module_paths.append(os.path.abspath(fn))
for source in sources:
options.per_module_options.setdefault(source.module, {})['mypyc'] = True
try:
result = emitmodule.parse_and_typecheck(
sources=sources,
options=options,
alt_lib_path='.')
ctext = emitmodule.compile_modules_to_c(
result,
module_names=module_names,
use_shared_lib=len(module_names) > 1)
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
cpath = os.path.abspath(os.path.join(workdir, '__native.c'))
with open(cpath, 'w') as f:
f.write(ctext)
setup_file = os.path.abspath(os.path.join(workdir, 'setup.py'))
with open(setup_file, 'w') as f:
f.write(setup_format.format(module_paths))
run_setup(setup_file, ['build_ext', '--inplace'])
# Oh argh run_setup doesn't propagate failure. For now we'll just assert
# that the file is there.
if not glob.glob('native.*.so'):
show_c(ctext)
assert False, "Compilation failed"
for p in to_delete:
os.remove(p)
driver_path = 'driver.py'
env = os.environ.copy()
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(['lldb', '--', 'python', driver_path], env=env)
assert False, ("Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(['gdb', '--args', 'python', driver_path], env=env)
assert False, ("Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen(['python', driver_path], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, env=env)
output, _ = proc.communicate()
output = output.decode('utf8')
outlines = output.splitlines()
show_c(ctext)
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
assert_test_output(testcase, outlines, 'Invalid output')
assert proc.returncode == 0
def run_case(self, testcase: DataDrivenTestCase) -> None:
bench = testcase.config.getoption(
'--bench', False) and 'Benchmark' in testcase.name
with use_custom_builtins(
os.path.join(self.data_prefix, ICODE_GEN_BUILTINS), testcase):
text = '\n'.join(testcase.input)
options = Options()
options.use_builtins_fixtures = True
options.show_traceback = True
options.strict_optional = True
options.python_version = (3, 6)
options.export_types = True
workdir = 'build'
os.mkdir(workdir)
os.mkdir('tmp/py')
source_path = 'tmp/py/native.py'
with open(source_path, 'w') as f:
f.write(text)
with open('tmp/interpreted.py', 'w') as f:
f.write(text)
source = build.BuildSource(source_path, 'native', text)
sources = [source]
module_names = ['native']
# Hard code another module name to compile in the same compilation unit.
to_delete = []
for fn, text in testcase.files:
if os.path.basename(fn).startswith('other'):
name = os.path.basename(fn).split('.')[0]
module_names.append(name)
sources.append(build.BuildSource(fn, name, text))
to_delete.append(fn)
try:
result = emitmodule.parse_and_typecheck(
sources=sources,
options=options,
alt_lib_path=test_temp_dir)
ctext = emitmodule.compile_modules_to_c(
result,
module_names=module_names,
use_shared_lib=len(module_names) > 1)
except CompileError as e:
for line in e.messages:
print(line)
assert False, 'Compile error'
# If compiling more than one native module, compile a shared
# library that contains all the modules. Also generate shims that
# just call into the shared lib.
use_shared_lib = len(module_names) > 1
if use_shared_lib:
common_path = os.path.abspath(
os.path.join(test_temp_dir, '__shared_stuff.c'))
with open(common_path, 'w') as f:
f.write(ctext)
try:
shared_lib = buildc.build_shared_lib_for_modules(
common_path, module_names, workdir)
except buildc.BuildError as err:
show_c_error(common_path, err.output)
raise
for mod in module_names:
cpath = os.path.abspath(
os.path.join(test_temp_dir, '%s.c' % mod))
with open(cpath, 'w') as f:
f.write(ctext)
try:
if use_shared_lib:
native_lib_path = buildc.build_c_extension_shim(
mod, shared_lib, workdir)
else:
native_lib_path = buildc.build_c_extension(
cpath, mod, workdir)
except buildc.BuildError as err:
show_c_error(cpath, err.output)
raise
# # TODO: is the location of the shared lib good?
# shared_lib = buildc.build_shared_lib_for_modules(cpath)
for p in to_delete:
os.remove(p)
driver_path = os.path.join(test_temp_dir, 'driver.py')
env = os.environ.copy()
path = [
os.path.dirname(native_lib_path),
os.path.join(PREFIX, 'extensions')
]
env['PYTHONPATH'] = ':'.join(path)
env['MYPYC_RUN_BENCH'] = '1' if bench else '0'
lib_env = 'DYLD_LIBRARY_PATH' if sys.platform == 'darwin' else 'LD_LIBRARY_PATH'
env[lib_env] = workdir
# XXX: This is an ugly hack.
if 'MYPYC_RUN_GDB' in os.environ:
if platform.system() == 'Darwin':
subprocess.check_call(
['lldb', '--', 'python', driver_path], env=env)
assert False, (
"Test can't pass in lldb mode. (And remember to pass -s to "
"pytest)")
elif platform.system() == 'Linux':
subprocess.check_call(
['gdb', '--args', 'python', driver_path], env=env)
assert False, (
"Test can't pass in gdb mode. (And remember to pass -s to "
"pytest)")
else:
assert False, 'Unsupported OS'
proc = subprocess.Popen(['python', driver_path],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env)
output, _ = proc.communicate()
output = output.decode('utf8')
outlines = output.splitlines()
heading('Generated C')
with open(cpath) as f:
print(f.read().rstrip())
heading('End C')
if proc.returncode != 0:
print()
print('*** Exit status: %d' % proc.returncode)
# Verify output.
if bench:
print('Test output:')
print(output)
else:
assert_test_output(testcase, outlines, 'Invalid output')
assert proc.returncode == 0
