def test_errors(self):
reader = CodeReader({'.py'})
with self.assertRaises(OSError):
reader.read_file(
pathlib.Path(EXAMPLES_ROOTS['python3'], 'non-existing',
'script.py'))
with self.assertRaises(ValueError):
reader.read_file(pathlib.Path(EXAMPLES_FILES['cpp14'][0]))
with self.assertRaises(OSError):
reader.read_folder(
pathlib.Path(EXAMPLES_ROOTS['python3'], 'non-existing'))
def test_read_file(self):
for name, extensions in EXAMPLES_EXTENSIONS.items():
with self.subTest(name=name, extensions=extensions):
reader = CodeReader(extensions)
for file_path in EXAMPLES_FILES[name]:
code = reader.read_file(file_path)
self.assertGreater(len(code), 0, msg=file_path)
def test_parse_examples(self):
code_reader = CodeReader()
parser = CppParser()
for path in EXAMPLES_CPP14_FILES:
code = code_reader.read_file(path)
tree = parser.parse(code, path)
basic_check_cpp_ast(self, path, tree)
def test_parse_examples(self, input_path):
code_reader = CodeReader()
code = code_reader.read_file(input_path)
parser = CppParser()
with _TIME.measure('parse.{}'.format(input_path.name.replace('.', '_'))) as timer:
cpp_ast = parser.parse(code, input_path)
basic_check_cpp_ast(self, input_path, cpp_ast)
_LOG.info('parsed "%s" in %fs', input_path, timer.elapsed)
def test_translate_python_to_fortran(self, input_path):
reader = CodeReader()
python_code = reader.read_file(input_path)
parser = TypedPythonParserWithComments(default_mode='exec')
tree = parser.parse(python_code, input_path)
unparser = Fortran77Unparser()
fortran_code = unparser.unparse(tree)
basic_check_fortran_code(self, input_path, fortran_code)
def test_translate_cpp_to_python(self, input_path):
reader = CodeReader()
cpp_code = reader.read_file(input_path)
language_from = Language.find('C++')
language_to = Language.find('Python')
translator = AutoTranslator(language_from, language_to,
ast_generalizer_kwargs={'scope': {'path': input_path}})
python_code = translator.translate(cpp_code, input_path)
basic_check_python_code(self, input_path, python_code)
def test_translate_fortran_to_python(self, input_path):
reader = CodeReader()
code = reader.read_file(input_path)
parser = FortranParser()
fortran_ast = parser.parse(code, input_path)
generalizer = FortranAstGeneralizer()
syntax = generalizer.generalize(fortran_ast)
unparser = TypedPythonUnparserWithComments()
python_code = unparser.unparse(syntax)
basic_check_python_code(self, input_path, python_code)
def test_try_parse_invalid(self):
input_path = EXAMPLES_ROOT.joinpath('invalid', 'invalid_cpp.cpp')
code_reader = CodeReader()
code = code_reader.read_file(input_path)
parser = CppParser()
with self.assertLogs(level=logging.ERROR):
with self.assertRaises(RuntimeError) as err:
parser.parse(code, input_path)
_LOG.debug('%s', err.exception)
def test_try_compile_invalid(self):
input_path = EXAMPLES_ROOT.joinpath('invalid', 'invalid_cpp.cpp')
output_dir = make_swig_tmp_folder(input_path)
code_reader = CodeReader()
code = code_reader.read_file(input_path)
compiler = CppSwigCompiler()
with self.assertLogs(level=logging.ERROR):
with self.assertRaises(RuntimeError) as err:
compiler.compile(code, input_path, output_dir)
_LOG.debug('%s', err.exception)
def test_translate_python_to_cpp(self, input_path):
reader = CodeReader()
python_code = reader.read_file(input_path)
language_from = Language.find('Python')
language_to = Language.find('C++')
translator = AutoTranslator(language_from, language_to)
with _TIME.measure('translate.python3_to_cpp14.{}'
.format(input_path.name.replace('.', '_'))) as timer:
cpp_code = translator.translate(python_code)
basic_check_cpp_code(self, input_path, cpp_code)
_LOG.info('translated "%s" to C++ in %fs', input_path, timer.elapsed)
def test_translate_c_to_python(self, input_path):
reader = CodeReader()
c_code = reader.read_file(input_path)
language_from = Language.find('C11')
language_to = Language.find('Python')
translator = AutoTranslator(language_from, language_to)
with _TIME.measure('translate.c11_to_python3.{}'
.format(input_path.name.replace('.', '_'))) as timer:
python_code = translator.translate(c_code, input_path)
basic_check_python_code(self, input_path, python_code)
_LOG.info('translated "%s" to Python in %fs', input_path, timer.elapsed)
def test_generalize_examples(self):
code_reader = CodeReader()
parser = C99Parser()
ast_generalizer = CAstGeneralizer()
for path in EXAMPLES_C11_FILES:
code = code_reader.read_file(path)
tree = parser.parse(code, path)
basic_check_c_ast(self, path, tree)
tree = ast_generalizer.generalize(tree)
basic_check_python_ast(self, path, tree)
_LOG.debug('%s', typed_astunparse.dump(tree))
_LOG.debug('%s', typed_astunparse.unparse(tree))
def test_python_to_fortran(self):
for input_path in EXAMPLES_PY3_FILES:
reader = CodeReader()
parser = TypedPythonParserWithComments()
unparser = Fortran77Unparser()
writer = CodeWriter('.f')
with self.subTest(input_path=input_path):
python_code = reader.read_file(input_path)
tree = parser.parse(python_code, input_path, mode='exec')
fortran_code = unparser.unparse(tree)
writer.write_file(fortran_code,
pathlib.Path('/tmp', input_path.name + '.f'))
def test_generalize_examples(self):
code_reader = CodeReader()
parser = CppParser()
for path in EXAMPLES_CPP14_FILES:
ast_generalizer = CppAstGeneralizer(scope={'path': path})
code = code_reader.read_file(path)
tree = parser.parse(code, path)
basic_check_cpp_ast(self, path, tree)
with self.subTest(path=path):
tree = ast_generalizer.generalize(tree)
basic_check_python_ast(self, path, tree)
_LOG.debug('%s', typed_astunparse.dump(tree))
_LOG.debug('%s', typed_astunparse.unparse(tree))
def test_generalize_examples(self, input_path):
code_reader = CodeReader()
code = code_reader.read_file(input_path)
parser = CppParser()
cpp_ast = parser.parse(code, input_path)
basic_check_cpp_ast(self, input_path, cpp_ast)
ast_generalizer = CppAstGeneralizer(scope={'path': input_path})
with _TIME.measure('generalize.{}'.format(input_path.name.replace('.', '_'))) as timer:
syntax = ast_generalizer.generalize(cpp_ast)
basic_check_python_ast(self, input_path, syntax)
_LOG.info('generalized "%s" in %fs', input_path, timer.elapsed)
_LOG.debug('%s', typed_astunparse.dump(syntax))
_LOG.debug('%s', typed_astunparse.unparse(syntax))
def test_fortran_to_python(self):
for input_path in EXAMPLES_F77_FILES:
reader = CodeReader()
parser = FortranParser()
generalizer = FortranAstGeneralizer()
unparser = TypedPythonUnparserWithComments()
writer = CodeWriter('.py')
with self.subTest(input_path=input_path):
code = reader.read_file(input_path)
fortran_ast = parser.parse(code, input_path)
tree = generalizer.generalize(fortran_ast)
python_code = unparser.unparse(tree)
writer.write_file(
python_code, pathlib.Path('/tmp', input_path.name + '.py'))
def test_python_to_cpp(self):
language_from = Language.find('Python')
language_to = Language.find('C++')
reader = CodeReader()
root = EXAMPLES_ROOTS['python3']
for input_path in {
root.joinpath('do_nothing.py'),
root.joinpath('gemm.py')
}:
translator = AutoTranslator(language_from, language_to)
with self.subTest(input_path=input_path):
python_code = reader.read_file(input_path)
cpp_code = translator.translate(python_code)
basic_check_cpp_code(self, input_path, cpp_code)
def test_openmp(self):
code_reader = CodeReader()
binder = Binder()
input_path = EXAMPLES_ROOTS['f77'].joinpath('matmul.f')
code = code_reader.read_file(input_path)
output_dir = make_f2py_tmp_folder(input_path)
compiler = F2PyCompiler()
output_path = compiler.compile(code, input_path, output_dir)
with binder.temporarily_bind(output_path) as binding:
self.assertIsInstance(binding, types.ModuleType)
with _TIME.measure('run.matmul.simple'):
ret_val = binding.intmatmul(20, 1024, 1024)
self.assertEqual(ret_val, 0)
output_path.unlink()
input_path = EXAMPLES_ROOTS['f77'].joinpath('matmul_openmp.f')
code = code_reader.read_file(input_path)
output_dir = make_f2py_tmp_folder(input_path)
compiler_omp = F2PyCompiler(GfortranInterface({'OpenMP'}))
output_path = compiler_omp.compile(code, input_path, output_dir)
with binder.temporarily_bind(output_path) as binding:
self.assertIsInstance(binding, types.ModuleType)
with _TIME.measure('run.matmul.openmp'):
ret_val = binding.intmatmul(20, 1024, 1024)
self.assertEqual(ret_val, 0)
timings_name = '.'.join([__name__, 'run', 'matmul'])
summary = timing.query_cache(timings_name).summary
_LOG.warning('%s', summary)
json_to_file(summary,
PERFORMANCE_RESULTS_ROOT.joinpath(timings_name + '.json'))
output_path.unlink()
try:
output_dir.rmdir()
except OSError:
pass
def test_auto_parser(self):
for language_codename, paths in EXAMPLES_FILES.items():
language_name = EXAMPLES_LANGS_NAMES[language_codename]
if language_name in NOT_PARSED_LANGS:
continue
language = Language.find(language_name)
self.assertIsInstance(language, Language, msg=(language_codename, language_name))
auto_parser = GeneralizingAutoParser(language)
reader = CodeReader()
for path in paths:
code = reader.read_file(path)
with self.subTest(language_name=language_name, language=language):
pair1 = auto_parser.parse_and_generalize(code, path)
pair2 = auto_parser.parse_and_generalize_file(path)
self.assertEqual(horast.dump(pair1), horast.dump(pair2))
def test_auto_processing(self):
for language_codename, paths in EXAMPLES_FILES.items():
language_name = EXAMPLES_LANGS_NAMES[language_codename]
if language_name in NOT_PARSED_LANGS:
continue
language = Language.find(language_name)
self.assertIsInstance(language, Language, msg=(language_codename, language_name))
reader = CodeReader()
for path in paths:
code = reader.read_file(path)
with self.subTest(language_name=language_name, language=language):
parser = Parser.find(language)()
specific_ast = parser.parse(code, path)
ast_generalizer = AstGeneralizer.find(language)()
general_ast = ast_generalizer.generalize(specific_ast)
def test_run_fundamentals(self, input_path):
output_dir = make_f2py_tmp_folder(input_path)
code_reader = CodeReader()
code = code_reader.read_file(input_path)
compiler = F2PyCompiler()
output_path = compiler.compile(code, input_path, output_dir)
binder = Binder()
with binder.temporarily_bind(output_path) as binding:
self.assertIsInstance(binding, types.ModuleType)
prefix = {
'fundamentals': '',
'fundamentals_arrays': 'itemwise_'
}[input_path.stem]
shape = None if prefix == '' else (1024 * 1024, )
for type_ in ('integer', 'real'):
py_type = {'integer': int, 'real': float}[type_]
input1 = random_data(shape, dtype=py_type)
input2 = random_data(shape, dtype=py_type)
for operation in ('add', 'subtract', 'multiply'):
py_operator = {
'add': operator.add,
'subtract': operator.sub,
'multiply': operator.mul
}[operation]
expected = py_operator(input1, input2)
function_name = '{}{}_{}'.format(prefix, operation, type_)
function = getattr(binding, function_name)
with self.subTest(function=function_name):
with _TIME.measure('run.{}.{}.{}'.format(
input_path.name.replace('.', '_'), type_,
'{}{}'.format(prefix, operation))) as timer:
output = function(input1, input2)
_LOG.warning('ran %s from "%s" in %fs', function_name,
input_path, timer.elapsed)
if type_ == 'integer':
self.assertTrue(np.all(np.equal(expected, output)),
msg=(input1, input2, output,
expected))
else:
self.assertTrue(np.allclose(expected,
output,
atol=1e-4),
msg=(input1, input2, output,
expected))
def test_transpile_with_openmp(self, input_path):
output_dir = make_f2py_tmp_folder(input_path)
transpiler = AutoTranspiler(Language.find('Python'), Language.find('Fortran'))
self.assertIsNotNone(transpiler)
reader = CodeReader()
with tempfile.NamedTemporaryFile(suffix='.f90', delete=False) as output_file:
# TODO: this leaves garbage behind in /tmp/ but is neeeded
# by subsequent transpiler passes
# code_writer = CodeWriter('.py')
# target_inlined_path = pathlib.Path(output_file.name)
# code_writer.write_file(target_inlined_code, target_inlined_path)
output_path = pathlib.Path(output_file.name)
compiled_path = transpiler.transpile(
reader.read_file(input_path), input_path, output_path, output_dir)
def test_compile_and_bind_examples(self, input_path):
output_dir = make_f2py_tmp_folder(input_path)
code_reader = CodeReader()
code = code_reader.read_file(input_path)
compiler = F2PyCompiler()
with _TIME.measure('compile.{}'.format(
input_path.name.replace('.', '_'))) as timer:
output_path = compiler.compile(code, input_path, output_dir)
binder = Binder()
with binder.temporarily_bind(output_path) as binding:
self.assertIsInstance(binding, types.ModuleType)
_LOG.warning('compiled "%s" in %fs', input_path, timer.elapsed)
output_path.unlink()
try:
output_dir.rmdir()
except OSError:
pass
def test_unparse_examples(self, input_path):
code_reader = CodeReader()
code = code_reader.read_file(input_path)
parser = CppParser()
cpp_ast = parser.parse(code, input_path)
basic_check_cpp_ast(self, input_path, cpp_ast)
generalizer = CppAstGeneralizer(scope={'path': input_path})
syntax = generalizer.generalize(cpp_ast)
basic_check_python_ast(self, input_path, syntax)
unparser = Cpp14Unparser()
with _TIME.measure('unparse.{}'.format(input_path.name.replace('.', '_'))) as timer:
code = unparser.unparse(syntax)
basic_check_cpp_code(self, input_path, code)
_LOG.info('unparsed "%s" in %fs', input_path, timer.elapsed)
header_unparser = Cpp14Unparser(headers=True)
with _TIME.measure('unparse.{}.headers'.format(input_path.name.replace('.', '_'))) as timer:
code = header_unparser.unparse(syntax)
basic_check_cpp_code(self, input_path, code, suffix='.hpp')
_LOG.info('unparsed "%s" in %fs', input_path, timer.elapsed)
