def test_parse_examples(self): code_reader = CodeReader() parser = CppParser() for path in EXAMPLES_CPP14_FILES: code = code_reader.read_file(path) tree = parser.parse(code, path) basic_check_cpp_ast(self, path, tree)
def test_read_file(self): for name, extensions in EXAMPLES_EXTENSIONS.items(): with self.subTest(name=name, extensions=extensions): reader = CodeReader(extensions) for file_path in EXAMPLES_FILES[name]: code = reader.read_file(file_path) self.assertGreater(len(code), 0, msg=file_path)
def test_construct(self): for name, extensions in EXAMPLES_EXTENSIONS.items(): with self.subTest(name=name, extensions=extensions): reader = CodeReader(extensions) self.assertSetEqual(reader.extensions, set(extensions), msg=reader) self.assertIn(str(set(extensions)), str(reader))
def test_parse_examples(self, input_path): code_reader = CodeReader() code = code_reader.read_file(input_path) parser = CppParser() with _TIME.measure('parse.{}'.format(input_path.name.replace('.', '_'))) as timer: cpp_ast = parser.parse(code, input_path) basic_check_cpp_ast(self, input_path, cpp_ast) _LOG.info('parsed "%s" in %fs', input_path, timer.elapsed)
def test_translate_python_to_fortran(self, input_path): reader = CodeReader() python_code = reader.read_file(input_path) parser = TypedPythonParserWithComments(default_mode='exec') tree = parser.parse(python_code, input_path) unparser = Fortran77Unparser() fortran_code = unparser.unparse(tree) basic_check_fortran_code(self, input_path, fortran_code)
def test_read_project_folder(self): for name, extensions in EXAMPLES_EXTENSIONS.items(): with self.subTest(name=name, extensions=extensions): reader = CodeReader(extensions) results = reader.read_folder(_HERE.parent) self.assertGreater(len(results), 0, msg=EXAMPLES_ROOTS[name]) for path, contents in results.items(): self.assertIn(path.suffix, extensions) self.assertIsNotNone(contents)
def test_translate_cpp_to_python(self, input_path): reader = CodeReader() cpp_code = reader.read_file(input_path) language_from = Language.find('C++') language_to = Language.find('Python') translator = AutoTranslator(language_from, language_to, ast_generalizer_kwargs={'scope': {'path': input_path}}) python_code = translator.translate(cpp_code, input_path) basic_check_python_code(self, input_path, python_code)
def test_translate_fortran_to_python(self, input_path): reader = CodeReader() code = reader.read_file(input_path) parser = FortranParser() fortran_ast = parser.parse(code, input_path) generalizer = FortranAstGeneralizer() syntax = generalizer.generalize(fortran_ast) unparser = TypedPythonUnparserWithComments() python_code = unparser.unparse(syntax) basic_check_python_code(self, input_path, python_code)
def test_try_parse_invalid(self): input_path = EXAMPLES_ROOT.joinpath('invalid', 'invalid_cpp.cpp') code_reader = CodeReader() code = code_reader.read_file(input_path) parser = CppParser() with self.assertLogs(level=logging.ERROR): with self.assertRaises(RuntimeError) as err: parser.parse(code, input_path) _LOG.debug('%s', err.exception)
def test_read_folder(self): for (name, extensions), recursive in itertools.product( EXAMPLES_EXTENSIONS.items(), (False, True)): with self.subTest(name=name, extensions=extensions): reader = CodeReader(extensions) results = reader.read_folder(EXAMPLES_ROOTS[name], recursive=recursive) self.assertGreater(len(results), 0, msg=EXAMPLES_ROOTS[name]) for path, contents in results.items(): self.assertIn(path.suffix, extensions) self.assertIsNotNone(contents)
def test_try_compile_invalid(self): input_path = EXAMPLES_ROOT.joinpath('invalid', 'invalid_cpp.cpp') output_dir = make_swig_tmp_folder(input_path) code_reader = CodeReader() code = code_reader.read_file(input_path) compiler = CppSwigCompiler() with self.assertLogs(level=logging.ERROR): with self.assertRaises(RuntimeError) as err: compiler.compile(code, input_path, output_dir) _LOG.debug('%s', err.exception)
def test_translate_python_to_cpp(self, input_path): reader = CodeReader() python_code = reader.read_file(input_path) language_from = Language.find('Python') language_to = Language.find('C++') translator = AutoTranslator(language_from, language_to) with _TIME.measure('translate.python3_to_cpp14.{}' .format(input_path.name.replace('.', '_'))) as timer: cpp_code = translator.translate(python_code) basic_check_cpp_code(self, input_path, cpp_code) _LOG.info('translated "%s" to C++ in %fs', input_path, timer.elapsed)
def test_errors(self): reader = CodeReader({'.py'}) with self.assertRaises(OSError): reader.read_file( pathlib.Path(EXAMPLES_ROOTS['python3'], 'non-existing', 'script.py')) with self.assertRaises(ValueError): reader.read_file(pathlib.Path(EXAMPLES_FILES['cpp14'][0])) with self.assertRaises(OSError): reader.read_folder( pathlib.Path(EXAMPLES_ROOTS['python3'], 'non-existing'))
def test_translate_c_to_python(self, input_path): reader = CodeReader() c_code = reader.read_file(input_path) language_from = Language.find('C11') language_to = Language.find('Python') translator = AutoTranslator(language_from, language_to) with _TIME.measure('translate.c11_to_python3.{}' .format(input_path.name.replace('.', '_'))) as timer: python_code = translator.translate(c_code, input_path) basic_check_python_code(self, input_path, python_code) _LOG.info('translated "%s" to Python in %fs', input_path, timer.elapsed)
def test_python_to_fortran(self): for input_path in EXAMPLES_PY3_FILES: reader = CodeReader() parser = TypedPythonParserWithComments() unparser = Fortran77Unparser() writer = CodeWriter('.f') with self.subTest(input_path=input_path): python_code = reader.read_file(input_path) tree = parser.parse(python_code, input_path, mode='exec') fortran_code = unparser.unparse(tree) writer.write_file(fortran_code, pathlib.Path('/tmp', input_path.name + '.f'))
def test_generalize_examples(self): code_reader = CodeReader() parser = C99Parser() ast_generalizer = CAstGeneralizer() for path in EXAMPLES_C11_FILES: code = code_reader.read_file(path) tree = parser.parse(code, path) basic_check_c_ast(self, path, tree) tree = ast_generalizer.generalize(tree) basic_check_python_ast(self, path, tree) _LOG.debug('%s', typed_astunparse.dump(tree)) _LOG.debug('%s', typed_astunparse.unparse(tree))
def test_generalize_examples(self): code_reader = CodeReader() parser = CppParser() for path in EXAMPLES_CPP14_FILES: ast_generalizer = CppAstGeneralizer(scope={'path': path}) code = code_reader.read_file(path) tree = parser.parse(code, path) basic_check_cpp_ast(self, path, tree) with self.subTest(path=path): tree = ast_generalizer.generalize(tree) basic_check_python_ast(self, path, tree) _LOG.debug('%s', typed_astunparse.dump(tree)) _LOG.debug('%s', typed_astunparse.unparse(tree))
def test_generalize_examples(self, input_path): code_reader = CodeReader() code = code_reader.read_file(input_path) parser = CppParser() cpp_ast = parser.parse(code, input_path) basic_check_cpp_ast(self, input_path, cpp_ast) ast_generalizer = CppAstGeneralizer(scope={'path': input_path}) with _TIME.measure('generalize.{}'.format(input_path.name.replace('.', '_'))) as timer: syntax = ast_generalizer.generalize(cpp_ast) basic_check_python_ast(self, input_path, syntax) _LOG.info('generalized "%s" in %fs', input_path, timer.elapsed) _LOG.debug('%s', typed_astunparse.dump(syntax)) _LOG.debug('%s', typed_astunparse.unparse(syntax))
def test_python_to_cpp(self): language_from = Language.find('Python') language_to = Language.find('C++') reader = CodeReader() root = EXAMPLES_ROOTS['python3'] for input_path in { root.joinpath('do_nothing.py'), root.joinpath('gemm.py') }: translator = AutoTranslator(language_from, language_to) with self.subTest(input_path=input_path): python_code = reader.read_file(input_path) cpp_code = translator.translate(python_code) basic_check_cpp_code(self, input_path, cpp_code)
def test_fortran_to_python(self): for input_path in EXAMPLES_F77_FILES: reader = CodeReader() parser = FortranParser() generalizer = FortranAstGeneralizer() unparser = TypedPythonUnparserWithComments() writer = CodeWriter('.py') with self.subTest(input_path=input_path): code = reader.read_file(input_path) fortran_ast = parser.parse(code, input_path) tree = generalizer.generalize(fortran_ast) python_code = unparser.unparse(tree) writer.write_file( python_code, pathlib.Path('/tmp', input_path.name + '.py'))
def test_auto_parser(self): for language_codename, paths in EXAMPLES_FILES.items(): language_name = EXAMPLES_LANGS_NAMES[language_codename] if language_name in NOT_PARSED_LANGS: continue language = Language.find(language_name) self.assertIsInstance(language, Language, msg=(language_codename, language_name)) auto_parser = GeneralizingAutoParser(language) reader = CodeReader() for path in paths: code = reader.read_file(path) with self.subTest(language_name=language_name, language=language): pair1 = auto_parser.parse_and_generalize(code, path) pair2 = auto_parser.parse_and_generalize_file(path) self.assertEqual(horast.dump(pair1), horast.dump(pair2))
def test_auto_processing(self): for language_codename, paths in EXAMPLES_FILES.items(): language_name = EXAMPLES_LANGS_NAMES[language_codename] if language_name in NOT_PARSED_LANGS: continue language = Language.find(language_name) self.assertIsInstance(language, Language, msg=(language_codename, language_name)) reader = CodeReader() for path in paths: code = reader.read_file(path) with self.subTest(language_name=language_name, language=language): parser = Parser.find(language)() specific_ast = parser.parse(code, path) ast_generalizer = AstGeneralizer.find(language)() general_ast = ast_generalizer.generalize(specific_ast)
def test_run_fundamentals(self, input_path): output_dir = make_f2py_tmp_folder(input_path) code_reader = CodeReader() code = code_reader.read_file(input_path) compiler = F2PyCompiler() output_path = compiler.compile(code, input_path, output_dir) binder = Binder() with binder.temporarily_bind(output_path) as binding: self.assertIsInstance(binding, types.ModuleType) prefix = { 'fundamentals': '', 'fundamentals_arrays': 'itemwise_' }[input_path.stem] shape = None if prefix == '' else (1024 * 1024, ) for type_ in ('integer', 'real'): py_type = {'integer': int, 'real': float}[type_] input1 = random_data(shape, dtype=py_type) input2 = random_data(shape, dtype=py_type) for operation in ('add', 'subtract', 'multiply'): py_operator = { 'add': operator.add, 'subtract': operator.sub, 'multiply': operator.mul }[operation] expected = py_operator(input1, input2) function_name = '{}{}_{}'.format(prefix, operation, type_) function = getattr(binding, function_name) with self.subTest(function=function_name): with _TIME.measure('run.{}.{}.{}'.format( input_path.name.replace('.', '_'), type_, '{}{}'.format(prefix, operation))) as timer: output = function(input1, input2) _LOG.warning('ran %s from "%s" in %fs', function_name, input_path, timer.elapsed) if type_ == 'integer': self.assertTrue(np.all(np.equal(expected, output)), msg=(input1, input2, output, expected)) else: self.assertTrue(np.allclose(expected, output, atol=1e-4), msg=(input1, input2, output, expected))
def test_transpile_with_openmp(self, input_path): output_dir = make_f2py_tmp_folder(input_path) transpiler = AutoTranspiler(Language.find('Python'), Language.find('Fortran')) self.assertIsNotNone(transpiler) reader = CodeReader() with tempfile.NamedTemporaryFile(suffix='.f90', delete=False) as output_file: # TODO: this leaves garbage behind in /tmp/ but is neeeded # by subsequent transpiler passes # code_writer = CodeWriter('.py') # target_inlined_path = pathlib.Path(output_file.name) # code_writer.write_file(target_inlined_code, target_inlined_path) output_path = pathlib.Path(output_file.name) compiled_path = transpiler.transpile( reader.read_file(input_path), input_path, output_path, output_dir)
def test_compile_and_bind_examples(self, input_path): output_dir = make_f2py_tmp_folder(input_path) code_reader = CodeReader() code = code_reader.read_file(input_path) compiler = F2PyCompiler() with _TIME.measure('compile.{}'.format( input_path.name.replace('.', '_'))) as timer: output_path = compiler.compile(code, input_path, output_dir) binder = Binder() with binder.temporarily_bind(output_path) as binding: self.assertIsInstance(binding, types.ModuleType) _LOG.warning('compiled "%s" in %fs', input_path, timer.elapsed) output_path.unlink() try: output_dir.rmdir() except OSError: pass
def test_unparse_examples(self, input_path): code_reader = CodeReader() code = code_reader.read_file(input_path) parser = CppParser() cpp_ast = parser.parse(code, input_path) basic_check_cpp_ast(self, input_path, cpp_ast) generalizer = CppAstGeneralizer(scope={'path': input_path}) syntax = generalizer.generalize(cpp_ast) basic_check_python_ast(self, input_path, syntax) unparser = Cpp14Unparser() with _TIME.measure('unparse.{}'.format(input_path.name.replace('.', '_'))) as timer: code = unparser.unparse(syntax) basic_check_cpp_code(self, input_path, code) _LOG.info('unparsed "%s" in %fs', input_path, timer.elapsed) header_unparser = Cpp14Unparser(headers=True) with _TIME.measure('unparse.{}.headers'.format(input_path.name.replace('.', '_'))) as timer: code = header_unparser.unparse(syntax) basic_check_cpp_code(self, input_path, code, suffix='.hpp') _LOG.info('unparsed "%s" in %fs', input_path, timer.elapsed)
def test_openmp(self): code_reader = CodeReader() binder = Binder() input_path = EXAMPLES_ROOTS['f77'].joinpath('matmul.f') code = code_reader.read_file(input_path) output_dir = make_f2py_tmp_folder(input_path) compiler = F2PyCompiler() output_path = compiler.compile(code, input_path, output_dir) with binder.temporarily_bind(output_path) as binding: self.assertIsInstance(binding, types.ModuleType) with _TIME.measure('run.matmul.simple'): ret_val = binding.intmatmul(20, 1024, 1024) self.assertEqual(ret_val, 0) output_path.unlink() input_path = EXAMPLES_ROOTS['f77'].joinpath('matmul_openmp.f') code = code_reader.read_file(input_path) output_dir = make_f2py_tmp_folder(input_path) compiler_omp = F2PyCompiler(GfortranInterface({'OpenMP'})) output_path = compiler_omp.compile(code, input_path, output_dir) with binder.temporarily_bind(output_path) as binding: self.assertIsInstance(binding, types.ModuleType) with _TIME.measure('run.matmul.openmp'): ret_val = binding.intmatmul(20, 1024, 1024) self.assertEqual(ret_val, 0) timings_name = '.'.join([__name__, 'run', 'matmul']) summary = timing.query_cache(timings_name).summary _LOG.warning('%s', summary) json_to_file(summary, PERFORMANCE_RESULTS_ROOT.joinpath(timings_name + '.json')) output_path.unlink() try: output_dir.rmdir() except OSError: pass
def test_construct_default(self): reader = CodeReader() self.assertIsNotNone(reader) self.assertSetEqual(reader.extensions, set(), msg=reader) self.assertIn(str(set()), str(reader))