def _emulate_yield_from(self, targets: Optional[List[ast.Name]],
node: ast.YieldFrom) -> Iterable[ast.AST]:
generator = ast.Name(
id='_py_backwards_generator_{}'.format(self._name_suffix))
exception = ast.Name(
id='_py_backwards_generator_exception_{}'.format(self._name_suffix))
yield ast.Assign(targets=[generator],
value=ast.Call(func=ast.Name(id='iter'),
args=[node.value],
keywords=[]))
assign_to_targets = [
ast.If(test=ast.Call(func=ast.Name(id='hasattr'), args=[
exception, ast.Str(s='value'),
], keywords=[]), body=[
ast.Assign(targets=targets,
value=ast.Attribute(
value=exception, attr='value')),
], orelse=[]),
ast.Break()] if targets else [ast.Break()]
yield ast.While(test=ast.NameConstant(value=True), body=[
ast.Try(body=[
ast.Expr(value=ast.Yield(value=ast.Call(
func=ast.Name(id='next'),
args=[generator], keywords=[]))),
], handlers=[
ast.ExceptHandler(
type=ast.Name(id='StopIteration'),
name=exception.id,
body=assign_to_targets),
], orelse=[], finalbody=[]),
], orelse=[])
self._name_suffix += 1
def assign_target(self, node, value):
if isinstance(value.tp, EmptyList):
func_node = ast.Expr(value=ast.Call(
func=ast.Attribute(value=node, attr="clear"), args=[]))
try:
self.visit(func_node)
except UnknownVariable:
raise StaticTypeError(
"Must specify a list type when assigning an empty list")
return
if isinstance(node, ast.Name):
left = self.context.assign_type(node.id, value.tp)
self.start_line("{} = {};\n".format(left.code, value.as_value()))
elif isinstance(node, ast.Subscript):
container = self.get_expression_code(node.value)
if isinstance(node.slice, ast.Index):
index = self.get_expression_code(node.slice.value)
setter = container.tp.get_method("set_item")
code = setter.get_code(self.context, container, index, value)
self.start_line(f"{code.code};\n")
else:
raise UnimplementedFeature("Slices not yet implemented")
elif isinstance(node, ast.Attribute):
owner = self.get_expression_code(node.value)
owner.tp.set_attr(node.attr, value)
left = self.get_expression_code(node)
self.start_line(f"{left.code} = {value.code};\n")
def _Class(self, node: ET.Element): # pylint: disable=invalid-name
context = node.attrib['context']
assert context in self.namespaces, context
cls_name = node.attrib['name']
if '<' in cls_name:
_LOG.warning('processing template class %s', cls_name)
assert '>' in cls_name
cls_name, _, rest = cls_name.partition('<')
rest = rest[:-1]
generic_args = [_.strip() for _ in rest.split(',')]
_LOG.warning('found generic args: %s', generic_args)
full_name = '{}::{}'.format(self.namespaces[context], cls_name)
is_stl_class = full_name in CPP_STL_CLASSES and generic_args
value_type = None
body = []
for member_id in node.attrib['members'].split():
if not is_stl_class:
# TODO: handle non-STL classes too
break
if member_id not in self.all_types:
continue
member_type = self.all_types[member_id]
if member_type.tag == 'Typedef' and member_type.attrib[
'name'] == 'value_type':
referenced_id = member_type.attrib['type']
assert referenced_id in self.all_types
if referenced_id not in self.relevant_types \
and referenced_id not in self._new_relevant_types:
self._new_relevant_types[referenced_id] = self.all_types[
referenced_id]
_LOG.debug(
'type marked as relevant due to being container value type %s',
ET.tostring(
self.all_types[referenced_id]).decode().rstrip())
body.append(typed_ast3.Expr(typed_ast3.Str(referenced_id, '')))
value_type = referenced_id
'''
if member_id not in self.relevant_types and member_id not in self._new_relevant_types:
self._new_relevant_types[member_id] = member_type
_LOG.warning('marked %s as relevant type',
ET.tostring(member_type).decode().rstrip())
body.append(typed_ast3.Expr(typed_ast3.Str(member_id, '')))
'''
base_class = typed_ast3.parse(CPP_PYTHON_CLASS_PAIRS[full_name],
mode='eval').body
if is_stl_class:
assert value_type is not None
base_class = typed_ast3.Subscript(
value=base_class,
slice=typed_ast3.Index(typed_ast3.Str(value_type, '')),
ctx=typed_ast3.Load())
return base_class
def fix_stmts_in_body(stmts: t.List[typed_ast3.AST]) -> t.List[typed_ast3.AST]:
assert isinstance(stmts, list)
if not stmts:
return [typed_ast3.Pass()]
return [
typed_ast3.Expr(value=stmt)
if isinstance(stmt, STANDALONE_EXPRESSION_TYPES) else stmt
for stmt in stmts
]
def _show_store_contents_expr(self) -> ast3.Expr:
"""Returns an ast3.Expr which prints the value of the store in the screen. Useful
for debugging.
"""
# print(st)
return ast3.Expr(value=ast3.Call(
func=ast3.Name(id='print', ctx=ast3.Load()),
args=[ast3.Name(id='st', ctx=ast3.Load())],
keywords=[],
), )
def visit_Exec(self, n):
new_globals = self.maybe_visit(n.globals)
if new_globals is None:
new_globals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1)
new_locals = self.maybe_visit(n.locals)
if new_locals is None:
new_locals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1)
return ast3.Expr(ast3.Call(ast3.Name("exec", ast3.Load(), lineno=n.lineno, col_offset=-1),
[self.visit(n.body), new_globals, new_locals],
[],
lineno=n.lineno, col_offset=-1))
def visit_Print(self, n):
keywords = []
if n.dest is not None:
keywords.append(ast3.keyword("file", self.visit(n.dest)))
if not n.nl:
keywords.append(ast3.keyword("end", ast3.Str(" ", lineno=n.lineno, col_offset=-1)))
return ast3.Expr(ast3.Call(ast3.Name("print", ast3.Load(), lineno=n.lineno, col_offset=-1),
self.visit(n.values),
keywords,
lineno=n.lineno, col_offset=-1))
def visit_Expr(self, node: ast3.Expr) -> VisitorOutput:
"""Only the internal parts of an Expr are modified, an Expr keeps being an Expr"""
self.generic_visit(node)
# In console mode ("single" for Python's compile) any expression statement should
# print to console
if self.console:
return ast3.Expr(value=ast3.Call(
func=ast3.Name(id='print_console', ctx=ast3.Load()),
args=[node.value],
keywords=[],
), )
return node
def visit_ListComp(self, node):
from parser.functions import FunctionImplementation
# calculate result type
if len(node.generators) > 1:
raise InvalidOperation(
"Only one for statement permitted in comprehensions")
comp = node.generators[0]
if len(comp.ifs) > 1:
raise InvalidOperation(
"Only one if statement allowed in List Comprehension")
assign_node = ast.Assign(targets=[comp.target],
value=ast.Subscript(value=comp.iter,
slice=ast.Index(
ast.Num(0))))
return_node = ast.Return(value=node.elt)
function_node = ast.FunctionDef(name="temp",
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[assign_node, return_node])
function_interpreter = FunctionImplementation(function_node, (),
self.context)
result_type = TypeDB.get_list([function_interpreter.retval.tp])
# create temp list to hold values
result = self.context.get_temp_var(result_type)
self.prepends.append(
f"{result.code} = {result_type.as_literal([])};\n")
# create for expression
append_node = ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(id=result.code,
ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[node.elt],
keywords=[]))
if comp.ifs:
body = ast.If(test=comp.ifs[0], body=[append_node], orelse=[])
else:
body = append_node
for_node = ast.For(target=comp.target,
iter=comp.iter,
body=[body],
orelse=[])
self.prepends.append(for_node)
return result
def _Function(self, node: ET.Element): # pylint: disable=invalid-name
if not self.types.is_relevant(node):
raise ContinueIteration()
name = node.attrib['name']
arguments = typed_ast3.arguments(
args=self.transform_all_subnodes(node),
vararg=None,
kwonlyargs=[],
kwarg=None,
defaults=[],
kw_defaults=[])
body = [typed_ast3.Expr(value=typed_ast3.Ellipsis())]
returns = self.types.resolved_types[node.attrib['returns']]
return typed_ast3.FunctionDef(name=name,
args=arguments,
body=body,
decorator_list=[],
returns=returns)
def visit_AnnAssign(self, node: ast3.AnnAssign) -> VisitorOutput:
"""Transforms an assignment with annotation into a pytropos type hint assignment.
For example, it converts::
var: ann = expr
into::
`var` = pt.type_hint(`ann`, `expr`)
"""
if node.value is None:
raise AstTransformerError(
f"{self.filename}:{node.lineno}:{node.col_offset}: Fatal Error: "
"Only annotated assignments are allowed (variables with initial values). "
"I.e., no full support for PEP 526 yet. Sorry :(")
pos = pos_as_tuple(node)
# Deleting annotation :S
self.generic_visit(node)
# new_node = ast3.Assign(targets=[node.target], value=node.value)
return ast3.Assign(
targets=[node.target],
value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='pt', ctx=ast3.Load()),
attr='annotation',
ctx=ast3.Load(),
),
args=[
node.annotation, node.value, pos if pos else ast3.Expr(
value=ast3.NameConstant(value=None))
],
keywords=[],
),
)
def test_inline_flash_subset_hydro(self):
app_name = 'FLASH-SUBSET'
if app_name not in _APPS_ROOT_PATHS and app_name in _APPS_OPTIONAL:
self.skipTest('{} directory not found'.format(app_name))
language = Language.find('Fortran')
reader = CodeReader()
parser = Parser.find(language)()
ast_generalizer = AstGeneralizer.find(language)()
f_unparser = Unparser.find(language)()
py_unparser = Unparser.find(Language.find('Python'))()
writer = CodeWriter()
dir_name = app_name.lower()
results_path = pathlib.Path(RESULTS_ROOT, 'transformations', 'inlining', dir_name)
results_path.mkdir(parents=True, exist_ok=True)
path_pairs = [
(pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux_loop.F90'),
pathlib.Path('physics/Hydro/HydroMain/unsplit/hy_upwindTransverseFlux.F90'),
(1, 1)),
(pathlib.Path('physics/Eos/EosMain/Eos_getData_loop1.F90'),
pathlib.Path('physics/Eos/EosMain/Eos_getData.F90'),
(1, 2))]
for inlined_path, target_path, (index, extra_lines) in path_pairs:
inlined_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', inlined_path)
target_path = pathlib.Path(_APPS_ROOT_PATHS[app_name], 'source', target_path)
output_inlined_path = results_path.joinpath(inlined_path.name)
output_target_path = results_path.joinpath(target_path.name)
output_path = results_path.joinpath(target_path.with_suffix('').name + '_inlined.F90')
inlined_xml = parser.parse('', inlined_path)
inlined_xml = inlined_xml.find('.//subroutine')
writer.write_file(ET.tostring(inlined_xml, 'utf-8').decode(),
output_inlined_path.with_suffix('.xml'))
inlined_syntax = ast_generalizer.generalize(inlined_xml)
writer.write_file(typed_astunparse.dump(inlined_syntax),
output_inlined_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(inlined_syntax),
output_inlined_path.with_suffix('.py'))
writer.write_file(f_unparser.unparse(inlined_syntax),
output_inlined_path.with_suffix('.f95'))
target_code = reader.read_file(target_path)
target_xml = parser.parse(target_code, target_path)
# import ipdb; ipdb.set_trace()
target_xml = target_xml.findall('.//call')[index]
writer.write_file(ET.tostring(target_xml, 'utf-8').decode(),
output_target_path.with_suffix('.xml'))
target_syntax = ast_generalizer.generalize(target_xml)
writer.write_file(typed_astunparse.dump(target_syntax),
output_target_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(target_syntax),
output_target_path.with_suffix('.py'))
writer.write_file(f_unparser.unparse(target_syntax),
output_target_path.with_suffix('.f95'))
mock_function = typed_ast3.FunctionDef(
'f', typed_ast3.arguments([], None, [], None, [], []),
[typed_ast3.Expr(target_syntax)], [], None, None)
output_syntax = inline_syntax(mock_function, inlined_syntax, globals_=globals())
output_syntax = st.augment(typed_ast3.Module(output_syntax.body, []), eval_=False)
writer.write_file(typed_astunparse.dump(output_syntax),
output_path.with_suffix('.ast.py'))
writer.write_file(py_unparser.unparse(output_syntax),
output_path.with_suffix('.py'))
output_code = f_unparser.unparse(output_syntax)
writer.write_file(output_code, output_path.with_suffix('.f95'))
_LOG.warning('[%s %s] <- %i', target_xml.attrib['line_begin'],
target_xml.attrib['line_end'], len(output_code))
total_code = replace_scope(
target_code, int(target_xml.attrib['line_begin']),
int(target_xml.attrib['line_end']) + extra_lines, output_code)
writer.write_file(total_code, output_path)
def visit_ImportFrom(self, node: ast3.ImportFrom) -> VisitorOutput:
"""Defines how to import (from) modules (supported and nonsupported)
For example, it converts::
from numpy import array
from numpy import *
from somelib import var, othervar as var2
from otherlib import *
into::
from pytropos.libs_checking import numpy_module
st['array'] = numpy_module.attr['array', pos...]
from pytropos.libs_checking import numpy_module
st.importStar(numpy_module)
st['var'] = pt.Top
st['var2'] = pt.Top
st.importStar()
"""
libs: 'List[ast3.AST]' = []
if node.module in self._supported_modules:
module_name = self._supported_modules[node.module]
# from pytropos.libs_checking import module_name
libs.append(
ast3.ImportFrom(
module='pytropos.libs_checking',
names=[ast3.alias(name=module_name, asname=None)],
level=0,
))
if node.names[0].name == '*':
# st.importStar(module_name)
libs.append(
ast3.Expr(value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='st', ctx=ast3.Load()),
attr='importStar',
ctx=ast3.Load(),
),
args=[ast3.Name(id=module_name, ctx=ast3.Load())],
keywords=[],
), ))
else:
for alias in node.names:
# st['asname'] = modname.attr['name']
pos = pos_as_tuple(node)
if pos is not None:
attrname = ast3.Tuple(
elts=[ast3.Str(s=alias.name), pos],
ctx=ast3.Load()) # type: ast3.expr
else:
attrname = ast3.Str(s=alias.name)
libs.append(
ast3.Assign(
targets=[
ast3.Subscript(
value=ast3.Name(id='st', ctx=ast3.Load()),
slice=ast3.Index(value=ast3.Str(
s=alias.asname if alias.
asname else alias.name), ),
ctx=ast3.Store(),
),
],
value=ast3.Subscript(
value=ast3.Attribute(
value=ast3.Name(id=module_name,
ctx=ast3.Load()),
attr='attr',
ctx=ast3.Load(),
),
slice=ast3.Index(value=attrname, ),
ctx=ast3.Load(),
),
))
else:
if node.names[0].name == '*':
# st.importStar()
libs.append(
ast3.Expr(value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='st', ctx=ast3.Load()),
attr='importStar',
ctx=ast3.Load(),
),
args=[],
keywords=[],
), ))
else:
libs.extend(
ast3.parse( # type: ignore
'\n'.join([
"st['{asname}'] = pt.Top".format(
asname=alias.asname if alias.asname else alias.
name) for alias in node.names
])).body)
return libs
