def _CastXML(self, node: ET.Element): # pylint: disable=invalid-name
self.file_id = self._determine_file_id(node)
self.find_types(node)
while self._new_relevant_types:
_LOG.debug('there are %i new relevant types',
len(self._new_relevant_types))
self.relevant_types.update(self._new_relevant_types)
self._new_relevant_types = {}
new_resolved_types = {}
for id_, node_ in self.relevant_types.items():
if id_ in self.resolved_types:
continue
new_resolved_types[id_] = self.transform_one(node_)
_LOG.debug('resolved %s into %s',
ET.tostring(node_).decode().rstrip(),
horast.unparse(new_resolved_types[id_]))
self.resolved_types.update(new_resolved_types)
self._fix_resolved_types(self.resolved_types)
_LOG.info(
'type resolution complete: %s',
pprint.pformat({
_: horast.unparse(type_).strip()
for _, type_ in self.resolved_types.items()
}))
def _Call(self, t):
func_name = horast.unparse(t.func).strip()
if func_name == 'np.zeros':
self._includes['valarray'] = True
self.write('std::valarray<')
assert len(t.keywords) == 1, len(t.keywords)
assert t.keywords[0].arg == 'dtype'
self.dispatch_type(t.keywords[0].value)
self.write('>(')
comma = False
for arg in t.args:
if comma:
self.write(",")
else:
comma = True
self.dispatch(arg)
return
if t.keywords:
self._unsupported_syntax(t, 'with keyword arguments')
if func_name == 'print':
self._includes['iostream'] = True
self.write('std::cout << ')
comma = False
for arg in itertools.chain(t.args, t.keywords):
if comma:
self.write(" << ")
else:
comma = True
self.dispatch(arg)
return
super()._Call(t)
def inline(target_function, inlined_function, globals_=None) -> object:
"""Inline all calls to given inlined function within the target.
Can be used as decorator.
"""
assert isinstance(target_function, types.FunctionType)
assert isinstance(inlined_function, types.FunctionType)
language = Language.find('Python 3')
parser = Parser.find(language)()
target_code = CodeReader.read_function(target_function)
inlined_code = CodeReader.read_function(inlined_function)
target_syntax = parser.parse(target_code).body[0]
inlined_syntax = parser.parse(inlined_code).body[0]
target_inlined_syntax = inline_syntax(target_syntax, inlined_syntax, globals_=globals_,
verbose=False)
target_inlined_code = horast.unparse(target_inlined_syntax).lstrip()
with tempfile.NamedTemporaryFile(suffix='.py', delete=False) as output_file:
# TODO: this leaves garbage behind in /tmp/ but is neeeded by subsequent transpiler passes
code_writer = CodeWriter('.py')
target_inlined_path = pathlib.Path(output_file.name)
code_writer.write_file(target_inlined_code, target_inlined_path)
code_obj = compile(target_inlined_code, filename=str(target_inlined_path), mode='exec')
if globals_ is None:
globals_ = {'__builtins__': globals()['__builtins__']}
locals_ = {}
eval_result = eval(code_obj, globals_, locals_)
assert eval_result is None, eval_result
assert target_function.__name__ in locals_
target_inlined_function = locals_[target_function.__name__]
assert isinstance(target_inlined_function, types.FunctionType)
return target_inlined_function
def dispatch_var_type(self, tree):
code = horast.unparse(tree)
stripped_code = code.strip()
if stripped_code in PYTHON_FORTRAN_TYPE_PAIRS:
type_name, precision = PYTHON_FORTRAN_TYPE_PAIRS[stripped_code]
self.write(type_name)
if precision is not None:
self.write('*')
self.write(str(precision))
elif _match_array(tree):
sli = tree.slice
assert isinstance(sli,
typed_ast3.Index), typed_astunparse.dump(tree)
assert isinstance(sli.value, typed_ast3.Tuple)
assert len(sli.value.elts) in (2, 3), sli.value.elts
elts = sli.value.elts
self.dispatch_var_type(elts[1])
self.write(', ')
self.write('dimension(')
if len(sli.value.elts) == 2:
self.write(':')
else:
if not self._context_input_args:
self.dispatch(elts[2])
else:
assert isinstance(elts[2], typed_ast3.Tuple)
_LOG.warning('coercing indices of %i dims to 0-based',
len(elts[2].elts))
# _LOG.warning('coercing indices of %s in %s to 0-based', arg.arg, t.name)
tup_elts = []
for elt in elts[2].elts:
if isinstance(elt, typed_ast3.Num):
assert isinstance(elt.n, int)
upper = typed_ast3.Num(n=elt.n - 1)
else:
assert isinstance(elt, typed_ast3.Name)
upper = typed_ast3.BinOp(left=elt,
op=typed_ast3.Sub(),
right=typed_ast3.Num(n=1))
tup_elts.append(
typed_ast3.Slice(lower=typed_ast3.Num(n=0),
upper=upper,
step=None))
tup = typed_ast3.Tuple(elts=tup_elts,
ctx=typed_ast3.Load())
self.dispatch(tup)
self.write(')')
elif _match_io(tree):
self.write('integer')
elif isinstance(tree, typed_ast3.Call) and isinstance(tree.func, typed_ast3.Name) \
and tree.func.id == 'type':
self.dispatch(tree)
else:
raise NotImplementedError('not yet implemented: {}'.format(
typed_astunparse.dump(tree)))
def tree_to_str(tree: Module) -> str:
# TODO why this fails?
# validator.visit(tree)
fix_missing_locations(tree)
generated_code: str = unparse(tree)
generated_code = autopep8.fix_code(generated_code,
options={'aggressive': 1})
return generated_code
def dispatch_type(self, type_hint):
_LOG.debug('dispatching type hint %s', type_hint)
if is_generic_type(type_hint):
if type_hint.value.attr in CPP_GENERIC_TYPE_INCLUDES:
self._includes[CPP_GENERIC_TYPE_INCLUDES[
type_hint.value.attr]] = True
self.dispatch_type(type_hint.value)
self.write("<")
self.dispatch_type(type_hint.slice)
self.write(">")
return
if is_pointer(type_hint):
# _LOG.warning('dispatching pointer type %s', horast.unparse(type_hint).strip())
assert isinstance(type_hint.slice,
typed_ast3.Index), type(type_hint.slice)
if isinstance(type_hint.slice.value, typed_ast3.Name) \
and type_hint.slice.value.id == 'str':
self.write('char')
else:
self.dispatch_type(type_hint.slice.value)
self.write('*')
return
if isinstance(type_hint, typed_ast3.Subscript):
_LOG.error('encountered unsupported subscript form: %s',
horast.unparse(type_hint).strip())
self._unsupported_syntax(type_hint)
if isinstance(type_hint, typed_ast3.Attribute):
if isinstance(type_hint.value, typed_ast3.Name):
unparsed = horast.unparse(type_hint).strip()
self.write(PY_TO_CPP_TYPES[unparsed])
return
_LOG.error('encountered unsupported attribute form: %s',
horast.unparse(type_hint).strip())
self._unsupported_syntax(type_hint)
if isinstance(type_hint, typed_ast3.NameConstant):
assert type_hint.value is None
self.write('void')
return
self.dispatch(type_hint)
def unparsing_unsupported(language_name: str, syntax, comment: str = None, error: bool = True):
unparsed = 'invalid'
try:
unparsed = '"""{}"""'.format(horast.unparse(syntax).strip())
except AttributeError:
pass
if comment is not None:
comment = ' ' + comment
_LOG.error('unparsing %s%s like """%s""" (%s in Python) is unsupported for %s',
syntax.__class__.__name__, comment, horast.dump(syntax), unparsed, language_name)
if error:
raise SyntaxError(
'unparsing {}{} like """{}""" ({} in Python) is unsupported for {}'.format(
syntax.__class__.__name__, comment, horast.dump(syntax), unparsed, language_name))
def dispatch_type(self, type_hint):
_LOG.debug('dispatching type hint %s', type_hint)
if isinstance(type_hint, typed_ast3.Subscript):
if isinstance(type_hint.value, typed_ast3.Attribute) \
and isinstance(type_hint.value.value, typed_ast3.Name):
unparsed = horast.unparse(type_hint.value).strip()
self.write(PY_TO_CPP_TYPES[unparsed])
if unparsed == 'st.ndarray':
self.write('<')
sli = type_hint.slice
self.write('>')
return
self._unsupported_syntax(type_hint)
if isinstance(type_hint, typed_ast3.Attribute):
if isinstance(type_hint.value, typed_ast3.Name):
unparsed = horast.unparse(type_hint).strip()
self.write(PY_TO_CPP_TYPES[unparsed])
return
self._unsupported_syntax(type_hint)
if isinstance(type_hint, typed_ast3.NameConstant):
assert type_hint.value is None
self.write('void')
return
self.dispatch(type_hint)
def test_inline_syntax(self):
language = Language.find('Python 3')
parser = Parser.find(language)()
for (target, inlined), target_inlined in INLINING_EXAMPLES.items():
target_code = CodeReader.read_function(target)
inlined_code = CodeReader.read_function(inlined)
reference_code = CodeReader.read_function(target_inlined)
target_syntax = parser.parse(target_code).body[0]
inlined_syntax = parser.parse(inlined_code).body[0]
with self.subTest(target=target, inlined=inlined):
target_inlined_syntax = inline_syntax(target_syntax, inlined_syntax, verbose=False)
target_inlined_code = horast.unparse(target_inlined_syntax)
_LOG.warning('%s', target_inlined_code)
self.assertEqual(reference_code.replace('_inlined(', '(').lstrip(),
target_inlined_code.lstrip())
def transform_code(code: str, xformer: "ASTMigrator") -> str:
"""Apply a transformer to a given chunk of source code
This will parse the code using the AST and find the expressions that are interesting according to xformer.
If those are found the resulting statements will be rewritten and merged into the final source code
"""
line_ends = list(accumulate([len(x) for x in code.splitlines(keepends=True)]))
line_starts = [0] + [x for x in line_ends[:-1]]
try:
tree = horast.parse(code)
except Exception as e_horast:
# fallback to regular typed ast
try:
tree = ast.parse(code)
except Exception as e:
raise CantParseException(str(e), code)
matched = list(xformer.scan_ast(tree))
astmonkey.transformers.ParentChildNodeTransformer().visit(tree)
def node_to_code_offset(node, use_col_offset=True):
return line_starts[node.lineno - 1] + use_col_offset * node.col_offset
# Replace the matched patterns in reverse line order
for match in sorted(
matched, key=lambda node: (node.lineno, node.col_offset), reverse=True
):
xformer.transform_match(match)
parent_statement = find_parent_statement(match)
next_statement = find_next_sibling(parent_statement)
code_start = node_to_code_offset(parent_statement)
if next_statement:
code_end = node_to_code_offset(next_statement, use_col_offset=False)
else:
code_end = len(code)
new_code = horast.unparse(parent_statement)
new_code = new_code.strip()
code = code[:code_start] + new_code + "\n" + code[code_end:]
return code
def _Subscript(self, t):
val = t.value
unparsed_val = horast.unparse(val).strip()
if unparsed_val in PYTHON_FORTRAN_INTRINSICS:
new_val = PYTHON_FORTRAN_INTRINSICS[unparsed_val]
if isinstance(new_val, collections.abc.Callable):
self.dispatch(new_val(t))
return
t = copy.copy(t)
t.value = typed_ast3.Name(id=new_val)
if isinstance(val, typed_ast3.Attribute) and isinstance(val.value, typed_ast3.Name) \
and val.value.id == 'Fortran':
attr = val.attr
if attr == 'file_handles':
self.dispatch(t.slice)
elif attr == 'TypeByNamePrefix':
base_type, letter_ranges = t.slice.value.elts
assert isinstance(letter_ranges,
typed_ast3.Tuple), type(letter_ranges)
# _LOG.warning('%s', type(letter_ranges))
# assert False, (type(letter_ranges), letter_ranges)
self.dispatch_var_type(base_type)
self.write(' (')
interleave(lambda: self.write(', '), lambda _: _.s[1:-1],
letter_ranges.elts)
self.write(')')
else:
raise NotImplementedError(
'Fortran.{}[] cannot be handled yet.'.format(attr))
return
self.dispatch(t.value)
self.write("(")
self.dispatch(t.slice)
# if isinstance(t.slice, typed_ast3.Index):
# elif isinstance(t.slice, typed_ast3.Slice):
# raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
# elif isinstance(t.slice, typed_ast3.ExtSlice):
# raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
# else:
# raise ValueError()
self.write(")")
def unparse(self, tree: typed_ast.ast3.AST) -> str:
code = horast.unparse(tree)
return code
def _Call(self, t):
if getattr(t, 'fortran_metadata', {}).get('is_procedure_call', False):
self.write('call ')
func_name = horast.unparse(t.func).strip()
if has_annotation(t, 'is_mpi_call'):
self.write('call ')
# val = last_attribute_value(t.func)
# fname = t.func.attr
# assert isinstance(val, typed_ast3.Name), type(val)
func_name = MPI_PYTHON_TO_FORTRAN[t.func.attr]
# attribute = t.func
# while isinstance(attribute.value, typed_ast3.Attribute):
# attribute = attribute.value
components = attribute_chain_components(t.func)[:-1]
t.func = typed_ast3.Name(func_name, typed_ast3.Load())
t.args += components
# print()
# self.write(func_name)
# raise
# raise NotImplementedError(self.f.getvalue())
elif func_name.startswith('Fortran.file_handles['):
t = copy.copy(t)
for suffix in ('read', 'close'):
if func_name.endswith('].{}'.format(suffix)):
t.args.insert(0, t.func.value.slice.value)
t.func = typed_ast3.Name(id=suffix, ctx=typed_ast3.Load())
break
# if func_name.endswith('].read'):
# t.func = typed_ast3.Name(id='read', ctx=typed_ast3.Load())
# elif func_name.endswith('].close'):
# t.func = typed_ast3.Name(id='close', ctx=typed_ast3.Load())
else:
raise NotImplementedError(func_name)
elif func_name.endswith('.format'):
t = copy.copy(t)
prefix, _, label = t.func.value.id.rpartition('_')
assert prefix == 'format_label', prefix
self.write(label)
self.write(' ')
t.func = typed_ast3.Name(id='format', ctx=typed_ast3.Load())
elif func_name.endswith('.rstrip'):
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='trim', ctx=typed_ast3.Load())
elif func_name.endswith('.sum'):
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='count', ctx=typed_ast3.Load())
elif func_name.endswith('.size'):
_LOG.warning('assuming np.size()')
t = copy.copy(t)
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='size', ctx=typed_ast3.Load())
elif func_name.endswith('.shape'):
_LOG.warning('assuming np.shape()')
t = copy.copy(t)
t.args[0].n += 1
t.args.insert(0, t.func.value)
t.func = typed_ast3.Name(id='size', ctx=typed_ast3.Load())
elif func_name in PYTHON_FORTRAN_INTRINSICS \
and not getattr(t, 'fortran_metadata', {}).get('is_transformed', False):
new_func = PYTHON_FORTRAN_INTRINSICS[func_name]
if isinstance(new_func, collections.abc.Callable):
self.dispatch(new_func(t))
return
t = copy.copy(t)
t.func = typed_ast3.Name(id=new_func, ctx=typed_ast3.Load())
elif func_name.startswith('np.'):
raise NotImplementedError('not yet implemented: {}'.format(typed_astunparse.dump(t)))
if func_name not in ('print',):
super()._Call(t)
return
self.dispatch(t.func)
self.write(' ')
comma = False
for arg in itertools.chain(t.args, t.keywords):
if comma:
self.write(", ")
else:
comma = True
self.dispatch(arg)