def visit_For(self, node: ast.For) -> None:
# create list to iterate over
lst_name = self.context.get_temp_name()
assign_node = ast.Assign(
targets=[ast.Name(id=lst_name, ctx=ast.Store())], value=node.iter)
self.visit(assign_node)
lst = self.context[lst_name]
index = self.context.get_temp_var(TypeDB.get_type_by_name("int"))
length = lst.tp.get_method("len")
length_code = length.get_code(self.context, lst).code
# construct for statement
self.start_line(
f"for({index.code}=0; {index.code} < {length_code}; {index.code}++) {{\n"
)
self.indent += 4
assign_node = ast.Assign(
targets=[node.target],
value=ast.Subscript(
value=ast.Name(id=lst_name, ctx=ast.Load()),
slice=ast.Index(value=ast.Name(id=index.code, ctx=ast.Load())),
ctx=ast.Load()))
self.visit(assign_node)
for statement in node.body:
self.visit(statement)
self.indent -= 4
self.start_line("}\n")
self.all_paths_return = False
def _Attribute(self, t):
if isinstance(t.value, typed_ast3.Name) and t.value.id == 'Fortran':
raise NotImplementedError(
'Fortran.{} can be handled only when subscripted.'.format(
t.attr))
if isinstance(t.value, typed_ast3.Name) and t.attr == 'size':
call = typed_ast3.Call(func=typed_ast3.Name(id='size',
ctx=typed_ast3.Load()),
args=[t.value],
keywords=[])
self._Call(call)
return
if syntax_matches(
t,
typed_ast3.Attribute(value=typed_ast3.Attribute(
value=typed_ast3.Name(id='st', ctx=typed_ast3.Load()),
attr='generic',
ctx=typed_ast3.Load()),
attr='GenericVar',
ctx=typed_ast3.Load())):
# t._fortran_metadata = {'is_generic_var': True}
# self._generic_vars.append()
return
self._unsupported_syntax(t)
'''
def _set_arguments_to_super(self, call: ast.Call, first_argument: str):
super_cls = ast.Name(
id='cls') if first_argument == 'cls' else ast.Call(
func=ast.Name(id='type'),
args=[ast.Name(id=first_argument)],
keywords=[])
call.args = [super_cls, ast.Name(id=first_argument)]
def _show_store_contents_expr(self) -> ast3.Expr:
"""Returns an ast3.Expr which prints the value of the store in the screen. Useful
for debugging.
"""
# print(st)
return ast3.Expr(value=ast3.Call(
func=ast3.Name(id='print', ctx=ast3.Load()),
args=[ast3.Name(id='st', ctx=ast3.Load())],
keywords=[],
), )
def visit_Exec(self, n):
new_globals = self.maybe_visit(n.globals)
if new_globals is None:
new_globals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1)
new_locals = self.maybe_visit(n.locals)
if new_locals is None:
new_locals = ast3.Name("None", ast3.Load(), lineno=-1, col_offset=-1)
return ast3.Expr(ast3.Call(ast3.Name("exec", ast3.Load(), lineno=n.lineno, col_offset=-1),
[self.visit(n.body), new_globals, new_locals],
[],
lineno=n.lineno, col_offset=-1))
def visit_Typedef(self, node): # pylint: disable=invalid-name
"""Transform Typedef."""
name = node.name
assert isinstance(name, str), type(name)
quals = node.quals
if quals:
_LOG.warning('ignoring unsupported C grammar: %s', quals)
assert node.storage == ['typedef'], node.storage
name_, type_ = self.visit(node.type)
assert name == name_, (name, name_)
_ = self.visit(node.coord)
return typed_ast3.AnnAssign(target=typed_ast3.Name(name, typed_ast3.Store()), value=type_,
annotation=typed_ast3.Name('type', typed_ast3.Load()), simple=1)
def _replace_super_args(self, node: ast.Call) -> None:
try:
func = get_closest_parent_of(self._tree, node, ast.FunctionDef)
except NodeNotFound:
warn('super() outside of function')
return
try:
cls = get_closest_parent_of(self._tree, node, ast.ClassDef)
except NodeNotFound:
warn('super() outside of class')
return
node.args = [ast.Name(id=cls.name), ast.Name(id=func.args.args[0].arg)]
def visit_Struct(self, node): # pylint: disable=invalid-name
"""Transform Struct."""
name = node.name
assert isinstance(name, str), type(name)
assert node.decls is None, node.decls
_ = self.visit(node.coord)
return typed_ast3.Name(name, typed_ast3.Load())
def visit_Decl(self, node) -> t.Union[typed_ast3.AnnAssign, # pylint: disable=invalid-name
t.Tuple[str, typed_ast3.arguments, typed_ast3.AST]]:
"""Transform Decl."""
name = node.name
assert isinstance(name, str), type(name)
quals = node.quals
if quals:
_LOG.warning('ignoring unsupported C grammar: %s', quals)
storage = [self.visit(subnode) for subnode in node.storage]
if storage:
raise NotImplementedError(_node_debug(node.storage), str(storage))
funcspec = [self.visit(subnode) for subnode in node.funcspec]
if funcspec:
raise NotImplementedError(_node_debug(node.funcspec), str(funcspec))
type_data = self.visit(node.type)
assert isinstance(type_data, tuple)
assert len(type_data) == DECL_DATA_LENGTHS[type(node.type)], (type(node.type), type_data)
init = self.visit(node.init)
if init is not None:
assert isinstance(node.type, INITIALIZABLE_DECLARATIONS)
# assert isinstance(node.type, c_ast.TypeDecl), type(node.type)
# raise NotImplementedError(_node_debug(node.init), str(init))
bitsize = self.visit(node.bitsize)
if bitsize is not None:
raise NotImplementedError(_node_debug(node.bitsize), str(bitsize))
_ = self.visit(node.coord)
if init is not None or isinstance(node.type, INITIALIZABLE_DECLARATIONS):
name_, type_ = type_data
assert name_ == name
return typed_ast3.AnnAssign(target=typed_ast3.Name(id=name_, ctx=typed_ast3.Store()),
annotation=type_, value=init, simple=1)
if isinstance(node.type, (c_ast.FuncDecl,)):
return type_data
return self.generic_visit(node)
def visit_Cast(self, node): # pylint: disable=invalid-name
"""Transform C cast into cast() function call."""
to_type = self.visit(node.to_type)
expr = self.visit(node.expr)
_ = self.visit(node.coord)
return typed_ast3.Call(func=typed_ast3.Name(id='cast', ctx=typed_ast3.Load()), args=[expr],
keywords=[typed_ast3.keyword(arg='type', value=to_type)])
def visit_ArrayDecl( # pylint: disable=invalid-name
self, node) -> t.Tuple[str, typed_ast3.Subscript]:
"""Return tuple of: name, st.ndarray[..., ...] for given array type information."""
name, type_ = self.visit(node.type)
assert isinstance(name, str)
assert isinstance(type_, typed_ast3.AST)
dim = self.visit(node.dim)
if dim is not None:
raise NotImplementedError(_node_debug(node.dim), str(dim))
dim_quals = [self.visit(subnode) for subnode in node.dim_quals]
if dim_quals:
raise NotImplementedError(_node_debug(node.dim_quals),
str(dim_quals))
_ = self.visit(node.coord)
return name, typed_ast3.Subscript(
value=typed_ast3.Attribute(value=typed_ast3.Name(
id='st', ctx=typed_ast3.Load()),
attr='ndarray',
ctx=typed_ast3.Load()),
slice=typed_ast3.ExtSlice(dims=[
typed_ast3.Index(value=typed_ast3.Ellipsis()),
typed_ast3.Index(value=type_) # ,
# typed_ast3.Index(value=typed_ast3.Tuple(n=-1))
]),
ctx=typed_ast3.Load())
def _merge_dicts(self, xs: Iterable[Union[ast.Call, ast.Dict]]) \
-> ast.Call:
"""Creates call of function for merging dicts."""
return ast.Call(
func=ast.Name(id='_py_backwards_merge_dicts'),
args=[ast.List(elts=list(xs))],
keywords=[])
def evaluate_call(node: ast3.Call,
*,
scope: Scope,
module_path: catalog.Path) -> Node:
if not is_named_tuple_definition(node,
scope=scope,
module_path=module_path):
return node
class_name_node, fields_node = node.args
def field_to_parameter(field_node: ast3.expr) -> ast3.arg:
name_node, annotation_node = field_node.elts
return ast3.arg(ast3.literal_eval(name_node), annotation_node)
initializer_node = ast3.FunctionDef(
'__init__',
ast3.arguments([ast3.arg('self', None)]
+ list(map(field_to_parameter,
fields_node.elts)),
None, [], [], None, []),
[ast3.Pass()], [], None)
function_path = evaluate_node(node.func,
scope=scope,
module_path=module_path)
class_def = ast3.ClassDef(ast3.literal_eval(class_name_node),
[ast3.Name(str(function_path), ast3.Load())],
[], [initializer_node], [])
return ast3.fix_missing_locations(ast3.copy_location(class_def, node))
def make_numpy_constructor(function: str, arg: typed_ast3.AST,
data_type: typed_ast3.AST) -> typed_ast3.Call:
return typed_ast3.Call(
func=typed_ast3.Attribute(
value=typed_ast3.Name(id='np', ctx=typed_ast3.Load()),
attr=function, ctx=typed_ast3.Load()),
args=[arg],
keywords=[typed_ast3.keyword(arg='dtype', value=data_type)])
def visit_IdentifierType(self, node) -> typed_ast3.Name: # pylint: disable=invalid-name
"""Transform IdentifierType(names: t.List[str], coord: t.Optional[Coord])."""
names = node.names
assert len(names) == 1, names
name = names[0]
assert isinstance(name, str)
_ = self.visit(node.coord)
return typed_ast3.Name(id=name, ctx=typed_ast3.Load())
def _prepare_lists(self, xs: List[Splitted]) -> Iterable[ListEntry]:
"""Wrap starred in list call and list elts to just List."""
for x in xs:
if isinstance(x, ast.Starred):
yield ast.Call(func=ast.Name(id='list'),
args=[x.value],
keywords=[])
elif x:
yield ast.List(elts=x)
def pos_as_tuple(node: Union[ast3.expr, ast3.stmt]) -> Optional[ast3.Tuple]:
if not hasattr(node, 'lineno'):
return None
return ast3.Tuple(elts=[
ast3.Tuple(elts=[ast3.Num(node.lineno),
ast3.Num(node.col_offset)],
ctx=ast3.Load()),
ast3.Name(id='fn', ctx=ast3.Load())
],
ctx=ast3.Load())
def test_non_str_type_comment(self):
examples = {
typed_ast3.Assign(targets=[
typed_ast3.Name('x', typed_ast3.Store())
],
value=typed_ast3.Str('universe, life, and everything'),
type_comment=typed_ast3.Str('42')):
logging.DEBUG,
typed_ast3.Assign(targets=[
typed_ast3.Name('x', typed_ast3.Store())
],
value=typed_ast3.Str('universe, life, and everything'),
type_comment=42):
logging.WARNING
}
for example, expected_level in examples.items():
resolver = TypeHintResolver[typed_ast3, typed_ast3](eval_=False)
with self.subTest(example=example, expected_level=expected_level):
with self.assertLogs(level=expected_level):
resolver.visit(example)
def _prepare_splitted(self, splitted: Splitted) \
-> Iterable[Union[ast.Call, ast.Dict]]:
"""Wraps splitted in Call or Dict."""
for group in splitted:
if not isinstance(group, list):
yield ast.Call(func=ast.Name(id='dict'),
args=[group],
keywords=[])
elif group:
yield ast.Dict(keys=[key for key, _ in group],
values=[value for _, value in group])
def _PointerType(self, node: ET.Element): # pylint: disable=invalid-name
id_ = node.attrib['id']
type_ = node.attrib['type']
is_const = type_.endswith('c')
if is_const:
type_ = type_[:-1]
try:
base_type = self.fundamental_types[type_]
except KeyError:
# _LOG.debug()
base_type = typed_ast3.Str(type_, '')
type_info = typed_ast3.Subscript(value=typed_ast3.Name(
id='Pointer', ctx=typed_ast3.Load()),
slice=typed_ast3.Index(base_type),
ctx=typed_ast3.Load())
if is_const:
type_info = typed_ast3.Subscript(value=typed_ast3.Name(
id='Const', ctx=typed_ast3.Load()),
slice=typed_ast3.Index(type_info),
ctx=typed_ast3.Load())
return (id_, type_info)
def visit_UnaryOp(self, node): # pylint: disable=invalid-name
"""Transform UnaryOp."""
op_type, op_ = C_UNARY_OPERATORS_TO_PYTHON[node.op]
expr = self.visit(node.expr)
_ = self.visit(node.coord)
if op_type is typed_ast3.Call:
return op_type(func=typed_ast3.Name(id=op_, ctx=typed_ast3.Load()), args=[expr],
keywords=[])
if op_type is typed_ast3.AugAssign:
return op_type(target=expr, op=op_(), value=typed_ast3.Num(n=1))
# raise NotImplementedError()
return op_type(op=op_(), operand=expr)
def _replace_return(self, parent: Any, return_: ast.Return) -> None:
"""Replace return with exception raising."""
index = parent.body.index(return_)
parent.body.pop(index)
exception = ast.Name(id='_py_backwards_generator_return_{}'.format(
self._name_suffix))
raise_exception = ast.Raise(exc=exception, cause=None)
parent.body.insert(index, raise_exception)
set_value = ast.Assign(targets=[
ast.Attribute(value=exception, attr='value'),
], value=return_.value)
parent.body.insert(index, set_value)
assign = ast.Assign(targets=[exception],
value=ast.Call(func=ast.Name(id='StopIteration'),
args=[],
keywords=[]))
parent.body.insert(index, assign)
def visit_Print(self, n):
keywords = []
if n.dest is not None:
keywords.append(ast3.keyword("file", self.visit(n.dest)))
if not n.nl:
keywords.append(ast3.keyword("end", ast3.Str(" ", lineno=n.lineno, col_offset=-1)))
return ast3.Expr(ast3.Call(ast3.Name("print", ast3.Load(), lineno=n.lineno, col_offset=-1),
self.visit(n.values),
keywords,
lineno=n.lineno, col_offset=-1))
def _emulate_yield_from(self, targets: Optional[List[ast.Name]],
node: ast.YieldFrom) -> Iterable[ast.AST]:
generator = ast.Name(
id='_py_backwards_generator_{}'.format(self._name_suffix))
exception = ast.Name(
id='_py_backwards_generator_exception_{}'.format(self._name_suffix))
yield ast.Assign(targets=[generator],
value=ast.Call(func=ast.Name(id='iter'),
args=[node.value],
keywords=[]))
assign_to_targets = [
ast.If(test=ast.Call(func=ast.Name(id='hasattr'), args=[
exception, ast.Str(s='value'),
], keywords=[]), body=[
ast.Assign(targets=targets,
value=ast.Attribute(
value=exception, attr='value')),
], orelse=[]),
ast.Break()] if targets else [ast.Break()]
yield ast.While(test=ast.NameConstant(value=True), body=[
ast.Try(body=[
ast.Expr(value=ast.Yield(value=ast.Call(
func=ast.Name(id='next'),
args=[generator], keywords=[]))),
], handlers=[
ast.ExceptHandler(
type=ast.Name(id='StopIteration'),
name=exception.id,
body=assign_to_targets),
], orelse=[], finalbody=[]),
], orelse=[])
self._name_suffix += 1
def _attribute_to_name(node: ast3.Attribute) -> ast3.Name:
"""Recursively convert Attributes to Names."""
val = node.value
if isinstance(val, ast3.Name):
prefix = val.id
elif isinstance(val, ast3.Attribute):
prefix = _attribute_to_name(val).id
elif isinstance(val, (pytd.NamedType, pytd.Module)):
prefix = val.name
else:
msg = "Unexpected attribute access on %r [%s]" % (val, type(val))
raise ParseError(msg)
return ast3.Name(prefix + "." + node.attr)
def _Attribute(self, t):
if isinstance(t.value, typed_ast3.Name) and t.value.id == 'Fortran':
raise NotImplementedError(
'Fortran.{} can be handled only when subscripted.'.format(
t.attr))
if isinstance(t.value, typed_ast3.Name) and t.attr == 'size':
call = typed_ast3.Call(func=typed_ast3.Name(id='size',
ctx=typed_ast3.Load()),
args=[t.value],
keywords=[])
self._Call(call)
return
self._unsupported_syntax(t)
'''
def visit_PtrDecl(self, node): # pylint: disable=invalid-name
"""Return st.Pointer[...] for given pointer type."""
quals = node.quals
if quals:
_LOG.warning('ignoring unsupported C grammar: %s', quals)
name, type_ = self.visit(node.type)
assert name is None or isinstance(name, str)
assert isinstance(type_, typed_ast3.AST), type(type_)
_ = self.visit(node.coord)
# assert type_ is not None, _node_str(node)
return name, typed_ast3.Subscript(
value=typed_ast3.Attribute(value=typed_ast3.Name(id='st', ctx=typed_ast3.Load()),
attr='Pointer', ctx=typed_ast3.Load()),
slice=typed_ast3.Index(value=type_), ctx=typed_ast3.Load())
def visit_Expr(self, node: ast3.Expr) -> VisitorOutput:
"""Only the internal parts of an Expr are modified, an Expr keeps being an Expr"""
self.generic_visit(node)
# In console mode ("single" for Python's compile) any expression statement should
# print to console
if self.console:
return ast3.Expr(value=ast3.Call(
func=ast3.Name(id='print_console', ctx=ast3.Load()),
args=[node.value],
keywords=[],
), )
return node
def test_preserve_custom_fields(self):
example = typed_ast3.Name('spam', typed_ast3.Load())
typed_ast3.fix_missing_locations(example)
example.custom_field = 'some value'
transcriber = AstTranscriber[typed_ast3, ast]()
result = transcriber.visit(example)
_LOG.debug('%s', vars(example))
self.assertTrue(hasattr(example, 'lineno'), msg=vars(example))
self.assertTrue(hasattr(example, 'custom_field'), msg=vars(example))
_LOG.debug('%s', vars(result))
self.assertTrue(hasattr(result, 'lineno'), msg=vars(result))
self.assertTrue(hasattr(result, 'custom_field'), msg=vars(result))
def visit_ListComp(self, node):
from parser.functions import FunctionImplementation
# calculate result type
if len(node.generators) > 1:
raise InvalidOperation(
"Only one for statement permitted in comprehensions")
comp = node.generators[0]
if len(comp.ifs) > 1:
raise InvalidOperation(
"Only one if statement allowed in List Comprehension")
assign_node = ast.Assign(targets=[comp.target],
value=ast.Subscript(value=comp.iter,
slice=ast.Index(
ast.Num(0))))
return_node = ast.Return(value=node.elt)
function_node = ast.FunctionDef(name="temp",
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[assign_node, return_node])
function_interpreter = FunctionImplementation(function_node, (),
self.context)
result_type = TypeDB.get_list([function_interpreter.retval.tp])
# create temp list to hold values
result = self.context.get_temp_var(result_type)
self.prepends.append(
f"{result.code} = {result_type.as_literal([])};\n")
# create for expression
append_node = ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(id=result.code,
ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[node.elt],
keywords=[]))
if comp.ifs:
body = ast.If(test=comp.ifs[0], body=[append_node], orelse=[])
else:
body = append_node
for_node = ast.For(target=comp.target,
iter=comp.iter,
body=[body],
orelse=[])
self.prepends.append(for_node)
return result