def _emulate_yield_from(self, targets: Optional[List[ast.Name]],
node: ast.YieldFrom) -> Iterable[ast.AST]:
generator = ast.Name(
id='_py_backwards_generator_{}'.format(self._name_suffix))
exception = ast.Name(
id='_py_backwards_generator_exception_{}'.format(self._name_suffix))
yield ast.Assign(targets=[generator],
value=ast.Call(func=ast.Name(id='iter'),
args=[node.value],
keywords=[]))
assign_to_targets = [
ast.If(test=ast.Call(func=ast.Name(id='hasattr'), args=[
exception, ast.Str(s='value'),
], keywords=[]), body=[
ast.Assign(targets=targets,
value=ast.Attribute(
value=exception, attr='value')),
], orelse=[]),
ast.Break()] if targets else [ast.Break()]
yield ast.While(test=ast.NameConstant(value=True), body=[
ast.Try(body=[
ast.Expr(value=ast.Yield(value=ast.Call(
func=ast.Name(id='next'),
args=[generator], keywords=[]))),
], handlers=[
ast.ExceptHandler(
type=ast.Name(id='StopIteration'),
name=exception.id,
body=assign_to_targets),
], orelse=[], finalbody=[]),
], orelse=[])
self._name_suffix += 1
def visit_For(self, node: ast.For) -> None:
# create list to iterate over
lst_name = self.context.get_temp_name()
assign_node = ast.Assign(
targets=[ast.Name(id=lst_name, ctx=ast.Store())], value=node.iter)
self.visit(assign_node)
lst = self.context[lst_name]
index = self.context.get_temp_var(TypeDB.get_type_by_name("int"))
length = lst.tp.get_method("len")
length_code = length.get_code(self.context, lst).code
# construct for statement
self.start_line(
f"for({index.code}=0; {index.code} < {length_code}; {index.code}++) {{\n"
)
self.indent += 4
assign_node = ast.Assign(
targets=[node.target],
value=ast.Subscript(
value=ast.Name(id=lst_name, ctx=ast.Load()),
slice=ast.Index(value=ast.Name(id=index.code, ctx=ast.Load())),
ctx=ast.Load()))
self.visit(assign_node)
for statement in node.body:
self.visit(statement)
self.indent -= 4
self.start_line("}\n")
self.all_paths_return = False
def for_header_to_tuple(
target, target_type,
iter_) -> t.Tuple[typed_ast3.Assign, typed_ast3.AST, typed_ast3.AST]:
if match_range_call(iter_):
begin, end, step = inferred_range_args(iter_)
# raise NotImplementedError('TODO')
else:
raise NotImplementedError(
'only range() iterator in for loops is currently supported')
if target_type is None:
init = typed_ast3.Assign(targets=[target],
value=begin,
type_comment=None)
else:
init = typed_ast3.AnnAssign(target=target,
annotation=target_type,
value=begin,
simple=True)
condition = typed_ast3.Compare(left=target,
ops=[typed_ast3.Lt()],
comparators=[end])
increment = typed_ast3.AugAssign(target=target,
op=typed_ast3.Add(),
value=step)
return init, condition, increment
def visit_AnnAssign(self, node: ast.AnnAssign) -> Optional[ast.Assign]:
if node.value is None:
return None
return self.generic_visit(
ast.Assign(
targets=[node.target], # type: ignore
value=node.value))
def test_non_str_type_comment(self):
examples = {
typed_ast3.Assign(targets=[
typed_ast3.Name('x', typed_ast3.Store())
],
value=typed_ast3.Str('universe, life, and everything'),
type_comment=typed_ast3.Str('42')):
logging.DEBUG,
typed_ast3.Assign(targets=[
typed_ast3.Name('x', typed_ast3.Store())
],
value=typed_ast3.Str('universe, life, and everything'),
type_comment=42):
logging.WARNING
}
for example, expected_level in examples.items():
resolver = TypeHintResolver[typed_ast3, typed_ast3](eval_=False)
with self.subTest(example=example, expected_level=expected_level):
with self.assertLogs(level=expected_level):
resolver.visit(example)
def _replace_return(self, parent: Any, return_: ast.Return) -> None:
"""Replace return with exception raising."""
index = parent.body.index(return_)
parent.body.pop(index)
exception = ast.Name(id='_py_backwards_generator_return_{}'.format(
self._name_suffix))
raise_exception = ast.Raise(exc=exception, cause=None)
parent.body.insert(index, raise_exception)
set_value = ast.Assign(targets=[
ast.Attribute(value=exception, attr='value'),
], value=return_.value)
parent.body.insert(index, set_value)
assign = ast.Assign(targets=[exception],
value=ast.Call(func=ast.Name(id='StopIteration'),
args=[],
keywords=[]))
parent.body.insert(index, assign)
def visit_ListComp(self, node):
from parser.functions import FunctionImplementation
# calculate result type
if len(node.generators) > 1:
raise InvalidOperation(
"Only one for statement permitted in comprehensions")
comp = node.generators[0]
if len(comp.ifs) > 1:
raise InvalidOperation(
"Only one if statement allowed in List Comprehension")
assign_node = ast.Assign(targets=[comp.target],
value=ast.Subscript(value=comp.iter,
slice=ast.Index(
ast.Num(0))))
return_node = ast.Return(value=node.elt)
function_node = ast.FunctionDef(name="temp",
args=ast.arguments(args=[],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[assign_node, return_node])
function_interpreter = FunctionImplementation(function_node, (),
self.context)
result_type = TypeDB.get_list([function_interpreter.retval.tp])
# create temp list to hold values
result = self.context.get_temp_var(result_type)
self.prepends.append(
f"{result.code} = {result_type.as_literal([])};\n")
# create for expression
append_node = ast.Expr(
ast.Call(func=ast.Attribute(value=ast.Name(id=result.code,
ctx=ast.Load()),
attr="append",
ctx=ast.Load()),
args=[node.elt],
keywords=[]))
if comp.ifs:
body = ast.If(test=comp.ifs[0], body=[append_node], orelse=[])
else:
body = append_node
for_node = ast.For(target=comp.target,
iter=comp.iter,
body=[body],
orelse=[])
self.prepends.append(for_node)
return result
def visit_AugAssign(self, node: ast3.AugAssign) -> VisitorOutput:
"""Converts `A (op)= Statement` into `A = A (op) (Statement)`
"""
left_value = copy_ast3(node.target)
left_value.ctx = ast3.Load() # type: ignore
new_target = self.visit(node.target)
new_value = self.visit_BinOp(
ast3.BinOp(left=left_value,
op=node.op,
right=node.value,
lineno=node.lineno,
col_offset=node.col_offset))
return ast3.Assign(
targets=[new_target], value=new_value,
type_comment=None) # TODO(helq): don't ignore the type comment!
def transform(cls, tree: ast.AST) -> TransformationResult:
tree_changed = False
for node in find(tree, ast.AnnAssign):
try:
parent, index = get_non_exp_parent_and_index(tree, node)
except NodeNotFound:
warn('Assignment outside of body')
continue
tree_changed = True
parent.body.pop(index) # type: ignore
if node.value is not None:
insert_at(index, parent,
ast.Assign(targets=[node.target], # type: ignore
value=node.value,
type_comment=node.annotation))
return TransformationResult(tree, tree_changed, [])
def visit_AnnAssign(self, node: ast3.AnnAssign) -> VisitorOutput:
"""Transforms an assignment with annotation into a pytropos type hint assignment.
For example, it converts::
var: ann = expr
into::
`var` = pt.type_hint(`ann`, `expr`)
"""
if node.value is None:
raise AstTransformerError(
f"{self.filename}:{node.lineno}:{node.col_offset}: Fatal Error: "
"Only annotated assignments are allowed (variables with initial values). "
"I.e., no full support for PEP 526 yet. Sorry :(")
pos = pos_as_tuple(node)
# Deleting annotation :S
self.generic_visit(node)
# new_node = ast3.Assign(targets=[node.target], value=node.value)
return ast3.Assign(
targets=[node.target],
value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='pt', ctx=ast3.Load()),
attr='annotation',
ctx=ast3.Load(),
),
args=[
node.annotation, node.value, pos if pos else ast3.Expr(
value=ast3.NameConstant(value=None))
],
keywords=[],
),
)
def transform(cls, tree: ast.AST) -> TransformationResult:
tree_changed = False
for node in find(tree, ast.AnnAssign):
try:
position = get_node_position(tree, node)
except NodeNotFound:
warn('Assignment outside of body')
continue
tree_changed = True
position.holder.pop(position.index) # type: ignore
if node.value is not None:
insert_at(
position.index,
position.parent,
ast.Assign(
targets=[node.target], # type: ignore
value=node.value,
type_comment=node.annotation),
position.attribute)
return TransformationResult(tree, tree_changed, [])
def visit_ImportFrom(self, node: ast3.ImportFrom) -> VisitorOutput:
"""Defines how to import (from) modules (supported and nonsupported)
For example, it converts::
from numpy import array
from numpy import *
from somelib import var, othervar as var2
from otherlib import *
into::
from pytropos.libs_checking import numpy_module
st['array'] = numpy_module.attr['array', pos...]
from pytropos.libs_checking import numpy_module
st.importStar(numpy_module)
st['var'] = pt.Top
st['var2'] = pt.Top
st.importStar()
"""
libs: 'List[ast3.AST]' = []
if node.module in self._supported_modules:
module_name = self._supported_modules[node.module]
# from pytropos.libs_checking import module_name
libs.append(
ast3.ImportFrom(
module='pytropos.libs_checking',
names=[ast3.alias(name=module_name, asname=None)],
level=0,
))
if node.names[0].name == '*':
# st.importStar(module_name)
libs.append(
ast3.Expr(value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='st', ctx=ast3.Load()),
attr='importStar',
ctx=ast3.Load(),
),
args=[ast3.Name(id=module_name, ctx=ast3.Load())],
keywords=[],
), ))
else:
for alias in node.names:
# st['asname'] = modname.attr['name']
pos = pos_as_tuple(node)
if pos is not None:
attrname = ast3.Tuple(
elts=[ast3.Str(s=alias.name), pos],
ctx=ast3.Load()) # type: ast3.expr
else:
attrname = ast3.Str(s=alias.name)
libs.append(
ast3.Assign(
targets=[
ast3.Subscript(
value=ast3.Name(id='st', ctx=ast3.Load()),
slice=ast3.Index(value=ast3.Str(
s=alias.asname if alias.
asname else alias.name), ),
ctx=ast3.Store(),
),
],
value=ast3.Subscript(
value=ast3.Attribute(
value=ast3.Name(id=module_name,
ctx=ast3.Load()),
attr='attr',
ctx=ast3.Load(),
),
slice=ast3.Index(value=attrname, ),
ctx=ast3.Load(),
),
))
else:
if node.names[0].name == '*':
# st.importStar()
libs.append(
ast3.Expr(value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='st', ctx=ast3.Load()),
attr='importStar',
ctx=ast3.Load(),
),
args=[],
keywords=[],
), ))
else:
libs.extend(
ast3.parse( # type: ignore
'\n'.join([
"st['{asname}'] = pt.Top".format(
asname=alias.asname if alias.asname else alias.
name) for alias in node.names
])).body)
return libs
def visit_While(self, node: ast3.While) -> VisitorOutput:
"""Transforms an if statement into what Pytropos understands:
For example, it converts::
while question:
body
into::
if_qstn = TRANSFORMED(question)
def while_qst(st):
return question
def while_(st):
body
return st
st = pt.runWhile(st, while_qstn, while_)
"""
if node.orelse:
raise AstTransformerError(
f"Pytropos doesn't support else statement in while loop yet, sorry :("
)
self.generic_visit(node)
new_body = node.body.copy()
new_body.append(
ast3.Return(value=ast3.Name(id='st', ctx=ast3.Load()), ))
new_node = [
ast3.FunctionDef(
name='while_qst',
args=ast3.arguments(args=[ast3.arg(arg='st', annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=[ast3.Return(value=node.test)],
decorator_list=[],
returns=None,
),
ast3.FunctionDef(
name='while_',
args=ast3.arguments(args=[ast3.arg(arg='st', annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=new_body,
decorator_list=[],
returns=None,
),
ast3.Assign(targets=[ast3.Name(id='st', ctx=ast3.Store())],
value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='pt', ctx=ast3.Load()),
attr='runWhile',
ctx=ast3.Load(),
),
args=[
ast3.Name(id='st', ctx=ast3.Load()),
ast3.Name(id='while_qst', ctx=ast3.Load()),
ast3.Name(id='while_', ctx=ast3.Load())
],
keywords=[],
))
]
return new_node # type: ignore
def visit_If(self, node: ast3.If) -> VisitorOutput:
"""Transforms an if statement into what Pytropos understands:
For example, it converts::
if question:
body1
else:
body2
into::
if_qstn = TRANSFORMED(question)
def if_(st):
body1
return st
def else_(st):
body2
return st
st = pt.runIf(st, if_qstn, if_, else_)
"""
self.generic_visit(node)
new_body = node.body.copy()
new_orelse = node.orelse.copy()
orelse = bool(node.orelse)
new_body.append(
ast3.Return(value=ast3.Name(id='st', ctx=ast3.Load()), ))
new_node = [
ast3.Assign(targets=[ast3.Name(id='if_qstn', ctx=ast3.Store())],
value=node.test),
ast3.FunctionDef(
name='if_',
args=ast3.arguments(args=[ast3.arg(arg='st', annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=new_body,
decorator_list=[],
returns=None,
),
]
if orelse:
# adding "return st"
new_orelse.append(
ast3.Return(value=ast3.Name(id='st', ctx=ast3.Load()), ))
new_node.append(
ast3.FunctionDef(
name='else_',
args=ast3.arguments(
args=[ast3.arg(arg='st', annotation=None)],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[]),
body=new_orelse,
decorator_list=[],
returns=None,
))
new_node.append(
ast3.Assign(targets=[ast3.Name(id='st', ctx=ast3.Store())],
value=ast3.Call(
func=ast3.Attribute(
value=ast3.Name(id='pt', ctx=ast3.Load()),
attr='runIf',
ctx=ast3.Load(),
),
args=[
ast3.Name(id='st', ctx=ast3.Load()),
ast3.Name(id='if_qstn', ctx=ast3.Load()),
ast3.Name(id='if_', ctx=ast3.Load()),
ast3.Name(id='else_', ctx=ast3.Load())
] if orelse else [
ast3.Name(id='st', ctx=ast3.Load()),
ast3.Name(id='if_qstn', ctx=ast3.Load()),
ast3.Name(id='if_', ctx=ast3.Load())
],
keywords=[],
)))
return new_node # type: ignore
def convert_return_to_assign(target, return_statement):
if isinstance(return_statement, typed_ast3.Return):
return typed_ast3.Assign(targets=[target], value=return_statement.value)
return return_statement
def expression_to_assignment(node: ast3.expr, *, name: str) -> ast3.Assign:
name_node = ast3.copy_location(ast3.Name(name, ast3.Store()), node)
result = ast3.Assign([name_node], node, None)
return ast3.copy_location(result, node)