def build_JoinedStr(ctx, node):
str_spec = ''
args = []
for sub_node in node.values:
if isinstance(sub_node, ast.FormattedValue):
str_spec += '{}'
args.append(build_expr(ctx, sub_node.value))
elif isinstance(sub_node, ast.Constant):
str_spec += sub_node.value
elif isinstance(sub_node, ast.Str):
# ast.Str has been deprecated in Python 3.8,
# but constant string is a ast.Str node in Python 3.6
str_spec += sub_node.s
args.insert(0, ast.copy_location(ast.Constant(value=str_spec), node))
call = ast.Call(func=parse_expr('ti.ti_format'),
args=args,
keywords=[])
return ast.copy_location(call, node)
def to_ast(self) -> ast.ClassDef:
""" Generate Python AST for this model info """
return ast.ClassDef(
self.name,
bases=[
ast.Attribute(ast.Name(base.__module__), base.__name__)
for base in self.bases
],
decorator_list=[],
keywords=[],
body=[
# Docstring
ast.Expr(ast.Constant(self.docstring)),
# Attributes
*(
field.to_ast()
for field in self.fields
)
]
)
def create_event_parsers_module(global_context: GlobalContext):
referenced_domain_modules = set()
event_names = []
event_parsers = []
for domain in global_context.domains.values():
for event in domain.events:
event_names.append(
ast.Constant(f"{event.context.domain_name}.{event.name}"))
event_parsers.append(
ast.Name(f"{event.context.module_name}.{event.classname}"))
referenced_domain_modules.add(event.context.module_name)
body = [
ast_import_from(".", *list(referenced_domain_modules)),
ast.Assign([ast.Name("event_parsers")],
ast.Dict(event_names, event_parsers),
lineno=0),
]
return ast_module(body)
def _generate_function(self, function, module, resolve):
name = _id_snake(function.name)
definition = ast.FunctionDef(
name=name,
decorator_list=[],
args=ast.arguments(
args=[],
defaults=[],
vararg=None,
kwarg=None,
),
body=[
ast.Expr(value=ast.Constant(
value=_DocString(function.description),
kind=None,
))
],
)
module.body.append(definition)
hook = (f"return __import__('skyhook')"
f".Hook.current().call('{function.name}')")
hook_ast = ast.parse(hook)
definition.body.extend(hook_ast.body)
for argument in function.arguments:
hook_ast.body[0].value.args.append(ast.Name(id=argument.name))
annotation, preamble = _annotate_schema(
argument.schema,
resolve,
[function.name, argument.name],
)
definition.args.args.append(
ast.arg(
arg=argument.name,
annotation=annotation,
))
module.preamble.extend(preamble)
if function.return_:
definition.returns, preamble = \
_annotate_schema(function.return_.schema, resolve)
module.preamble.extend(preamble)
module.body.append(self._generate_function_lambda_ast(name, function))
def test_gen_arguments_py(monkeypatch):
assert isinstance(rm.gen_arguments_py([]), types.GeneratorType)
assert list(rm.gen_arguments_py([])) == []
ret = rm.gen_arguments_py(my_parameters)
assert ast.dump(ret.__next__().value) == ast.dump(
ast.Dict(
keys=[[ast.Constant(value="type")]], values=[[ast.Constant(value="bool")]]
)
)
assert ast.dump(ret.__next__().value) == ast.dump(
ast.Dict(
keys=[[ast.Constant(value="required")], [ast.Constant(value="type")]],
values=[[ast.Constant(value=True)], [ast.Constant(value="int")]],
)
)
def visit_Call(self, node):
fname = self.visit(node.func)
if (get_id(fname) == "print" and len(node.args) == 1
and not (isinstance(node.args[0], ast.Name)
or isinstance(node.args[0], ast.Constant))):
tmp = ast.Name(id=self._get_temp(), lineno=node.lineno)
ret = ast.If(
test=ast.Constant(value=True),
body=[
ast.Assign(targets=[tmp],
value=node.args[0],
lineno=node.lineno),
node,
],
orelse=[],
lineno=node.lineno,
)
node.args[0] = tmp
return ret
return node
def visit_Dict(self, node: ast.Dict):
if len(node.keys) == 0:
raise self._syntax_error("empty dict makes no sense here", node)
keys = []
for key in node.keys:
if isinstance(key, ast.Str):
value = key.s
elif isinstance(key, ast.Num):
value = key.n
elif isinstance(key, ast.NameConstant):
value = key.value
elif isinstance(key, ast.Tuple) and all(
[isinstance(elt, ast.Num) for elt in key.elts]):
value = tuple([elt.n for elt in key.elts])
else:
raise self._syntax_error(
"only keys of type 'str' or 'int' are supported in dicts",
node)
keys.append(_cl(ast.Constant(value=value), key))
values = [self.visit(item) for item in node.values]
return _cl(ast.Dict(keys=keys, values=values), node)
def __init__(self,
tree: ast_mod.Module,
module_name: str,
to_concat: bool,
module: ast_mod.Module = None):
self.functions = []
self.imports = [
ast_mod.Import(names=[ast_mod.alias(name='pytest', asname=None)]),
ast_mod.Import(names=[ast_mod.alias(name='io', asname=None)]),
ast_mod.ImportFrom(module='CommonServerPython',
names=[ast_mod.alias(name='*', asname=None)],
level=0)
]
self.module = module
self.server_url = ast_mod.Assign(
targets=[ast_name('SERVER_URL')],
value=ast_mod.Constant(value='https://test_url.com'))
self.tree = tree
self.module_name = module_name
self.global_args = []
self.to_concat = to_concat
def visit_With(self, node):
ret_node = super().visit_With(node)
if len(ret_node.items) > 1:
self.error(node, "With statements may only have one item")
new_nodes = []
if isinstance(ret_node.items[0].optional_vars, ast.Name):
n = ret_node.items[0].optional_vars.id
new_node = ast.Expr(value=ast.Call(func=ast.Name(
'_check_assign_name_', ast.Load()),
args=[ast.Constant(value=n)],
keywords=[]))
copy_locations(new_node, node)
new_nodes.append(new_node)
new_nodes.append(ret_node)
return new_nodes
def add_imports_to_all(dir: str) -> None:
init_file = root_module / dir / "__init__.py"
with open(init_file) as f:
init_ast = ast.parse(f.read(), filename=f"mautrix/{dir}/__init__.py")
imports: list[str] = []
all_node: ast.List | None = None
for node in ast.iter_child_nodes(init_ast):
if isinstance(node, (ast.Import, ast.ImportFrom)):
imports += (name.name for name in node.names)
elif isinstance(node, ast.Assign) and isinstance(node.value, ast.List):
target = node.targets[0]
if len(node.targets) == 1 and isinstance(
target, ast.Name) and target.id == "__all__":
all_node = node.value
all_node.elts = [ast.Constant(name) for name in imports]
with open(init_file, "w") as f:
f.write(black.format_str(ast.unparse(init_ast), mode=black_mode))
def print_shell_outputs(statement: ast.stmt) -> None:
"""Set print_it to True on every top level run_shell"""
if isinstance(statement, ast.FunctionDef):
for substatement in statement.body:
print_shell_outputs(substatement)
if not isinstance(statement, ast.Expr):
return
expr = statement.value
if (
isinstance(expr, ast.Call)
and isinstance(expr.func, ast.Name)
and expr.func.id == 'run_shell'
):
expr.keywords = [
ast.keyword(
arg='print_it',
value=ast.Constant(value=True),
)
]
def create_constant(value: Any, node: Optional[ast.AST] = None) -> ast.AST:
"""
Cross-Python-AST-version helper function that creates an AST constant node from a given value.
:param value: The value to create a constant from.
:param node: An optional node to copy the source location information from.
:return: An AST node (``ast.Constant`` after Python 3.8) that represents this value.
"""
if sys.version_info >= (3, 8):
newnode = ast.Constant(value=value, kind='')
else:
if value is None:
newnode = ast.NameConstant(value=None)
elif isinstance(value, str):
newnode = ast.Str(s=value)
else:
newnode = ast.Num(n=value)
if node is not None:
newnode = ast.copy_location(newnode, node)
return newnode
def visit_Assign(self, node: ast.Assign) -> Any:
if isinstance(node.value, ast.ListComp) and any(isinstance(c, ast.Yield) for c in walk(node.value.elt)):
kw = meta(node)
lcomp = node.value
# assert the ListComp is over only one generator and its target is a Name
iter_var = lcomp.generators[0].target
source_iter = lcomp.generators[0].iter
# assert there's only one target of the Assign, and it's a Name
target_list = node.targets[0]
target_list_load = deepcopy(target_list)
target_list_load.ctx = ast.Load()
emptylist = ast.List(elts=[], ctx=ast.Load(), **kw)
append_stmt = ast.Expr(value=ast.Call(
func=ast.Attribute(value=target_list_load, attr='append', ctx=ast.Load(), **kw),
args=[lcomp.elt],
keywords=[],
**kw), **kw)
forloop = ast.For(
target=iter_var,
iter=source_iter,
orelse=[],
body=[
append_stmt
], **kw)
# replacing with an If, just to have something to put in a single slot
replacements = [
ast.Assign(targets=[target_list], value=emptylist, **kw),
forloop
]
replacement = ast.If(test=ast.Constant(value=True, **kw),
body=replacements,
orelse=[],
**kw)
self.replace(replacements)
def visitAtom(self, ctx: expressoParser.AtomContext):
if ctx.expr():
return self.visit(ctx.expr())
elif ctx.ID():
id = str(ctx.ID())
if id not in self.id_table:
symbol = ctx.ID().getSymbol()
raise KeyError(self.file_name + ':' + str(symbol.line) + ':' +
str(symbol.column) + ' variable ' + id +
' is not defined')
return ast.Name(id=id, ctx=ast.Load())
elif ctx.INT():
return ast.Constant(value=int(str(ctx.INT())))
elif ctx.call():
return self.visit(ctx.call())
elif ctx.INPUT():
input_call = ast.Call(func=ast.Name(id='input', ctx=ast.Load()),
args=[],
keywords=[])
return ast.Call(func=ast.Name(id='int', ctx=ast.Load()),
args=[input_call],
keywords=[])
def test_code_runner():
assert should_quiet("1+1;")
assert not should_quiet("1+1#;")
assert not should_quiet("5-2 # comment with trailing semicolon ;")
# Normal usage
assert CodeRunner("1+1").compile().run() == 2
assert CodeRunner("x + 7").compile().run({"x": 3}) == 10
cr = CodeRunner("x + 7")
# Ast transform
import ast
l = cr.ast.body[0].value.left
cr.ast.body[0].value.left = ast.BinOp(left=l,
op=ast.Mult(),
right=ast.Constant(value=2))
assert cr.compile().run({"x": 3}) == 13
# Code transform
cr.code = cr.code.replace(co_consts=(0, 3, 5, None))
assert cr.run({"x": 4}) == 17
def visit_BinOp(self, node):
# transform the left and right arguments of the binary operation:
node.left = pythonTransformer().visit(node.left)
node.right = pythonTransformer().visit(node.right)
# formatted strings with %
# note: we have extended the pythong syntax slightly, to accommodate both tuples and lists
# so both '%_%' % (1,2) and '%_%' % [1,2] are successfully transpiled
if isinstance(node.op, ast.Mod) and isinstance(node.left, ast.Str):
# transform the node into an f-string node:
stringFormat = node.left.value
stringTuple = node.right.elts if (
isinstance(node.right, ast.Tuple) or isinstance(node.right, ast.List))\
else [node.right]
values = []
tupleIndex = 0
while True:
# TODO deal with more complicated formats, such as %.3f
match = re.search(r'%.', stringFormat)
if match is None:
break
values.append(ast.Constant(value=stringFormat[0:match.span(0)[0]], kind=None))
values.append(
self.visit_FormattedValue(
ast.FormattedValue(
value=stringTuple[tupleIndex],
conversion=-1,
format_spec=None
)
)
)
stringFormat = stringFormat[match.span(0)[1]:]
tupleIndex += 1
return ast.JoinedStr(values)
return node
def visit_BinOp(
self, ast_node_BinOp: ast.BinOp) -> Union[ast.BinOp, ast.Constant]:
"""
Evaluate binary operation and return ast.BinOp or ast.Constant if
operands are bound.
:param ast_node_BinOp: the AST node on which the binary operation
is performed.
:return: evaluated value.
"""
ast_node_BinOp.left = self.visit(ast_node_BinOp.left)
ast_node_BinOp.right = self.visit(ast_node_BinOp.right)
left, right = ast_node_BinOp.left, ast_node_BinOp.right
if isinstance(left, ast.Constant) and isinstance(right, ast.Constant):
new_ast_node_BinOp = ast.Constant(value=self._match_operator(
ast_node_BinOp.op, self._ast_math_BinaryOperators, left.value,
right.value))
return ast.copy_location(new_ast_node_BinOp, ast_node_BinOp)
return ast_node_BinOp
def setUp(self):
# Translated from https://github.com/gvanrossum/pegen/blob/ec2b354f64f6dbfcb46133757fe4c0e07880f526/test/test_pegen.py#L232
ident = Alpha + Alnum.repeat()
atom = (ident.value(lambda v: ast.Name(id=v, ctx=ast.Load()))
| Digit.repeat(1).value(
lambda v: ast.Constant(value=ast.literal_eval(v))))
expr = Forward()
factor = (Terminal('(').then(expr).skip(')') | atom)
term = Forward()
term.is_(
term.skip('*').then(
factor, lambda l, r: ast.BinOp(l, ast.Mult(), r)) ^ term.skip(
'/').then(factor, lambda l, r: ast.BinOp(l, ast.Div(), r))
^ factor)
expr.is_(
expr.skip('+').then(term, lambda l, r: ast.BinOp(l, ast.Add(), r))
^ expr.skip('-').then(
term, lambda l, r: ast.BinOp(l, ast.Sub(), r)) ^ term)
start = expr.skip(Terminal('\n').repeat(0, 1)).filter(
lambda r: not r.remain).value(lambda v: ast.fix_missing_locations(
ast.Expression(v, lineno=1, col_offset=0)))
self.parser = start
def add_before_call_used_value_capturing_call(code, node):
"""
When the method of some object is called, capture the value of the object before executing the method
"""
if hasattr(node.func, "value"):
old_value_node = node.func.value
value_code = astunparse.unparse(old_value_node)
new_value_node = ast.Call(
func=ast.Name(id='before_call_used_value', ctx=ast.Load()),
args=[
ast.Constant(n=False, kind=None),
ast.Constant(n=code, kind=None),
ast.Constant(n=value_code, kind=None), old_value_node,
ast.Constant(n=node.lineno, kind=None),
ast.Constant(n=node.col_offset, kind=None),
ast.Constant(n=node.end_lineno, kind=None),
ast.Constant(n=node.end_col_offset, kind=None)
],
keywords=[])
node.func.value = new_value_node
def check_assign_name(self, node, ret_node):
if isinstance(ret_node, list):
ret_node = ret_node[0]
# Add check for variable assignment to prevent overwriting
# builtins or global variables
names = []
if hasattr(node, 'targets'):
t = node.targets[0]
else:
t = node.target
if not (isinstance(t, ast.Tuple)):
if isinstance(t, ast.Name):
names.append(t.id)
elif isinstance(t, (ast.Tuple, ast.List)):
names.extend(self.get_assign_names_tuple(t))
else:
for t1 in t.elts:
if isinstance(t1, ast.Name):
names.append(t1.id)
elif isinstance(t1, (ast.Tuple, ast.List)):
names.extend(self.get_assign_names_tuple(t1))
new_nodes = []
if (len(names) != 0):
for n in names:
new_node = ast.Expr(value=ast.Call(
func=ast.Name('_check_assign_name_', ast.Load()),
args=[ast.Constant(value=n)],
keywords=[]))
copy_locations(new_node, node)
new_nodes.append(new_node)
new_nodes.append(ret_node)
return new_nodes
def get_instrument_node(self, trigger: str, name: str) -> ast.Expr:
if self.inst_type in ("log_var", "log_number"):
if self.inst_type == "log_var":
event = "instant"
elif self.inst_type == "log_number":
event = "counter"
return self.get_add_variable_node(name=f"{trigger} - {name}",
var_node=ast.Name(
id=name, ctx=ast.Load()),
event=event)
elif self.inst_type == "log_func_exec":
return self.get_add_func_exec_node(name=f"{name}",
val=ast.Name(id=name,
ctx=ast.Load()),
lineno=self.curr_lineno)
elif self.inst_type == "log_func_entry":
return self.get_add_variable_node(
name=f"{trigger} - {name}",
var_node=ast.Constant(value="{} is called".format(name)),
event="instant")
else:
raise ValueError("{} is not supported".format(name))
def add_before_call_used_args_capturing_call(code, node):
"""
When a method is called, capture the arguments of the method before executing it
"""
old_args_nodes_ast = ast.List(node.args, ctx=ast.Load())
old_args_code = ast.List([
ast.Constant(n=astunparse.unparse(arg).split("\n", 1)[0],
kind=None) for arg in node.args
],
ctx=ast.Load())
new_args_node = ast.Starred(value=ast.Call(
func=ast.Name(id='before_call_used_args', ctx=ast.Load()),
args=[
ast.Constant(n=False, kind=None),
ast.Constant(n=code, kind=None), old_args_code,
ast.Constant(n=node.lineno, kind=None),
ast.Constant(n=node.col_offset, kind=None),
ast.Constant(n=node.end_lineno, kind=None),
ast.Constant(n=node.end_col_offset, kind=None),
ast.Constant(n=False, kind=None), old_args_nodes_ast
],
keywords=[]),
ctx=ast.Load())
node.args = [new_args_node]
def visit_For(self, node):
iter_sym = Symbol()
return self.visit(ast.copy_location(ast.Expr(
value=LetBinding(
symbol=iter_sym,
inner=iter_sym,
bound_expr=ast.Call(
func=ast.Name(id="iter", ctx=ast.Load()),
args=[node.iter],
keywords=[]
),
ex_stmts=[
ast.While(
test=ast.Constant(value=True),
body=[ast.Try(
body=[ast.Assign(
targets=[node.target],
value=ast.Call(
func=ast.Name(id="next", ctx=ast.Load()),
args=[iter_sym],
keywords=[]
)
)],
handlers=[ast.ExceptHandler(
type=ast.Name(id="StopIteration", ctx=ast.Load()),
name=None,
body=[
ast.Break()
]
)],
orelse=[],
finalbody=[]
)] + node.body,
orelse=node.orelse
)
]
)
), node))
def visit_Field(self, node):
target = ast.Name(node.name, ast.Store())
annotation = ast.Name(node.kind, ast.Load())
if node.qualifier is not None:
if node.qualifier is pyasdl.FieldQualifier.SEQUENCE:
qualifier = "List"
default = ast.Call(
_FIELD,
[],
[
ast.keyword(
"default_factory", ast.Name("list", ast.Load())
)
],
)
elif node.qualifier is pyasdl.FieldQualifier.OPTIONAL:
qualifier = "Optional"
default = ast.Call(
_FIELD, [], [ast.keyword("default", ast.Constant(None))]
)
else:
raise ValueError(
f"Unexpected field qualifier: {node.qualifier}"
)
annotation = ast.Subscript(
value=ast.Attribute(_TYPING, qualifier, ast.Load()),
slice=annotation,
ctx=ast.Load(),
)
else:
# no-default!
default = None
if not self.with_defaults:
default = None
return ast.AnnAssign(target, annotation, default, simple=1)
def visit_Assign(self, node):
if self._disable:
return node
target = node.targets[0]
if isinstance(target, ast.Tuple) and not (isinstance(
target.elts[0], ast.Name)):
temps = []
orig = [None] * len(target.elts)
body = [node]
for i in range(len(target.elts)):
temps.append(ast.Name(id=self._get_temp(), lineno=node.lineno))
# The irony!
target.elts[i], orig[i] = temps[i], target.elts[i]
body.append(
ast.Assign(targets=[orig[i]],
value=temps[i],
lineno=node.lineno))
ret = ast.If(test=ast.Constant(value=True),
body=body,
orelse=[],
lineno=node.lineno)
return ret
return node
def add_after_call_used_capturing(code, node, subscript):
"""
After a method got executed, capture the return value
"""
instrumented_call_node = ast.Call(
func=ast.Name(id='after_call_used', ctx=ast.Load()),
args=[
ast.Constant(n=subscript, kind=None),
ast.Constant(n=code, kind=None), node,
ast.Constant(n=node.lineno, kind=None),
ast.Constant(n=node.col_offset, kind=None),
ast.Constant(n=node.end_lineno, kind=None),
ast.Constant(n=node.end_col_offset, kind=None)
],
keywords=[])
instrumented_call_node = ast.copy_location(instrumented_call_node,
node)
return instrumented_call_node
def test_capture_nested(self):
ast_node = ast.List(elts=[
ast.Constant(value=1),
ast.Constant(value=2),
ast.Constant(value=3)
])
pattern_node = ast.List(elts=[
ast.Constant(value=anm.Capture("first")),
ast.Constant(value=anm.Capture("second")),
ast.Constant(value=anm.Capture("third")),
])
captures = {}
matches = anm.match(ast_node, pattern_node, captures)
assert matches
assert captures["first"] == 1
assert captures["second"] == 2
assert captures["third"] == 3
pass
def visit_Assign(self, node):
"""Rewrite an assignment expression.
Before::
x = y + z
After::
x = _ptera_interact('x', None, y + z)
"""
(target, ) = node.targets
if isinstance(target, ast.Tuple):
var_all = gensym()
ass_all = ast.copy_location(
ast.Assign(
targets=[ast.Name(id=var_all, ctx=ast.Store())],
value=node.value,
),
node,
)
accum = [ass_all]
for i, tgt in enumerate(target.elts):
accum += self.visit_Assign(
ast.copy_location(
ast.Assign(
targets=[tgt],
value=ast.Subscript(
value=ast.Name(id=var_all, ctx=ast.Load()),
slice=ast.Index(value=ast.Constant(i)),
ctx=ast.Load(),
),
),
node,
))
return accum
else:
return self.make_interaction(target, None, node.value, orig=node)
def process_func_args(self, func: nodes.FunctionDef,
z3_result) -> List[ast_mod.Constant]:
"""
from a function arguments, query the args from the z3 model, and use `model.eval(model_completion=True)
for argument not part of constraint.
For argument that contain defaults, if it's in the model, the arg will have the model's value.
If it's not in the model, the defaults will be used.
"""
args = []
def fail():
MANAGER.logger.warning(
"CONTRACT",
"Failed to determine the return result for argument: {}. ",
arg)
raise ValueError
for arg in func.args.args:
try:
res = next(arg.infer(self.context))
if res.status:
expanded_res = res.expand(self.context, z3_result)
if expanded_res.status and isinstance(
expanded_res.result,
(nodes.Const, nodes.NameConstant)):
returned_result = ast_mod.Constant(
expanded_res.result.value)
args.append(returned_result)
else:
fail()
else:
fail()
except StopIteration:
fail()
return args
def _annotate_object(schema, resolve, names=()):
# https://github.com/python/mypy/issues/7654
names = list(names)
name = _case_pascal("-".join(names))
name_object = f"_{name}Dict"
name_required = f"_{name}Required"
name_optional = f"_{name}Optional"
properties = schema.get("properties", {})
required = set(schema.get("required", []))
required_not = set(properties.keys()) - required
preamble = []
anno_required = {}
anno_optional = {}
for annotations, fields in [(anno_required, required),
(anno_optional, required_not)]:
for field in fields:
annotation, extra = _annotate_schema(properties[field], resolve,
names + [field])
annotations[field] = annotation
preamble.extend(extra)
ass_required = _typed_dict(name_required, True, anno_required)
ass_optional = _typed_dict(name_optional, False, anno_optional)
class_ = ast.ClassDef(
name=name_object,
bases=[ast.Name(id=name_required),
ast.Name(id=name_optional)],
body=[ast.Constant(value=...)],
keywords=[],
decorator_list=[],
)
return (
ast.Name(id=name_object),
preamble + [ass_required, ass_optional, class_],
)
