def parse(self):
"""
Convert the parsed tokens into a list of expressions then join them
"""
self.stream = tokenise(self.source)
steps = []
for token in self.stream:
code = self._token_to_code(token)
if code:
steps.append(code)
# result = [str(x) for x in steps]
return ast.Expression(
body=ast.ListComp(
elt=build_call(
ast.Name(id='str', ctx=ast.Load()),
args=[
ast.Name(id='x', ctx=ast.Load()),
],
),
generators=[
ast.comprehension(
target=ast.Name(id='x', ctx=ast.Store()),
iter=ast.List(elts=steps, ctx=ast.Load()),
ifs=[]
)
]
)
)
def _init_non_primitive_nested_class(node_assign, object_, prop):
"""
If the nested list is non-primitive, initialise sub-classes in a list comp
If the nest is primitive, we can simply get it
Marshmallow will do the type marshalling
"""
return ast.ListComp(
elt=ast.Call(
func=ast.Name(id=ObjectGenerator._nesting_class(node_assign)),
args=[ast.Name(id="el")],
keywords=[],
),
generators=[
ast.comprehension(
target=ast.Name(id="el"),
iter=ast.Call(
func=ast.Attribute(value=ast.Name(id=object_),
attr="get"),
args=[ast.Str(s=prop),
ast.Dict(keys=[], values=[])],
keywords=[],
),
ifs=[],
is_async=0,
)
],
)
def visit_ListComp(self, node: ListComp, *args, **kwargs) -> C.ListComp:
elt = self.visit(node.elt, *args, **kwargs)
generators = self.visit(node.generators, *args, **kwargs)
return C.ListComp(
elt=elt,
generators=generators,
)
def visitListComp(self, n, varchecks, *args):
# In comprehensions, we can't generate protectors to guard
# arguments since the body is just an expression. Instead we
# add variables to varchecks to indicate in visitName that the
# variable should be checked directly. This can lead to
# duplicated checks but I suspect that's relatively rare.
generators, varchecks = self.handleComprehensions(
n.generators, n.lineno, n.col_offset, varchecks, *args)
elt = self.dispatch(n.elt, varchecks, *args)
return ast.ListComp(elt=elt, generators=generators)
def generate(self, element:Element, GC:GenerationContext):
acode = element.code
if len(acode) is 2 and is_form(acode[1], "for"):
for_form = acode[1].code
# list comprehension
# («[]» (for (in i lst) (f i))) # list compr
in_el = for_form[1]
in_el_code = in_el.code
#with GC.let(domain=ExDom):
assert is_identifier(in_el, "in")
target_element = in_el_code[1]
iter_element = in_el_code[2]
with GC.let(domain=LVDom):
target_code = GC.generate(target_element)
with GC.let(domain=ExDom):
iter_code = GC.generate(iter_element)
generators = [ ast.comprehension(target=target_code,
iter=iter_code,
ifs=[]) ]
to_evaluate_element = for_form[2]
with GC.let(domain=ExDom):
to_evaluate_code = GC.generate(to_evaluate_element)
return ast.ListComp(to_evaluate_code, generators)
else:
els = self.generate_as_expressions(GC, *acode[1:])
if GC.domain == LVDom:
return ast.List(els, ast.Store())
return expr_wrap(ast.List(els, ast.Load()), GC)
def genProcessCollectionToList(
funcName: str,
parentName: str,
iterFuncName: str,
processorFunc: ast.Attribute,
firstArgName: str = "parsed",
iterVarName: str = "f",
returnType: None = None) -> typing.Iterator[ast.FunctionDef]:
"""Generates a function transforming a collection (`min`) or a macro around it into a python `list` of nodes"""
o = ast.Name(id=firstArgName, ctx=ast.Load())
yield ast.FunctionDef(
name=funcName,
args=ast.arguments(
posonlyargs=[],
args=[
astSelfArg,
ast.arg(arg=firstArgName, annotation=None, type_comment=None),
],
vararg=None,
kwonlyargs=[],
kw_defaults=[],
kwarg=None,
defaults=[],
),
body=[
ast.Return(value=ast.ListComp(
elt=ast.Call(
func=processorFunc,
args=[ast.Name(id=iterVarName)],
keywords=[],
),
generators=[
ast.comprehension(
target=ast.Name(id=iterVarName),
iter=ast.Call(func=ast.Attribute(
value=ASTSelf, attr=iterFuncName, ctx=ast.Load()),
args=[o],
keywords=[]),
ifs=[],
is_async=0)
],
))
],
decorator_list=[],
returns=(genTypingIterable(returnType) if returnType else None),
type_comment=None,
)
def handle_str(node):
"""Replaces an ast.Constant containing a string, with an ascii representation joined together
Args:
**node (:obj: `ast.Constant`)**: The node to replace with the ast.Call node
Returns:
A new obscured node with updated locations
"""
if type(node.parent) != ast.FormattedValue and type(
node.parent.parent) != ast.FormattedValue and type(
node.parent) != ast.JoinedStr and type(
node.parent.parent) != ast.JoinedStr:
new_node = ast.Call(
func=ast.Attribute(
value=ast.Constant(value='', kind=None),
attr='join',
ctx=ast.Load(),
),
args=[
ast.ListComp(
elt=ast.Call(func=ast.Name(id='chr', ctx=ast.Load()),
args=[ast.Name(id='x', ctx=ast.Load())],
keywords=[]),
generators=[
ast.comprehension(
target=ast.Name(id='x', ctx=ast.Store()),
iter=ast.List(
elts=[
ast.Constant(value=ord(x), kind=None)
for x in node.value
],
ctx=ast.Load(),
),
ifs=[],
is_async=0,
),
],
),
],
keywords=[])
return ast.copy_location(new_node, node)
else:
return node
def makeGroup(name, base_score, useNumPy=None):
groupTemp = ast.Subscript(value=treesAccumulator,
slice=ast.Index(value=ast.Name(name)))
code = []
res = ast.Name(id="res")
baseScore = astNum(n=base_score)
treeCall = ast.Call(func=treeTemp, args=[vector], keywords=[])
iterArgDict = {
"target": treeTemp,
"iter": groupTemp,
}
if not useNumPy:
code.append(ast.Assign(targets=[res], value=baseScore))
code.append(
ast.For(
body=[ast.AugAssign(target=res, op=ast.Add(), value=treeCall)],
orelse=[],
**iterArgDict,
))
else:
res = ast.BinOp(
left=baseScore,
op=ast.Add(),
right=ast.Call(
func=numpySum,
args=[
ast.ListComp(
elt=treeCall,
generators=[
ast.comprehension(
ifs=[],
**iterArgDict,
)
],
)
],
keywords=[],
),
)
code.append(ast.Return(res))
return wrapWithFunc(code, name)
def compile_list_comprehension(self, expr):
# (list-comp expr (target iter) cond?)
expr.pop(0)
expression = expr.pop(0)
tar_it = iter(expr.pop(0))
targets = zip(tar_it, tar_it)
cond = self.compile(expr.pop(0)) if expr != [] else None
ret = ast.ListComp(lineno=expr.start_line,
col_offset=expr.start_column,
elt=self.compile(expression),
generators=[])
for target, iterable in targets:
ret.generators.append(
ast.comprehension(target=self._storeize(self.compile(target)),
iter=self.compile(iterable),
ifs=[]))
if cond:
ret.generators[-1].ifs.append(cond)
return ret
def visit_SetComp(self, node):
node = self.generic_visit(node)
listexpr = ast.ListComp(elt=node.elt, generators=node.generators)
return mk_call('set', [listexpr])
def sequence_helper(self, root, sentence):
for i in range(len(root.children)):
self.sequence_helper(root.children[i], sentence)
for child in root.children[i].children:
assert (isinstance(child, LeafNode))
if isinstance(root.children[i], LoopNode):
loop_node = root.children[i]
loop_var = None
iter_obj = None
if not loop_node.body_ctx.computed:
loop_node.body_ctx.computed = True
for key, val in zip(loop_node.children_keys,
loop_node.children):
target = ast.Name(id=key, ctx=ast.Store())
value = self.choose_child_leaf_node(val)
init_assignement = ast.Assign(targets=[target],
value=value)
# checking forloop case
if is_forloop_initializer(init_assignement):
loop_var = get_forloop_var(init_assignement)
iter_obj = get_forloop_iter_obj(init_assignement)
else:
sentence.append(init_assignement)
iter_stmt = self.translate_ctx(loop_node.body_ctx.body)
if loop_var != None and iter_obj != None:
loop_node_stmt = loop_node.compute_forloop_stmt(
iter_stmt, loop_var, iter_obj)
else:
loop_node_stmt = loop_node.compute_whileloop_stmt(
iter_stmt)
sentence += loop_node_stmt
param_node = Param(loop_node.id, loop_node.output_var)
self.peg.replace_node(loop_node, param_node)
if isinstance(root.children[i], BranchNode):
branch_node = root.children[i]
if not branch_node.body_ctx.computed:
for key, val in zip(branch_node.children_keys,
branch_node.children):
target = ast.Name(id=key, ctx=ast.Store())
value = self.choose_child_leaf_node(val)
init_assignement = ast.Assign(targets=[target],
value=value)
sentence.append(init_assignement)
true_stmt = self.translate_ctx(
branch_node.body_ctx.true_branch)
false_stmt = self.translate_ctx(
branch_node.body_ctx.false_branch)
branch_node_stmt = branch_node.compute_stmt(
true_stmt, false_stmt)
branch_node.body_ctx.computed = True
for stmt in branch_node_stmt:
sentence.append(stmt)
param_node = Param(branch_node.id, branch_node.output_var)
self.peg.replace_node(branch_node, param_node)
if isinstance(root.children[i], BinOpNode):
operator = root.children[i]
fresh_var = self.create_fresh_var('op')
target = ast.Name(id=fresh_var, ctx=ast.Store())
args = [
self.choose_child_leaf_node(arg)
for arg in operator.children
]
op_exp = ast.BinOp(left=args[0], op=operator.op, right=args[1])
self.process_stmt(sentence, operator, fresh_var, op_exp)
param_node = Param(operator.id, fresh_var)
self.peg.replace_node(operator, param_node)
if isinstance(root.children[i], FunctionCall):
func_call = root.children[i]
fresh_var = self.create_fresh_var('func_call')
target = ast.Name(id=fresh_var, ctx=ast.Store())
name = self.choose_child_leaf_node(func_call.name())
args = [
self.choose_child_leaf_node(arg)
for arg in func_call.args()
]
func_call_exp = ast.Call(func=name, args=args, keywords=[])
self.process_stmt(sentence, func_call, fresh_var,
func_call_exp)
param_node = Param(func_call.id, fresh_var)
self.peg.replace_node(func_call, param_node)
if isinstance(root.children[i], IfExpNode):
if_e = root.children[i]
fresh_var = self.create_fresh_var('if_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
cond = self.choose_child_leaf_node(if_e.cond())
t = self.choose_child_leaf_node(if_e.t())
f = self.choose_child_leaf_node(if_e.f())
if_e_exp = ast.IfExp(test=cond, body=t, orelse=f)
self.process_stmt(sentence, if_e, fresh_var, if_e_exp)
param_node = Param(if_e.id, fresh_var)
self.peg.replace_node(if_e, param_node)
if isinstance(root.children[i], CompareNode):
compare = root.children[i]
fresh_var = self.create_fresh_var('comp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
head = self.choose_child_leaf_node(compare.head())
tail = [
self.choose_child_leaf_node(arg) for arg in compare.tail()
]
cmp_exp = ast.Compare(left=head,
ops=compare.ops,
comparators=tail)
self.process_stmt(sentence, compare, fresh_var, cmp_exp)
param_node = Param(compare.id, fresh_var)
self.peg.replace_node(compare, param_node)
if isinstance(root.children[i], BoolOpNode):
bool_op_node = root.children[i]
fresh_var = self.create_fresh_var('bool_op')
target = ast.Name(id=fresh_var, ctx=ast.Store())
values = [
self.choose_child_leaf_node(arg)
for arg in bool_op_node.children
]
bool_op_exp = ast.BoolOp(op=bool_op_node.op, values=values)
self.process_stmt(sentence, bool_op_node, fresh_var,
bool_op_exp)
param_node = Param(bool_op_node.id, fresh_var)
self.peg.replace_node(bool_op_node, param_node)
if isinstance(root.children[i], UnaryOpNode):
un_op_node = root.children[i]
fresh_var = self.create_fresh_var('unary_op_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
operand = self.choose_child_leaf_node(un_op_node.operand())
un_op_exp = ast.UnaryOp(op=un_op_node.op, operand=operand)
self.process_stmt(sentence, un_op_node, fresh_var, un_op_exp)
param_node = Param(un_op_node.id, fresh_var)
self.peg.replace_node(un_op_node, param_node)
if isinstance(root.children[i], ListNode):
list_node = root.children[i]
fresh_var = self.create_fresh_var('list_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in list_node.children
]
list_exp = ast.List(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, list_node, fresh_var, list_exp)
param_node = Param(list_node.id, fresh_var)
self.peg.replace_node(list_node, param_node)
if isinstance(root.children[i], TupleNode):
tuple_node = root.children[i]
fresh_var = self.create_fresh_var('tuple_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in tuple_node.children
]
tuple_exp = ast.Tuple(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, tuple_node, fresh_var, tuple_exp)
param_node = Param(tuple_node.id, fresh_var)
self.peg.replace_node(tuple_node, param_node)
if isinstance(root.children[i], SetNode):
set_node = root.children[i]
fresh_var = self.create_fresh_var('set_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in set_node.children
]
set_exp = ast.Set(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, set_node, fresh_var, set_exp)
param_node = Param(set_node.id, fresh_var)
self.peg.replace_node(set_node, param_node)
if isinstance(root.children[i], DictNode):
dict_node = root.children[i]
fresh_var = self.create_fresh_var('dict_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
keys = [
self.choose_child_leaf_node(k) for k in dict_node.keys()
]
values = [
self.choose_child_leaf_node(v) for v in dict_node.values()
]
dict_exp = ast.Dict(keys=keys, values=values, ctx=ast.Load())
self.process_stmt(sentence, dict_node, fresh_var, dict_exp)
param_node = Param(dict_node.id, fresh_var)
self.peg.replace_node(dict_node, param_node)
if isinstance(root.children[i], AttributeNode):
attr_node = root.children[i]
fresh_var = self.create_fresh_var('attr_exp')
value = self.choose_child_leaf_node(attr_node.value())
attr = attr_node.attr
attr_exp = ast.Attribute(value=value,
attr=attr,
ctx=ast.Load())
self.rhs_assignement_expressions[attr_node.id] = attr_exp
param_node = Param(attr_node.id, fresh_var)
self.peg.replace_node(attr_node, param_node)
if isinstance(root.children[i], SubscriptNode):
subs_node = root.children[i]
fresh_var = self.create_fresh_var('subs_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
value = self.choose_child_leaf_node(subs_node.value())
slice = self.choose_child_leaf_node(subs_node.slice())
subs_exp = ast.Subscript(value=value,
slice=slice,
ctx=ast.Load())
self.process_stmt(sentence, subs_node, fresh_var, subs_exp)
param_node = Param(subs_node.id, fresh_var)
self.peg.replace_node(subs_node, param_node)
if isinstance(root.children[i], IndexNode):
idx_node = root.children[i]
fresh_var = self.create_fresh_var('idx_exp')
value = self.choose_child_leaf_node(idx_node.value())
idx_exp = ast.Index(value=value)
# never append just index to sentence
self.rhs_assignement_expressions[idx_node.id] = idx_exp
param_node = Param(idx_node.id, fresh_var)
self.peg.replace_node(idx_node, param_node)
if isinstance(root.children[i], SliceNode):
slice_node = root.children[i]
fresh_var = self.create_fresh_var('slice_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
lower = self.choose_child_leaf_node(slice_node.lower())
upper = self.choose_child_leaf_node(slice_node.upper())
step = self.choose_child_leaf_node(slice_node.step())
slice_exp = ast.Slice(lower=lower, upper=upper, step=step)
self.rhs_assignement_expressions[slice_node.id] = slice_exp
param_node = Param(slice_node.id, fresh_var)
self.peg.replace_node(slice_node, param_node)
if isinstance(root.children[i], ExtSliceNode):
eslice_node = root.children[i]
fresh_var = self.create_fresh_var('eslice_exp')
dims = [
self.choose_child_leaf_node(d) for d in eslice_node.dims()
]
eslice_exp = ast.ExtSlice(dims=dims)
self.rhs_assignement_expressions[eslice_node.id] = eslice_exp
param_node = Param(eslice_node.id, fresh_var)
self.peg.replace_node(eslice_node, param_node)
if isinstance(root.children[i], NPArrayNode):
ndarr_node = root.children[i]
fresh_var = self.create_fresh_var('ndarr_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
lists = self.choose_child_leaf_node(ndarr_node.children[0])
ndarr_attr = ast.Attribute(value=ast.Name(id='np',
ctx=ast.Load()),
attr='array')
ndarr_exp = ast.Call(func=ndarr_attr,
args=[lists],
keywords=[])
self.process_stmt(sentence, ndarr_node, fresh_var, ndarr_exp)
param_node = Param(ndarr_node.id, fresh_var)
self.peg.replace_node(ndarr_node, param_node)
if isinstance(root.children[i], ComprehensionNode):
comp_node = root.children[i]
fresh_var = self.create_fresh_var('compr_exp')
t = self.choose_child_leaf_node(comp_node.target())
iter_obj = self.choose_child_leaf_node(comp_node.iter_obj())
ifs = [self.choose_child_leaf_node(i) for i in comp_node.ifs()]
comp_exp = ast.comprehension(target=t, iter=iter_obj, ifs=ifs)
self.rhs_assignement_expressions[comp_node.id] = comp_exp
param_node = Param(comp_node.id, fresh_var)
self.peg.replace_node(comp_node, param_node)
if isinstance(root.children[i], ListCompNode):
lcomp_node = root.children[i]
fresh_var = self.create_fresh_var('lcomp_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elt = self.choose_child_leaf_node(lcomp_node.element())
generators = [
self.choose_child_leaf_node(gen)
for gen in lcomp_node.generators()
]
lcomp_exp = ast.ListComp(elt=elt, generators=generators)
self.process_stmt(sentence, lcomp_node, fresh_var, lcomp_exp)
param_node = Param(lcomp_node.id, fresh_var)
self.peg.replace_node(lcomp_node, param_node)
def ListComp(draw, expression) -> ast.ListComp:
return ast.ListComp(elt=draw(expression),
generators=draw(
lists(comprehension(expression),
min_size=1,
max_size=3)))
def __build_function(self, dom_name, full_name, func_params):
assert 'name' in func_params
func_name = func_params['name']
docstr = self.__build_desc_string(dom_name, func_name, func_params)
args = [ast.arg('self', None)]
message_params = []
func_body = []
if docstr:
func_body.append(ast.Expr(ast.Str("\n"+docstr+"\n\t\t")))
for param in func_params.get("parameters", []):
argname = param['name']
param_optional = param.get("optional", False)
if param_optional is False:
message_params.append(ast.keyword(argname, ast.Name(id=argname, ctx=ast.Load())))
args.append(ast.arg(argname, None))
if self.do_debug_prints:
func_body.append(self.__build_debug_print(argname, argname))
param_type = param.get("type", None)
if param_type in CHECKS:
if param_optional:
check = self.__build_conditional_arg_check(argname, CHECKS[param_type])
else:
check = self.__build_unconditional_arg_check(argname, CHECKS[param_type])
if check:
func_body.append(check)
optional_params = [param.get("name") for param in func_params.get("parameters", []) if param.get("optional", False)]
func_kwargs = None
if len(optional_params):
value = ast.List(elts=[ast.Str(s=param, ctx=ast.Store()) for param in optional_params], ctx=ast.Load())
create_list = ast.Assign(targets=[ast.Name(id='expected', ctx=ast.Store())], value=value)
func_body.append(create_list)
passed_arg_list = ast.Assign(targets=[ast.Name(id='passed_keys', ctx=ast.Store())],
value=ast.Call(func=ast.Name(id='list', ctx=ast.Load()),
args=[ast.Call(func=ast.Attribute(value=ast.Name(id='kwargs', ctx=ast.Load()), attr='keys', ctx=ast.Load()), args=[], keywords=[])],
keywords=[]))
func_body.append(passed_arg_list)
comprehension = ast.comprehension(target=ast.Name(id='key', ctx=ast.Store()), iter=ast.Name(id='passed_keys', ctx=ast.Load()), ifs=[], is_async=False)
comparator = ast.Name(id='expected', ctx=ast.Load())
listcomp = ast.ListComp(elt=ast.Compare(left=ast.Name(id='key', ctx=ast.Load()), ops=[ast.In()], comparators=[comparator]), generators=[comprehension])
check_message = ast.BinOp(
left = ast.Str(s="Allowed kwargs are {}. Passed kwargs: %s".format(optional_params)),
op = ast.Mod(),
right = ast.Name(id='passed_keys', ctx=ast.Load()),
lineno = self.__get_line())
kwarg_check = ast.Assert(test=ast.Call(func=ast.Name(id='all', ctx=ast.Load()), args=[listcomp], keywords=[]), msg=check_message)
func_body.append(kwarg_check)
func_kwargs = ast.Name(id='kwargs', ctx=ast.Load())
fname = "{}.{}".format(dom_name, func_name)
fname = ast.Str(s=fname, ctx=ast.Load())
if (sys.version_info[0], sys.version_info[1]) == (3, 5) or \
(sys.version_info[0], sys.version_info[1]) == (3, 6):
# More irritating minor semantic differences in the AST between 3.4 and 3.5
if func_kwargs:
message_params.append(ast.keyword(arg=None, value=ast.Name(id='kwargs', ctx=ast.Load())))
communicate_call = ast.Call(
func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'),
args=[fname],
keywords=message_params)
elif (sys.version_info[0], sys.version_info[1]) == (3,4):
communicate_call = ast.Call(
func=ast.Attribute(value=ast.Name(id='self', ctx=ast.Load()), ctx=ast.Load(), attr='synchronous_command'),
args=[fname],
kwargs=func_kwargs,
keywords=message_params)
else:
print("Version:", sys.version_info)
raise RuntimeError("This script only functions on python 3.4, 3.5 or 3.6. Active python version {}.{}".format(*sys.version_info))
do_communicate = ast.Assign(targets=[ast.Name(id='subdom_funcs', ctx=ast.Store())], value=communicate_call)
func_ret = ast.Return(value=ast.Name(id='subdom_funcs', ctx=ast.Load()))
if len(optional_params) and self.do_debug_prints:
func_body.append(self.__build_debug_print('kwargs', 'kwargs'))
func_body.append(do_communicate)
func_body.append(func_ret)
if len(optional_params):
kwarg = ast.arg(arg='kwargs', annotation=None)
else:
kwarg = None
sig = ast.arguments(
args=args,
vararg=None,
varargannotation=None,
kwonlyargs=[],
kwarg=kwarg,
kwargannotation=None,
defaults=[],
kw_defaults=[])
func = ast.FunctionDef(
name = "{}_{}".format(full_name, func_name),
args = sig,
body = func_body,
decorator_list = [],
lineno = self.__get_line(),
col_offset = 0,
)
return func
kw_defaults=[1],
kwonlyargs=[],
),
ast.AnnAssign(simple=True, target=ast.Tuple(ast.Name("a",
ast.Load()))),
ast.With(items=[]),
ast.Raise(exc=None, cause=1),
ast.Try(body=[]),
ast.Try(body=[ast.Pass()], handlers=[], finalbody=[]),
ast.Try(
body=[ast.Pass()],
handlers=[],
finalbody=[ast.Pass()],
orelse=[ast.Pass()],
),
ast.ImportFrom(level=-1, names=[1]),
ast.BoolOp(values=[1]),
ast.Dict(keys=[1], values=[]),
ast.Compare(comparators=[]),
ast.Compare(comparators=[1], ops=[]),
ast.Constant(value=int),
ast.Constant(value=(1, 2, int)),
ast.ListComp(generators=[]),
],
)
def test_simple_ast_validator(node):
validator = SimpleASTValidator()
with pytest.raises(SyntaxError) as cm:
validator.validate(ast.fix_missing_locations(node))
assert cm.value.node is node
def test_operators(self):
boolop0 = ast.BoolOp()
boolop1 = ast.BoolOp(ast.And(),
[ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())])
boolop2 = ast.BoolOp(ast.And(),
[ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())],
0, 0)
binop0 = ast.BinOp()
binop1 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3))
binop2 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3), 0, 0)
unaryop0 = ast.UnaryOp()
unaryop1 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()))
unaryop2 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()), 0, 0)
lambda0 = ast.Lambda()
lambda1 = ast.Lambda(ast.arguments([ast.Name('x', ast.Param())], None, None, []), ast.Name('x', ast.Load()))
ifexp0 = ast.IfExp()
ifexp1 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0))
ifexp2 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0), 0, 0)
dict0 = ast.Dict()
dict1 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')])
dict2 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')], 0, 0)
set0 = ast.Set()
set1 = ast.Set([ast.Num(1), ast.Num(2)])
set2 = ast.Set([ast.Num(1), ast.Num(2)], 0, 0)
lc0 = ast.ListComp()
lc1 = ast.ListComp( ast.Name('x',ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
lc2 = ast.ListComp( ast.Name('x',ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])], 0, 0)
setcomp0 = ast.SetComp()
setcomp1 = ast.SetComp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()), ast.Str('abracadabra'),
[ast.Compare(ast.Name('x', ast.Load()), [ast.NotIn()],
[ast.Str('abc')])])])
comprehension0 = ast.comprehension()
comprehension1 = ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])
# "{i : chr(65+i) for i in (1,2)}")
dictcomp0 = ast.DictComp()
dictcomp1 = ast.DictComp(ast.Name('i', ast.Load()),
ast.Call(ast.Name('chr', ast.Load()),
[ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
[], None, None),
[ast.comprehension(ast.Name('i', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])])
dictcomp2 = ast.DictComp(ast.Name('i', ast.Load()),
ast.Call(ast.Name('chr', ast.Load()),
[ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
[], None, None),
[ast.comprehension(ast.Name('i', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])],0,0)
# (x for x in (1,2))
genexp0 = ast.GeneratorExp()
genexp1 = ast.GeneratorExp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
genexp2 = ast.GeneratorExp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])],0,0)
# yield 2
yield0 = ast.Yield()
yield1 = ast.Yield(ast.Num(2))
yield2 = ast.Yield(ast.Num(2),0,0)
yield20 = ast.Yield(lineno=0, col_offset=0)
# a>0
compare0 = ast.Compare()
compare1 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)])
compare2 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)],0,0)
# chr(65)
call0 = ast.Call()
call1 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None)
call2 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None, 0, 0)
call20 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [])
call21 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [], lineno=0, col_offset=0)
# 0
num0 = ast.Num()
num1 = ast.Num(0)
num2 = ast.Num(0,0,0)
# "foo"
str0 = ast.Str()
str1 = ast.Str("foo")
str2 = ast.Str("foo",0,0)
# TODO: come back
repr0 = ast.Repr()
repr1 = ast.Repr(ast.Num(0))
repr2 = ast.Repr(ast.Num(0),0,0)
# foo.bar
attr0 = ast.Attribute()
attr1 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load())
attr2 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load(), 0,0)
# a[1:2]
subscript0 = ast.Subscript()
subscript1 = ast.Subscript(ast.Name('a', ast.Load()), ast.Slice(ast.Num(1), ast.Num(2)), ast.Load())
subscript2 = ast.Subscript(ast.Name('a', ast.Load()), ast.ExtSlice([ast.Num(1), ast.Num(2)]), ast.Load(), 0, 0)
# name
name0 = ast.Name()
name1 = ast.Name("name", ast.Load())
name2 = ast.Name("name", ast.Load(),0,0)
# [1,2]
list0 = ast.List()
list1 = ast.List([ast.Num(1), ast.Num(2)], ast.Load())
list2 = ast.List([ast.Num(1), ast.Num(2)], ast.Load(),0,0)
# (1,2)
tuple0 = ast.Tuple()
tuple1 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load())
tuple2 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load(), 0, 0)
def test_empty_init(self):
# Jython 2.5.0 did not allow empty constructors for many ast node types
# but CPython ast nodes do allow this. For the moment, I don't see a
# reason to allow construction of the super types (like ast.AST and
# ast.stmt) as well as the op types that are implemented as enums in
# Jython (like boolop), but I've left them in but commented out for
# now. We may need them in the future since CPython allows this, but
# it may fall under implementation detail.
#ast.AST()
ast.Add()
ast.And()
ast.Assert()
ast.Assign()
ast.Attribute()
ast.AugAssign()
ast.AugLoad()
ast.AugStore()
ast.BinOp()
ast.BitAnd()
ast.BitOr()
ast.BitXor()
ast.BoolOp()
ast.Break()
ast.Call()
ast.ClassDef()
ast.Compare()
ast.Continue()
ast.Del()
ast.Delete()
ast.Dict()
ast.Div()
ast.Ellipsis()
ast.Eq()
ast.Exec()
ast.Expr()
ast.Expression()
ast.ExtSlice()
ast.FloorDiv()
ast.For()
ast.FunctionDef()
ast.GeneratorExp()
ast.Global()
ast.Gt()
ast.GtE()
ast.If()
ast.IfExp()
ast.Import()
ast.ImportFrom()
ast.In()
ast.Index()
ast.Interactive()
ast.Invert()
ast.Is()
ast.IsNot()
ast.LShift()
ast.Lambda()
ast.List()
ast.ListComp()
ast.Load()
ast.Lt()
ast.LtE()
ast.Mod()
ast.Module()
ast.Mult()
ast.Name()
ast.Not()
ast.NotEq()
ast.NotIn()
ast.Num()
ast.Or()
ast.Param()
ast.Pass()
ast.Pow()
ast.Print()
ast.RShift()
ast.Raise()
ast.Repr()
ast.Return()
ast.Slice()
ast.Store()
ast.Str()
ast.Sub()
ast.Subscript()
ast.Suite()
ast.TryExcept()
ast.TryFinally()
ast.Tuple()
ast.UAdd()
ast.USub()
ast.UnaryOp()
ast.While()
ast.With()
ast.Yield()
ast.alias()
ast.arguments()
#ast.boolop()
#ast.cmpop()
ast.comprehension()
#ast.excepthandler()
#ast.expr()
#ast.expr_context()
ast.keyword()
def makeRegressor(model: Xgboost,
useSigmoids: bool = False,
useNumPy: bool = False) -> ast.Module:
"""Converts a regression XGBoost decision tree model into a python module of if-else trees"""
#code:typing.List[ast.stmt]=[]
code = []
numpyModuleName = "numpy"
if useSigmoids:
code.extend(
generateFermiSplitFunction(
sigmoidSplitFuncName,
tanhModuleName=(numpyModuleName if useNumPy else "math")))
if useNumPy:
code.append(
ast.ImportFrom(module=numpyModuleName,
names=[ast.alias(name=numpySumName, asname=None)],
level=0))
features = int(model.param.num_feature)
trees = int(model.gbm_.param.num_trees)
groups = int(model.gbm_.param.num_output_group)
if groups == 0:
groups = 1
treesInAGroup = trees // groups
groupFunctions = []
groupFunctionsNames = []
elts = []
for j in range(groups):
group = []
for i in range(treesInAGroup):
t = model.gbm_.trees[j * treesInAGroup + i]
name = "g" + str(j) + "_t" + str(i)
code.extend(treesToFuncs(t, name, useSigmoids))
group.append(ast.Name(id=name))
elts.append(ast.Tuple(elts=group))
groupFuncName = "g" + str(j)
groupFunctionsNames.append(ast.Name(groupFuncName))
groupFunctions.append(
makeGroup(groupFuncName, model.param.base_score,
useNumPy=useNumPy))
code.extend(groupFunctions)
code.append(
ast.Assign(targets=[treesAccumulator],
value=ast.Dict(keys=groupFunctionsNames, values=elts)))
code.append(
wrapWithFunc([
ast.Expr(value=astStr(s=genModelSummary(model))),
ast.Assert(test=ast.Compare(left=ast.Call(
func=lenFunc, args=[vector], keywords=[]),
ops=[ast.Eq()],
comparators=[astNum(n=features)]),
msg=ast.BinOp(left=ast.BinOp(
left=astStr(s="This model requires exactly " +
str(features) + " feature, but "),
op=ast.Add(),
right=ast.Call(func=strFunc,
args=[
ast.Call(func=lenFunc,
args=[vector],
keywords=[])
],
keywords=[])),
op=ast.Add(),
right=astStr(s=" were given"))),
ast.Assign(
targets=[res],
value=ast.ListComp(
elt=ast.Call(func=groupTemp, args=[vector], keywords=[]),
generators=[
ast.comprehension(
target=groupTemp, iter=treesAccumulator, ifs=[])
])),
ast.Return(value=res)
], regressorFunctionName))
return ast.Module(body=code)
def visitListComp(self, n, *args):
generators = self.reduce(n.generators, *args)
elt = self.dispatch(n.elt, *args)
return ast.ListComp(elt=elt, generators=generators)
def as_ast(dct):
"""See https://docs.python.org/2/library/ast.html"""
if dct['ast_type'] == "Module":
return ast.Module(dct["body"])
elif dct['ast_type'] == "Interactive":
return ast.Interactive(dct["body"])
elif dct['ast_type'] == "Expression":
return ast.Expression(dct["body"])
elif dct['ast_type'] == "Suite":
return ast.Suite(dct["body"])
elif dct['ast_type'] == "FunctionDef":
return ast.FunctionDef(dct["name"], dct["args"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "ClassDef":
return ast.ClassDef(dct["name"], dct["bases"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "Return":
return ast.Return(dct["value"])
elif dct['ast_type'] == "Delete":
return ast.Delete(dct["targets"])
elif dct['ast_type'] == "Assign":
return ast.Assign(dct["targets"], dct["value"])
elif dct['ast_type'] == "AugAssign":
return ast.AugAssign(dct["target"], dct["op"], dct["value"])
elif dct['ast_type'] == "Print":
return ast.Print(dct["dest"], dct["values"], dct["nl"])
elif dct['ast_type'] == "For":
return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "While":
return ast.While(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "If":
return ast.If(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "With":
return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"])
elif dct['ast_type'] == "Raise":
return ast.Raise(dct["type"], dct["inst"], dct["tback"])
elif dct['ast_type'] == "TryExcept":
return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"])
elif dct['ast_type'] == "TryFinally":
return ast.TryFinally(dct["body"], dct["finalbody"])
elif dct['ast_type'] == "Assert":
return ast.Assert(dct["test"], dct["msg"])
elif dct['ast_type'] == "Import":
return ast.Import(dct["names"])
elif dct['ast_type'] == "ImportFrom":
return ast.ImportFrom(dct["module"], dct["names"], dct["level"])
elif dct['ast_type'] == "Exec":
return ast.Exec(dct["body"], dct["globals"], dct["locals"])
elif dct['ast_type'] == "Global":
return ast.Global(dct["names"])
elif dct['ast_type'] == "Expr":
return ast.Expr(dct["value"])
elif dct['ast_type'] == "Pass":
return ast.Pass()
elif dct['ast_type'] == "Break":
return ast.Break()
elif dct['ast_type'] == "Continue":
return ast.Continue()
elif dct['ast_type'] == "BoolOp":
return ast.BoolOp(dct["op"], dct["values"])
elif dct['ast_type'] == "BinOp":
return ast.BinOp(dct["left"], dct["op"], dct["right"])
elif dct['ast_type'] == "UnaryOp":
return ast.UnaryOp(dct["op"], dct["operand"])
elif dct['ast_type'] == "Lambda":
return ast.Lambda(dct["args"], dct["body"])
elif dct['ast_type'] == "IfExp":
return ast.IfExp(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "Dict":
return ast.Dict(dct["keys"], dct["values"])
elif dct['ast_type'] == "Set":
return ast.Set(dct["elts"])
elif dct['ast_type'] == "ListComp":
return ast.ListComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "SetComp":
return ast.SetComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "DictComp":
return ast.DictComp(dct["key"], dct["value"], dct["generators"])
elif dct['ast_type'] == "GeneratorExp":
return ast.GeneratorExp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "Yield":
return ast.Yield(dct["value"])
elif dct['ast_type'] == "Compare":
return ast.Compare(dct["left"], dct["ops"], dct["comparators"])
elif dct['ast_type'] == "Call":
return ast.Call(dct["func"], dct["args"], dct["keywords"],
dct["starargs"], dct["kwargs"])
elif dct['ast_type'] == "Repr":
return ast.Repr(dct["value"])
elif dct['ast_type'] == "Num":
return ast.Num(dct["n"])
elif dct['ast_type'] == "Str":
# Converting to ASCII
return ast.Str(dct["s"].encode('ascii', 'ignore'))
elif dct['ast_type'] == "Attribute":
return ast.Attribute(dct["value"], dct["attr"], dct["ctx"])
elif dct['ast_type'] == "Subscript":
return ast.Subscript(dct["value"], dct["slice"], dct["ctx"])
elif dct['ast_type'] == "Name":
return ast.Name(dct["id"], dct["ctx"])
elif dct['ast_type'] == "List":
return ast.List(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Tuple":
return ast.Tuple(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Load":
return ast.Load()
elif dct['ast_type'] == "Store":
return ast.Store()
elif dct['ast_type'] == "Del":
return ast.Del()
elif dct['ast_type'] == "AugLoad":
return ast.AugLoad()
elif dct['ast_type'] == "AugStore":
return ast.AugStore()
elif dct['ast_type'] == "Param":
return ast.Param()
elif dct['ast_type'] == "Ellipsis":
return ast.Ellipsis()
elif dct['ast_type'] == "Slice":
return ast.Slice(dct["lower"], dct["upper"], dct["step"])
elif dct['ast_type'] == "ExtSlice":
return ast.ExtSlice(dct["dims"])
elif dct['ast_type'] == "Index":
return ast.Index(dct["value"])
elif dct['ast_type'] == "And":
return ast.And()
elif dct['ast_type'] == "Or":
return ast.Or()
elif dct['ast_type'] == "Add":
return ast.Add()
elif dct['ast_type'] == "Sub":
return ast.Sub()
elif dct['ast_type'] == "Mult":
return ast.Mult()
elif dct['ast_type'] == "Div":
return ast.Div()
elif dct['ast_type'] == "Mod":
return ast.Mod()
elif dct['ast_type'] == "Pow":
return ast.Pow()
elif dct['ast_type'] == "LShift":
return ast.LShift()
elif dct['ast_type'] == "RShift":
return ast.RShift()
elif dct['ast_type'] == "BitOr":
return ast.BitOr()
elif dct['ast_type'] == "BitXor":
return ast.BitXor()
elif dct['ast_type'] == "BitAnd":
return ast.BitAnd()
elif dct['ast_type'] == "FloorDiv":
return ast.FloorDiv()
elif dct['ast_type'] == "Invert":
return ast.Invert()
elif dct['ast_type'] == "Not":
return ast.Not()
elif dct['ast_type'] == "UAdd":
return ast.UAdd()
elif dct['ast_type'] == "USub":
return ast.USub()
elif dct['ast_type'] == "Eq":
return ast.Eq()
elif dct['ast_type'] == "NotEq":
return ast.NotEq()
elif dct['ast_type'] == "Lt":
return ast.Lt()
elif dct['ast_type'] == "LtE":
return ast.LtE()
elif dct['ast_type'] == "Gt":
return ast.Gt()
elif dct['ast_type'] == "GtE":
return ast.GtE()
elif dct['ast_type'] == "Is":
return ast.Is()
elif dct['ast_type'] == "IsNot":
return ast.IsNot()
elif dct['ast_type'] == "In":
return ast.In()
elif dct['ast_type'] == "NotIn":
return ast.NotIn()
elif dct['ast_type'] == "comprehension":
return ast.comprehension(dct["target"], dct["iter"], dct["ifs"])
elif dct['ast_type'] == "ExceptHandler":
return ast.ExceptHandler(dct["type"], dct["name"], dct["body"])
elif dct['ast_type'] == "arguments":
return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"],
dct["defaults"])
elif dct['ast_type'] == "keyword":
return ast.keyword(dct["arg"], dct["value"])
elif dct['ast_type'] == "alias":
return ast.alias(dct["name"], dct["asname"])
else:
return dct
def visit_DictComp(self, node):
node = self.generic_visit(node)
elt = mk_tuple([node.key, node.value])
listexpr = ast.ListComp(elt=elt, generators=node.generators)
return mk_call('dict', [listexpr])
