def visit_Subscript(self, node):
valid_slice = False
s = node.slice
# if the slice is a negative number, then convert it into a simpler form
# so that it can be subsequently parsed the same way as positive numbers
if hasattr(s, "value") and isinstance(s.value, ast.UnaryOp):
if isinstance(s.value.op, ast.USub) and isinstance(
s.value.operand, ast.Num):
n = -s.value.operand.n
if n != -1:
raise Exception(
f"Negative index `{n}` not supported. "
"Only -1 supported, due to ak.pad_none() padding the end of an array."
)
s = ast.Index(value=ast.Num(n))
for attr in ["value", "upper", "lower", "step"]:
if isinstance(getattr(s, attr, None), (ast.Constant, ast.Num)):
valid_slice = True
if valid_slice:
if hasattr(s, "value"):
upper = s.value.n
dimslice = ast.Constant(upper)
self.nreducers += 1
elif hasattr(s, "upper"):
upper = s.upper.n
dimslice = s
else:
raise Exception(f"Slice node not supported: {s}")
absupper = abs(upper)
if upper >= 0:
clip = True
absupper = ast.Constant(absupper + 1)
else:
clip = False
absupper = ast.Constant(absupper)
value = ast.Call(
func=ast.Name("ak.pad_none"),
args=[node.value, absupper],
keywords=[ast.keyword("clip", ast.Constant(clip))],
)
node = ast.Subscript(
value=value,
slice=ast.ExtSlice(dims=[
ast.Slice(lower=None, upper=None, step=None),
dimslice,
]),
ctx=ast.Load(),
)
# else:
# # for when an index array is used as a slice
# # currently will only work if that array has a None
# # (https://github.com/scikit-hep/awkward-1.0/issues/708)
# new_slice = ast.Call(
# func=ast.Name("ak.singletons"), args=[node.slice], keywords=[],
# )
# node = ast.Subscript(value=node.value, slice=new_slice, ctx=ast.Load())
self.generic_visit(node)
return node
def visit_Subscript(self, node):
def adjust_slice(s):
if isinstance(s, ast.Slice):
return s
else:
return ast.Index(s)
if isinstance(node.slice, gast.Tuple):
if any(isinstance(elt, gast.slice) for elt in node.slice.elts):
new_slice = ast.ExtSlice([
adjust_slice(x) for x in self._visit(node.slice.elts)
])
else:
value = ast.Tuple(self._visit(node.slice.elts), ast.Load())
ast.copy_location(value, node.slice)
new_slice = ast.Index(value)
else:
new_slice = adjust_slice(self._visit(node.slice))
ast.copy_location(new_slice, node.slice)
new_node = ast.Subscript(
self._visit(node.value),
new_slice,
self._visit(node.ctx),
)
ast.copy_location(new_node, node)
return new_node
def test_Subscript(self):
sub = ast.Subscript(ast.Name('X', ast.Load()), [], ast.Load())
# Index
slice1 = ast.Index(ast.Num(42))
sub.slice = slice1
self.verify(sub, 'X[42]')
# Slice
slice2 = ast.Slice(None, None, ast.Num(2))
sub.slice = slice2
self.verify(sub, 'X[::2]')
# ExtSlice
sub.slice = ast.ExtSlice([slice1, slice2])
self.verify(sub, 'X[42,::2]')
# Issue #20
expect = ast.Expr(value=ast.Subscript(
value=ast.BinOp(left=ast.Name(id='p', ctx=ast.Load()),
op=ast.Add(),
right=ast.List(elts=[
ast.Num(n=1),
ast.Num(n=2),
ast.Num(n=3),
ast.Num(n=4)
],
ctx=ast.Load())),
slice=ast.Slice(lower=None, upper=ast.Num(n=4), step=None),
ctx=ast.Load()))
self.verify(expect, '(p+[1,2,3,4])[:4]')
def test_subscript(self):
sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)),
ast.Load())
self.expr(sub, "must have Load context")
x = ast.Name("x", ast.Load())
sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())),
ast.Load())
self.expr(sub, "must have Load context")
s = ast.Name("x", ast.Store())
for args in (s, None, None), (None, s, None), (None, None, s):
sl = ast.Slice(*args)
self.expr(ast.Subscript(x, sl, ast.Load()),
"must have Load context")
sl = ast.ExtSlice([])
self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice")
sl = ast.ExtSlice([ast.Index(s)])
self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context")
def visit_ExtSlice(self, ext_sl: ast.ExtSlice) -> VisitSliceReturnT:
dims = []
actions = []
for dim in ext_sl.dims:
dim_flattened, dim_actions = self.visit_slice(dim)
dims.append(dim_flattened)
actions.extend(dim_actions)
return ast.ExtSlice(dims=dims), actions
def range_to_type(_range: dict) -> ast.expr:
_id = _range["@id"]
if _id == "linkedql:PathStep":
return ast.Constant("Path")
if _id == "linkedql:PropertyPath":
return ast.Subscript(
value=ast.Attribute(value=ast.Name(id="typing"), attr="Union"),
slice=ast.Index(value=ast.ExtSlice([
ast.Name(id="str"),
ast.Subscript(
value=ast.Attribute(value=ast.Name(id="typing"),
attr="List"),
slice=ast.Index(ast.Name("str")),
),
])),
)
if _id == "rdfg:Graph":
return ast.Name("GraphPattern")
if _id == "rdf:JSON":
return ast.Subscript(
value=ast.Attribute(value=ast.Name(id="typing"), attr="Dict"),
slice=ast.Index(
value=ast.ExtSlice([ast.Name(
id="str"), ast.Name(id="str")])),
)
if _id == "xsd:string":
return ast.Name(id="str")
if _id == "xsd:int":
return ast.Name(id="int")
if _id == "xsd:float":
return ast.Name(id="float")
if _id == "xsd:boolean":
return ast.Name(id="bool")
if _id == "linkedql:Operator":
return ast.Name(id="Operator")
if _id == "rdfs:Resource":
return ast.Attribute(value=ast.Attribute(value=ast.Name(id="rdflib"),
attr="term"),
attr="Node")
if _id == "owl:Thing":
return ast.Attribute(
value=ast.Attribute(value=ast.Name(id="rdflib"), attr="term"),
attr="Identifier",
)
raise Exception(f"Unexpected range: {_range}")
def _annotate_tuple(schema, resolve, names=()):
preamble = []
tuple_ = _typing("Tuple")
elements = ast.ExtSlice(dims=[])
for subschema in schema["items"]:
annotated, extra = _annotate_schema(subschema, resolve, names)
preamble.extend(extra)
elements.dims.append(annotated)
subscript = ast.Subscript(value=tuple_, slice=elements)
return subscript, preamble
def make_unconcat_slice(axis, lower, upper):
dims = []
for i in range(axis):
dims.append(ast.Slice(lower=None, upper=None, step=None))
dims.append(ast.Slice(lower=lower, upper=upper, step=None))
dims.append(ast.Ellipsis())
ext_slice = ast.ExtSlice(dims=dims)
return ext_slice
def _generate_extended_slice(max_depth=None):
choices = [
_generate_index_slice,
_generate_simple_slice,
]
dims_len = random.randrange(2, 4)
dims = [
sl(max_depth=max_depth - 1)
for sl in random.choices(choices, k=dims_len)
]
return ast.ExtSlice(dims)
def visit_Name(self, node: ast.Name) -> Union[ast.Name, ast.Subscript]:
if node.id not in self.__vars_mapper or node.id in self.__functions:
return node
else:
n = self.__vars_mapper[node.id]
return ast.Subscript(
value=ast.Name(id=self.__name, ctx=ast.Load()),
slice=ast.ExtSlice(
dims=[ast.Slice(lower=None, upper=None, step=None),
ast.Index(value=ast.Num(n=n))]),
ctx=node.ctx
)
def p_cellarrayref(self, p):
"""
expr : expr LBRACE expr_list RBRACE
| expr LBRACE RBRACE
"""
if len(p) == 4:
p[0] = self._new_subscript(p, p[1], ast.Slice(None, None, None))
elif len(p[3]) == 1:
p[0] = self._new_subscript(p, p[1], ast.Index(p[3][0]))
else:
p[0] = self._new_subscript(p, p[1], ast.ExtSlice([
ast.Index(i) if not isinstance(i, ast.Slice) else i for i in p[3]
]))
def _subscripts(self, node: ET.Element, postprocess: bool = True):
subscripts = self.transform_all_subnodes(node, ignored={
'section-subscript-list__begin', 'section-subscript-list'})
assert len(subscripts) == int(node.attrib['count'])
if not postprocess:
return subscripts
if any(isinstance(_, ast.Slice) for _ in subscripts):
if len(subscripts) == 1:
return subscripts[0]
return ast.ExtSlice(dims=[
(_ if isinstance(_, (ast.Index, ast.Slice))
else ast.Index(value=_)) for _ in subscripts])
assert all(not isinstance(_, (ast.Index, ast.Slice, ast.ExtSlice))
for _ in subscripts), subscripts
if len(subscripts) == 1:
return ast.Index(value=subscripts[0])
return ast.Index(value=ast.Tuple(elts=subscripts, ctx=ast.Load()))
def visit_Index(self, node):
new_value = self._visit(node.value)
if isinstance(new_value, ast.Ellipsis):
new_node = new_value
elif isinstance(new_value, ast.Tuple):
if any(isinstance(elt, ast.Ellipsis) for elt in new_value.elts):
new_elts = [
elt if isinstance(elt, (ast.Ellipsis,
ast.Slice)) else ast.Index(elt)
for elt in new_value.elts
]
new_node = ast.ExtSlice(new_elts)
else:
new_node = ast.Index(new_value)
else:
new_node = ast.Index(new_value)
ast.copy_location(new_node, node)
return new_node
def visit_Subscript(self, node):
def adjust_slice(s):
if isinstance(s, (ast.Slice, ast.Ellipsis)):
return s
else:
return ast.Index(s)
if isinstance(node.slice, gast.Tuple):
new_slice = ast.ExtSlice(
[adjust_slice(self._visit(elt)) for elt in node.slice.elts])
else:
new_slice = adjust_slice(self._visit(node.slice))
ast.copy_location(new_slice, node.slice)
new_node = ast.Subscript(
self._visit(node.value),
new_slice,
self._visit(node.ctx),
)
ast.copy_location(new_node, node)
new_node.end_lineno = new_node.end_col_offset = None
return new_node
def as_ast(dct):
"""See https://docs.python.org/2/library/ast.html"""
if dct['ast_type'] == "Module":
return ast.Module(dct["body"])
elif dct['ast_type'] == "Interactive":
return ast.Interactive(dct["body"])
elif dct['ast_type'] == "Expression":
return ast.Expression(dct["body"])
elif dct['ast_type'] == "Suite":
return ast.Suite(dct["body"])
elif dct['ast_type'] == "FunctionDef":
return ast.FunctionDef(dct["name"], dct["args"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "ClassDef":
return ast.ClassDef(dct["name"], dct["bases"], dct["body"],
dct["decorator_list"])
elif dct['ast_type'] == "Return":
return ast.Return(dct["value"])
elif dct['ast_type'] == "Delete":
return ast.Delete(dct["targets"])
elif dct['ast_type'] == "Assign":
return ast.Assign(dct["targets"], dct["value"])
elif dct['ast_type'] == "AugAssign":
return ast.AugAssign(dct["target"], dct["op"], dct["value"])
elif dct['ast_type'] == "Print":
return ast.Print(dct["dest"], dct["values"], dct["nl"])
elif dct['ast_type'] == "For":
return ast.For(dct["target"], dct["iter"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "While":
return ast.While(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "If":
return ast.If(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "With":
return ast.With(dct["context_expr"], dct["optional_vars"], dct["body"])
elif dct['ast_type'] == "Raise":
return ast.Raise(dct["type"], dct["inst"], dct["tback"])
elif dct['ast_type'] == "TryExcept":
return ast.TryExcept(dct["body"], dct["handlers"], dct["orelse"])
elif dct['ast_type'] == "TryFinally":
return ast.TryFinally(dct["body"], dct["finalbody"])
elif dct['ast_type'] == "Assert":
return ast.Assert(dct["test"], dct["msg"])
elif dct['ast_type'] == "Import":
return ast.Import(dct["names"])
elif dct['ast_type'] == "ImportFrom":
return ast.ImportFrom(dct["module"], dct["names"], dct["level"])
elif dct['ast_type'] == "Exec":
return ast.Exec(dct["body"], dct["globals"], dct["locals"])
elif dct['ast_type'] == "Global":
return ast.Global(dct["names"])
elif dct['ast_type'] == "Expr":
return ast.Expr(dct["value"])
elif dct['ast_type'] == "Pass":
return ast.Pass()
elif dct['ast_type'] == "Break":
return ast.Break()
elif dct['ast_type'] == "Continue":
return ast.Continue()
elif dct['ast_type'] == "BoolOp":
return ast.BoolOp(dct["op"], dct["values"])
elif dct['ast_type'] == "BinOp":
return ast.BinOp(dct["left"], dct["op"], dct["right"])
elif dct['ast_type'] == "UnaryOp":
return ast.UnaryOp(dct["op"], dct["operand"])
elif dct['ast_type'] == "Lambda":
return ast.Lambda(dct["args"], dct["body"])
elif dct['ast_type'] == "IfExp":
return ast.IfExp(dct["test"], dct["body"], dct["orelse"])
elif dct['ast_type'] == "Dict":
return ast.Dict(dct["keys"], dct["values"])
elif dct['ast_type'] == "Set":
return ast.Set(dct["elts"])
elif dct['ast_type'] == "ListComp":
return ast.ListComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "SetComp":
return ast.SetComp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "DictComp":
return ast.DictComp(dct["key"], dct["value"], dct["generators"])
elif dct['ast_type'] == "GeneratorExp":
return ast.GeneratorExp(dct["elt"], dct["generators"])
elif dct['ast_type'] == "Yield":
return ast.Yield(dct["value"])
elif dct['ast_type'] == "Compare":
return ast.Compare(dct["left"], dct["ops"], dct["comparators"])
elif dct['ast_type'] == "Call":
return ast.Call(dct["func"], dct["args"], dct["keywords"],
dct["starargs"], dct["kwargs"])
elif dct['ast_type'] == "Repr":
return ast.Repr(dct["value"])
elif dct['ast_type'] == "Num":
return ast.Num(dct["n"])
elif dct['ast_type'] == "Str":
# Converting to ASCII
return ast.Str(dct["s"].encode('ascii', 'ignore'))
elif dct['ast_type'] == "Attribute":
return ast.Attribute(dct["value"], dct["attr"], dct["ctx"])
elif dct['ast_type'] == "Subscript":
return ast.Subscript(dct["value"], dct["slice"], dct["ctx"])
elif dct['ast_type'] == "Name":
return ast.Name(dct["id"], dct["ctx"])
elif dct['ast_type'] == "List":
return ast.List(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Tuple":
return ast.Tuple(dct["elts"], dct["ctx"])
elif dct['ast_type'] == "Load":
return ast.Load()
elif dct['ast_type'] == "Store":
return ast.Store()
elif dct['ast_type'] == "Del":
return ast.Del()
elif dct['ast_type'] == "AugLoad":
return ast.AugLoad()
elif dct['ast_type'] == "AugStore":
return ast.AugStore()
elif dct['ast_type'] == "Param":
return ast.Param()
elif dct['ast_type'] == "Ellipsis":
return ast.Ellipsis()
elif dct['ast_type'] == "Slice":
return ast.Slice(dct["lower"], dct["upper"], dct["step"])
elif dct['ast_type'] == "ExtSlice":
return ast.ExtSlice(dct["dims"])
elif dct['ast_type'] == "Index":
return ast.Index(dct["value"])
elif dct['ast_type'] == "And":
return ast.And()
elif dct['ast_type'] == "Or":
return ast.Or()
elif dct['ast_type'] == "Add":
return ast.Add()
elif dct['ast_type'] == "Sub":
return ast.Sub()
elif dct['ast_type'] == "Mult":
return ast.Mult()
elif dct['ast_type'] == "Div":
return ast.Div()
elif dct['ast_type'] == "Mod":
return ast.Mod()
elif dct['ast_type'] == "Pow":
return ast.Pow()
elif dct['ast_type'] == "LShift":
return ast.LShift()
elif dct['ast_type'] == "RShift":
return ast.RShift()
elif dct['ast_type'] == "BitOr":
return ast.BitOr()
elif dct['ast_type'] == "BitXor":
return ast.BitXor()
elif dct['ast_type'] == "BitAnd":
return ast.BitAnd()
elif dct['ast_type'] == "FloorDiv":
return ast.FloorDiv()
elif dct['ast_type'] == "Invert":
return ast.Invert()
elif dct['ast_type'] == "Not":
return ast.Not()
elif dct['ast_type'] == "UAdd":
return ast.UAdd()
elif dct['ast_type'] == "USub":
return ast.USub()
elif dct['ast_type'] == "Eq":
return ast.Eq()
elif dct['ast_type'] == "NotEq":
return ast.NotEq()
elif dct['ast_type'] == "Lt":
return ast.Lt()
elif dct['ast_type'] == "LtE":
return ast.LtE()
elif dct['ast_type'] == "Gt":
return ast.Gt()
elif dct['ast_type'] == "GtE":
return ast.GtE()
elif dct['ast_type'] == "Is":
return ast.Is()
elif dct['ast_type'] == "IsNot":
return ast.IsNot()
elif dct['ast_type'] == "In":
return ast.In()
elif dct['ast_type'] == "NotIn":
return ast.NotIn()
elif dct['ast_type'] == "comprehension":
return ast.comprehension(dct["target"], dct["iter"], dct["ifs"])
elif dct['ast_type'] == "ExceptHandler":
return ast.ExceptHandler(dct["type"], dct["name"], dct["body"])
elif dct['ast_type'] == "arguments":
return ast.arguments(dct["args"], dct["vararg"], dct["kwarg"],
dct["defaults"])
elif dct['ast_type'] == "keyword":
return ast.keyword(dct["arg"], dct["value"])
elif dct['ast_type'] == "alias":
return ast.alias(dct["name"], dct["asname"])
else:
return dct
def visitExtSlice(self, n, *args):
return ast.ExtSlice(dims=self.reduce(n.dims, *args))
def visit_ExtSlice(self, node: ExtSlice, *args, **kwargs) -> C.ExtSlice:
dims = self.visit(node.dims, *args, **kwargs)
return C.ExtSlice(dims=dims, )
def test_ExtSlice(self):
slice1 = ast.Index(ast.Num(42))
slice2 = ast.Slice(None, None, ast.Num(6))
self.verify(ast.ExtSlice([slice1, slice2]), '42,::6')
def _annotate_schema(schema, resolve, names=()):
if "const" in schema:
return _annotate_literal(schema["const"]), []
if "enum" in schema:
for enumerated in schema["enum"]:
return _annotate_literal(enumerated), []
if "anyOf" in schema: # or oneOf?
preamble = []
union = _typing("Union")
tuple_ = ast.ExtSlice(dims=[])
for subschema in schema["anyOf"]:
annotated, extra = _annotate_schema(subschema, resolve, names)
preamble.extend(extra)
# FIXME: Optional[T, U] is illegal
# if _is_literal_none(annotated):
# union = _typing("Optional")
# continue
tuple_.dims.append(annotated)
subscript = ast.Subscript(value=union, slice=tuple_)
return subscript, preamble
if "$ref" in schema:
reference = schema["$ref"]
resolved = resolve(reference)
return ast.Constant(value=resolved, kind=None), []
if "type" in schema:
if schema["type"] == "null":
return _annotate_literal(None), []
if schema["type"] == "integer":
return ast.Name(id="int"), []
if schema["type"] == "number":
return ast.Subscript(
value=ast.Name(id="Union"),
slice=ast.ExtSlice(dims=[
ast.Name(id="int"),
ast.Name(id="float"),
]),
), []
if schema["type"] == "boolean":
return ast.Name(id="bool"), []
if schema["type"] == "string":
return ast.Name(id="str"), []
if schema["type"] == "array":
if isinstance(schema["items"], list):
return _annotate_tuple(schema, resolve, names)
else:
sequence = _typing("List")
item, preamble = \
_annotate_schema(schema["items"], resolve, names)
subscript = ast.Subscript(value=sequence, slice=item)
return subscript, preamble
if schema["type"] == "object":
return _annotate_object(schema, resolve, names)
# return _typing("Any")
# TODO: optionally default to Any
raise Exception(schema)
def ExtSlice(draw, expression) -> ast.ExtSlice:
slice = draw(Slice(expression))
return ast.ExtSlice([slice] + draw(
lists(Index(expression) | Slice(expression), min_size=1, max_size=3)))
def p_expr2(self, p):
"""
expr2 : expr AND expr
| expr ANDAND expr
| expr BACKSLASH expr
| expr COLON expr
| expr DIV expr
| expr DOT expr
| expr DOTDIV expr
| expr DOTDIVEQ expr
| expr DOTEXP expr
| expr DOTMUL expr
| expr DOTMULEQ expr
| expr EQEQ expr
| expr EXP expr
| expr EXPEQ expr
| expr GE expr
| expr GT expr
| expr LE expr
| expr LT expr
| expr MINUS expr
| expr MUL expr
| expr NE expr
| expr OR expr
| expr OROR expr
| expr PLUS expr
| expr EQ expr
| expr MULEQ expr
| expr DIVEQ expr
| expr MINUSEQ expr
| expr PLUSEQ expr
| expr OREQ expr
| expr ANDEQ expr
"""
if p[2] == "=":
if isinstance(p[1], ast.Call):
subs = self._new_subscript(p, p[1].func, ast.ExtSlice(list(map(ast.Index, p[1].args))))
p[1] = [subs]
elif not isinstance(p[1], list):
p[1] = [p[1]]
for target in p[1]:
target.ctx = ast.Store()
p[0] = self._new_node(p, ast.Assign, p[1], p[3])
elif p[2] == ':':
p[0] = self._new_call(p, self._new_name(p, 'range'), p[1],
self._new_node(p, ast.BinOp, p[3], ast.Add(),
self._new_node(p, ast.Num, 1)))
elif p[2] == '.':
if isinstance(p[3], ast.Subscript):
p[0] = self._new_subscript(p, p[1], ast.Index(p[3]))
else:
p[0] = self._new_node(p, ast.Attribute, p[1], p[3], ast.Load())
elif p[2] in Parser.COMPARE_OPS:
p[0] = self._new_node(p, ast.Compare, p[1], [Parser.COMPARE_OPS[p[2]]()], [p[3]])
elif p[2] in Parser.LOGIC_OPS:
p[0] = self._new_node(p, ast.BoolOp, Parser.LOGIC_OPS[p[2]](), [p[1], p[3]])
elif p[2] in Parser.BINARY_OPS:
p[0] = self._new_node(p, ast.BinOp, p[1], Parser.BINARY_OPS[p[2]](), p[3])
elif p[2] in Parser.ASSIGN_OPS:
p[1].ctx = ast.Store()
p[0] = self._new_node(p, ast.AugAssign, p[1], Parser.ASSIGN_OPS[p[2]](), p[3])
else:
raise NotImplementedError(p[2])
def test_empty_init(self):
# Jython 2.5.0 did not allow empty constructors for many ast node types
# but CPython ast nodes do allow this. For the moment, I don't see a
# reason to allow construction of the super types (like ast.AST and
# ast.stmt) as well as the op types that are implemented as enums in
# Jython (like boolop), but I've left them in but commented out for
# now. We may need them in the future since CPython allows this, but
# it may fall under implementation detail.
#ast.AST()
ast.Add()
ast.And()
ast.Assert()
ast.Assign()
ast.Attribute()
ast.AugAssign()
ast.AugLoad()
ast.AugStore()
ast.BinOp()
ast.BitAnd()
ast.BitOr()
ast.BitXor()
ast.BoolOp()
ast.Break()
ast.Call()
ast.ClassDef()
ast.Compare()
ast.Continue()
ast.Del()
ast.Delete()
ast.Dict()
ast.Div()
ast.Ellipsis()
ast.Eq()
ast.Exec()
ast.Expr()
ast.Expression()
ast.ExtSlice()
ast.FloorDiv()
ast.For()
ast.FunctionDef()
ast.GeneratorExp()
ast.Global()
ast.Gt()
ast.GtE()
ast.If()
ast.IfExp()
ast.Import()
ast.ImportFrom()
ast.In()
ast.Index()
ast.Interactive()
ast.Invert()
ast.Is()
ast.IsNot()
ast.LShift()
ast.Lambda()
ast.List()
ast.ListComp()
ast.Load()
ast.Lt()
ast.LtE()
ast.Mod()
ast.Module()
ast.Mult()
ast.Name()
ast.Not()
ast.NotEq()
ast.NotIn()
ast.Num()
ast.Or()
ast.Param()
ast.Pass()
ast.Pow()
ast.Print()
ast.RShift()
ast.Raise()
ast.Repr()
ast.Return()
ast.Slice()
ast.Store()
ast.Str()
ast.Sub()
ast.Subscript()
ast.Suite()
ast.TryExcept()
ast.TryFinally()
ast.Tuple()
ast.UAdd()
ast.USub()
ast.UnaryOp()
ast.While()
ast.With()
ast.Yield()
ast.alias()
ast.arguments()
#ast.boolop()
#ast.cmpop()
ast.comprehension()
#ast.excepthandler()
#ast.expr()
#ast.expr_context()
ast.keyword()
def unellipsify(node, slices, subscript_node):
"""
Given an array node `node`, process all AST slices and create the
final type:
- process newaxes (None or numpy.newaxis)
- replace Ellipsis with a bunch of ast.Slice objects
- process integer indices
- append any missing slices in trailing dimensions
"""
type = node.variable.type
if not type.is_array:
assert type.is_object
return object_, node
if (len(slices) == 1 and nodes.is_constant_index(slices[0]) and
slices[0].value.pyval is Ellipsis):
# A[...]
return type, node
result = []
seen_ellipsis = False
# Filter out newaxes
newaxes = [newaxis for newaxis in slices if nodes.is_newaxis(newaxis)]
n_indices = len(slices) - len(newaxes)
full_slice = ast.Slice(lower=None, upper=None, step=None)
full_slice.variable = Variable(typesystem.slice_)
ast.copy_location(full_slice, slices[0])
# process ellipses and count integer indices
indices_seen = 0
for slice_node in slices[::-1]:
slice_type = slice_node.variable.type
if slice_type.is_ellipsis:
if seen_ellipsis:
result.append(full_slice)
else:
nslices = type.ndim - n_indices + 1
result.extend([full_slice] * nslices)
seen_ellipsis = True
elif (slice_type.is_slice or slice_type.is_int or
nodes.is_newaxis(slice_node)):
indices_seen += slice_type.is_int
result.append(slice_node)
else:
# TODO: Coerce all object operands to integer indices?
# TODO: (This will break indexing with the Ellipsis object or
# TODO: with slice objects that we couldn't infer)
return object_, nodes.CoercionNode(node, object_)
# Reverse our reversed processed list of slices
result.reverse()
# append any missing slices (e.g. a2d[:]
result_length = len(result) - len(newaxes)
if result_length < type.ndim:
nslices = type.ndim - result_length
result.extend([full_slice] * nslices)
subscript_node.slice = ast.ExtSlice(result)
ast.copy_location(subscript_node.slice, slices[0])
# create the final array type and set it in value.variable
result_dtype = node.variable.type.dtype
result_ndim = node.variable.type.ndim + len(newaxes) - indices_seen
if result_ndim > 0:
result_type = result_dtype[(slice(None),) * result_ndim]
elif result_ndim == 0:
result_type = result_dtype
else:
result_type = object_
return result_type, node
def test_operators(self):
boolop0 = ast.BoolOp()
boolop1 = ast.BoolOp(ast.And(),
[ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())])
boolop2 = ast.BoolOp(ast.And(),
[ ast.Name('True', ast.Load()), ast.Name('False', ast.Load()), ast.Name('a',ast.Load())],
0, 0)
binop0 = ast.BinOp()
binop1 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3))
binop2 = ast.BinOp(ast.Str('xy'), ast.Mult(), ast.Num(3), 0, 0)
unaryop0 = ast.UnaryOp()
unaryop1 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()))
unaryop2 = ast.UnaryOp(ast.Not(), ast.Name('True',ast.Load()), 0, 0)
lambda0 = ast.Lambda()
lambda1 = ast.Lambda(ast.arguments([ast.Name('x', ast.Param())], None, None, []), ast.Name('x', ast.Load()))
ifexp0 = ast.IfExp()
ifexp1 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0))
ifexp2 = ast.IfExp(ast.Name('True',ast.Load()), ast.Num(1), ast.Num(0), 0, 0)
dict0 = ast.Dict()
dict1 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')])
dict2 = ast.Dict([ast.Num(1), ast.Num(2)], [ast.Str('a'), ast.Str('b')], 0, 0)
set0 = ast.Set()
set1 = ast.Set([ast.Num(1), ast.Num(2)])
set2 = ast.Set([ast.Num(1), ast.Num(2)], 0, 0)
lc0 = ast.ListComp()
lc1 = ast.ListComp( ast.Name('x',ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
lc2 = ast.ListComp( ast.Name('x',ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])], 0, 0)
setcomp0 = ast.SetComp()
setcomp1 = ast.SetComp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()), ast.Str('abracadabra'),
[ast.Compare(ast.Name('x', ast.Load()), [ast.NotIn()],
[ast.Str('abc')])])])
comprehension0 = ast.comprehension()
comprehension1 = ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])
# "{i : chr(65+i) for i in (1,2)}")
dictcomp0 = ast.DictComp()
dictcomp1 = ast.DictComp(ast.Name('i', ast.Load()),
ast.Call(ast.Name('chr', ast.Load()),
[ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
[], None, None),
[ast.comprehension(ast.Name('i', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])])
dictcomp2 = ast.DictComp(ast.Name('i', ast.Load()),
ast.Call(ast.Name('chr', ast.Load()),
[ast.BinOp(ast.Num(65), ast.Add(), ast.Name('i', ast.Load()))],
[], None, None),
[ast.comprehension(ast.Name('i', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(n=2)], ast.Load()), [])],0,0)
# (x for x in (1,2))
genexp0 = ast.GeneratorExp()
genexp1 = ast.GeneratorExp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])])
genexp2 = ast.GeneratorExp(ast.Name('x', ast.Load()),
[ast.comprehension(ast.Name('x', ast.Store()),
ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load()), [])],0,0)
# yield 2
yield0 = ast.Yield()
yield1 = ast.Yield(ast.Num(2))
yield2 = ast.Yield(ast.Num(2),0,0)
yield20 = ast.Yield(lineno=0, col_offset=0)
# a>0
compare0 = ast.Compare()
compare1 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)])
compare2 = ast.Compare(ast.Name('a', ast.Load()), [ast.Gt()], [ast.Num(0)],0,0)
# chr(65)
call0 = ast.Call()
call1 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None)
call2 = ast.Call(ast.Name('chr', ast.Load()), [ast.Num(65)], [], None, None, 0, 0)
call20 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [])
call21 = ast.Call(ast.Name('f', ast.Load()), [ast.Num(0)], [], lineno=0, col_offset=0)
# 0
num0 = ast.Num()
num1 = ast.Num(0)
num2 = ast.Num(0,0,0)
# "foo"
str0 = ast.Str()
str1 = ast.Str("foo")
str2 = ast.Str("foo",0,0)
# TODO: come back
repr0 = ast.Repr()
repr1 = ast.Repr(ast.Num(0))
repr2 = ast.Repr(ast.Num(0),0,0)
# foo.bar
attr0 = ast.Attribute()
attr1 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load())
attr2 = ast.Attribute(ast.Name('foo', ast.Load()), 'bar', ast.Load(), 0,0)
# a[1:2]
subscript0 = ast.Subscript()
subscript1 = ast.Subscript(ast.Name('a', ast.Load()), ast.Slice(ast.Num(1), ast.Num(2)), ast.Load())
subscript2 = ast.Subscript(ast.Name('a', ast.Load()), ast.ExtSlice([ast.Num(1), ast.Num(2)]), ast.Load(), 0, 0)
# name
name0 = ast.Name()
name1 = ast.Name("name", ast.Load())
name2 = ast.Name("name", ast.Load(),0,0)
# [1,2]
list0 = ast.List()
list1 = ast.List([ast.Num(1), ast.Num(2)], ast.Load())
list2 = ast.List([ast.Num(1), ast.Num(2)], ast.Load(),0,0)
# (1,2)
tuple0 = ast.Tuple()
tuple1 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load())
tuple2 = ast.Tuple([ast.Num(1), ast.Num(2)], ast.Load(), 0, 0)
def sequence_helper(self, root, sentence):
for i in range(len(root.children)):
self.sequence_helper(root.children[i], sentence)
for child in root.children[i].children:
assert (isinstance(child, LeafNode))
if isinstance(root.children[i], LoopNode):
loop_node = root.children[i]
loop_var = None
iter_obj = None
if not loop_node.body_ctx.computed:
loop_node.body_ctx.computed = True
for key, val in zip(loop_node.children_keys,
loop_node.children):
target = ast.Name(id=key, ctx=ast.Store())
value = self.choose_child_leaf_node(val)
init_assignement = ast.Assign(targets=[target],
value=value)
# checking forloop case
if is_forloop_initializer(init_assignement):
loop_var = get_forloop_var(init_assignement)
iter_obj = get_forloop_iter_obj(init_assignement)
else:
sentence.append(init_assignement)
iter_stmt = self.translate_ctx(loop_node.body_ctx.body)
if loop_var != None and iter_obj != None:
loop_node_stmt = loop_node.compute_forloop_stmt(
iter_stmt, loop_var, iter_obj)
else:
loop_node_stmt = loop_node.compute_whileloop_stmt(
iter_stmt)
sentence += loop_node_stmt
param_node = Param(loop_node.id, loop_node.output_var)
self.peg.replace_node(loop_node, param_node)
if isinstance(root.children[i], BranchNode):
branch_node = root.children[i]
if not branch_node.body_ctx.computed:
for key, val in zip(branch_node.children_keys,
branch_node.children):
target = ast.Name(id=key, ctx=ast.Store())
value = self.choose_child_leaf_node(val)
init_assignement = ast.Assign(targets=[target],
value=value)
sentence.append(init_assignement)
true_stmt = self.translate_ctx(
branch_node.body_ctx.true_branch)
false_stmt = self.translate_ctx(
branch_node.body_ctx.false_branch)
branch_node_stmt = branch_node.compute_stmt(
true_stmt, false_stmt)
branch_node.body_ctx.computed = True
for stmt in branch_node_stmt:
sentence.append(stmt)
param_node = Param(branch_node.id, branch_node.output_var)
self.peg.replace_node(branch_node, param_node)
if isinstance(root.children[i], BinOpNode):
operator = root.children[i]
fresh_var = self.create_fresh_var('op')
target = ast.Name(id=fresh_var, ctx=ast.Store())
args = [
self.choose_child_leaf_node(arg)
for arg in operator.children
]
op_exp = ast.BinOp(left=args[0], op=operator.op, right=args[1])
self.process_stmt(sentence, operator, fresh_var, op_exp)
param_node = Param(operator.id, fresh_var)
self.peg.replace_node(operator, param_node)
if isinstance(root.children[i], FunctionCall):
func_call = root.children[i]
fresh_var = self.create_fresh_var('func_call')
target = ast.Name(id=fresh_var, ctx=ast.Store())
name = self.choose_child_leaf_node(func_call.name())
args = [
self.choose_child_leaf_node(arg)
for arg in func_call.args()
]
func_call_exp = ast.Call(func=name, args=args, keywords=[])
self.process_stmt(sentence, func_call, fresh_var,
func_call_exp)
param_node = Param(func_call.id, fresh_var)
self.peg.replace_node(func_call, param_node)
if isinstance(root.children[i], IfExpNode):
if_e = root.children[i]
fresh_var = self.create_fresh_var('if_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
cond = self.choose_child_leaf_node(if_e.cond())
t = self.choose_child_leaf_node(if_e.t())
f = self.choose_child_leaf_node(if_e.f())
if_e_exp = ast.IfExp(test=cond, body=t, orelse=f)
self.process_stmt(sentence, if_e, fresh_var, if_e_exp)
param_node = Param(if_e.id, fresh_var)
self.peg.replace_node(if_e, param_node)
if isinstance(root.children[i], CompareNode):
compare = root.children[i]
fresh_var = self.create_fresh_var('comp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
head = self.choose_child_leaf_node(compare.head())
tail = [
self.choose_child_leaf_node(arg) for arg in compare.tail()
]
cmp_exp = ast.Compare(left=head,
ops=compare.ops,
comparators=tail)
self.process_stmt(sentence, compare, fresh_var, cmp_exp)
param_node = Param(compare.id, fresh_var)
self.peg.replace_node(compare, param_node)
if isinstance(root.children[i], BoolOpNode):
bool_op_node = root.children[i]
fresh_var = self.create_fresh_var('bool_op')
target = ast.Name(id=fresh_var, ctx=ast.Store())
values = [
self.choose_child_leaf_node(arg)
for arg in bool_op_node.children
]
bool_op_exp = ast.BoolOp(op=bool_op_node.op, values=values)
self.process_stmt(sentence, bool_op_node, fresh_var,
bool_op_exp)
param_node = Param(bool_op_node.id, fresh_var)
self.peg.replace_node(bool_op_node, param_node)
if isinstance(root.children[i], UnaryOpNode):
un_op_node = root.children[i]
fresh_var = self.create_fresh_var('unary_op_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
operand = self.choose_child_leaf_node(un_op_node.operand())
un_op_exp = ast.UnaryOp(op=un_op_node.op, operand=operand)
self.process_stmt(sentence, un_op_node, fresh_var, un_op_exp)
param_node = Param(un_op_node.id, fresh_var)
self.peg.replace_node(un_op_node, param_node)
if isinstance(root.children[i], ListNode):
list_node = root.children[i]
fresh_var = self.create_fresh_var('list_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in list_node.children
]
list_exp = ast.List(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, list_node, fresh_var, list_exp)
param_node = Param(list_node.id, fresh_var)
self.peg.replace_node(list_node, param_node)
if isinstance(root.children[i], TupleNode):
tuple_node = root.children[i]
fresh_var = self.create_fresh_var('tuple_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in tuple_node.children
]
tuple_exp = ast.Tuple(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, tuple_node, fresh_var, tuple_exp)
param_node = Param(tuple_node.id, fresh_var)
self.peg.replace_node(tuple_node, param_node)
if isinstance(root.children[i], SetNode):
set_node = root.children[i]
fresh_var = self.create_fresh_var('set_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elts = [
self.choose_child_leaf_node(elem)
for elem in set_node.children
]
set_exp = ast.Set(elts=elts, ctx=ast.Load())
self.process_stmt(sentence, set_node, fresh_var, set_exp)
param_node = Param(set_node.id, fresh_var)
self.peg.replace_node(set_node, param_node)
if isinstance(root.children[i], DictNode):
dict_node = root.children[i]
fresh_var = self.create_fresh_var('dict_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
keys = [
self.choose_child_leaf_node(k) for k in dict_node.keys()
]
values = [
self.choose_child_leaf_node(v) for v in dict_node.values()
]
dict_exp = ast.Dict(keys=keys, values=values, ctx=ast.Load())
self.process_stmt(sentence, dict_node, fresh_var, dict_exp)
param_node = Param(dict_node.id, fresh_var)
self.peg.replace_node(dict_node, param_node)
if isinstance(root.children[i], AttributeNode):
attr_node = root.children[i]
fresh_var = self.create_fresh_var('attr_exp')
value = self.choose_child_leaf_node(attr_node.value())
attr = attr_node.attr
attr_exp = ast.Attribute(value=value,
attr=attr,
ctx=ast.Load())
self.rhs_assignement_expressions[attr_node.id] = attr_exp
param_node = Param(attr_node.id, fresh_var)
self.peg.replace_node(attr_node, param_node)
if isinstance(root.children[i], SubscriptNode):
subs_node = root.children[i]
fresh_var = self.create_fresh_var('subs_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
value = self.choose_child_leaf_node(subs_node.value())
slice = self.choose_child_leaf_node(subs_node.slice())
subs_exp = ast.Subscript(value=value,
slice=slice,
ctx=ast.Load())
self.process_stmt(sentence, subs_node, fresh_var, subs_exp)
param_node = Param(subs_node.id, fresh_var)
self.peg.replace_node(subs_node, param_node)
if isinstance(root.children[i], IndexNode):
idx_node = root.children[i]
fresh_var = self.create_fresh_var('idx_exp')
value = self.choose_child_leaf_node(idx_node.value())
idx_exp = ast.Index(value=value)
# never append just index to sentence
self.rhs_assignement_expressions[idx_node.id] = idx_exp
param_node = Param(idx_node.id, fresh_var)
self.peg.replace_node(idx_node, param_node)
if isinstance(root.children[i], SliceNode):
slice_node = root.children[i]
fresh_var = self.create_fresh_var('slice_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
lower = self.choose_child_leaf_node(slice_node.lower())
upper = self.choose_child_leaf_node(slice_node.upper())
step = self.choose_child_leaf_node(slice_node.step())
slice_exp = ast.Slice(lower=lower, upper=upper, step=step)
self.rhs_assignement_expressions[slice_node.id] = slice_exp
param_node = Param(slice_node.id, fresh_var)
self.peg.replace_node(slice_node, param_node)
if isinstance(root.children[i], ExtSliceNode):
eslice_node = root.children[i]
fresh_var = self.create_fresh_var('eslice_exp')
dims = [
self.choose_child_leaf_node(d) for d in eslice_node.dims()
]
eslice_exp = ast.ExtSlice(dims=dims)
self.rhs_assignement_expressions[eslice_node.id] = eslice_exp
param_node = Param(eslice_node.id, fresh_var)
self.peg.replace_node(eslice_node, param_node)
if isinstance(root.children[i], NPArrayNode):
ndarr_node = root.children[i]
fresh_var = self.create_fresh_var('ndarr_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
lists = self.choose_child_leaf_node(ndarr_node.children[0])
ndarr_attr = ast.Attribute(value=ast.Name(id='np',
ctx=ast.Load()),
attr='array')
ndarr_exp = ast.Call(func=ndarr_attr,
args=[lists],
keywords=[])
self.process_stmt(sentence, ndarr_node, fresh_var, ndarr_exp)
param_node = Param(ndarr_node.id, fresh_var)
self.peg.replace_node(ndarr_node, param_node)
if isinstance(root.children[i], ComprehensionNode):
comp_node = root.children[i]
fresh_var = self.create_fresh_var('compr_exp')
t = self.choose_child_leaf_node(comp_node.target())
iter_obj = self.choose_child_leaf_node(comp_node.iter_obj())
ifs = [self.choose_child_leaf_node(i) for i in comp_node.ifs()]
comp_exp = ast.comprehension(target=t, iter=iter_obj, ifs=ifs)
self.rhs_assignement_expressions[comp_node.id] = comp_exp
param_node = Param(comp_node.id, fresh_var)
self.peg.replace_node(comp_node, param_node)
if isinstance(root.children[i], ListCompNode):
lcomp_node = root.children[i]
fresh_var = self.create_fresh_var('lcomp_exp')
target = ast.Name(id=fresh_var, ctx=ast.Store())
elt = self.choose_child_leaf_node(lcomp_node.element())
generators = [
self.choose_child_leaf_node(gen)
for gen in lcomp_node.generators()
]
lcomp_exp = ast.ListComp(elt=elt, generators=generators)
self.process_stmt(sentence, lcomp_node, fresh_var, lcomp_exp)
param_node = Param(lcomp_node.id, fresh_var)
self.peg.replace_node(lcomp_node, param_node)
