def visit_function_dot(self, node, block):
"""
Visit function call with more than one path anme and lower
them into mutations.
"""
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'call_expression':
call_expr = node
if len(call_expr.path.children) > 1:
path_fragments = call_expr.path.children
if len(path_fragments) == 2:
# don't rewrite s.length.max() yet
call_expr.children = [
Tree('primary_expression', [
Tree('entity',
[Tree('path', [call_expr.path.children[0]])])
]),
Tree('mutation_fragment', [
path_fragments[-1].children[0],
*call_expr.children[1:]
])
]
call_expr.data = 'mutation'
for c in node.children:
self.visit_function_dot(c, block)
def visit_expr_values(self, node, fake_tree):
"""
Transforms a value with a direct index path into two lines.
"""
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'expression':
values = node.follow(['entity', 'values'])
if values is not None and len(values.children) > 1:
value_type = values.child(0).data
assert value_type == 'list' or value_type == 'map'
path_fragments = values.children[1:]
# insert the list as a new temporary reference
values.children = [values.children[0]]
obj = Tree('expression', node.children)
path = fake_tree.add_assignment(obj, original_line='1')
# insert all path fragments to it
path.children.extend(path_fragments)
# replace assignment with new path
node.children = [Tree('entity', [path])]
if node.data == 'block':
fake_tree = self.fake_tree(node)
for c in node.children:
self.visit_expr_values(c, fake_tree)
def add_strings(n1, *other_nodes):
"""
Create an AST for concating two or more nodes.
"""
# concatenation is currently defined as
# expression = expression arith_operator expression
base_tree = Tree("expression", [n1])
base_tree.kind = "arith_expression"
# if we only got one node, no concatenation is required. return
# directly
if len(other_nodes) == 0:
return base_tree.children[0]
base_tree.children.append(
Tree("arith_operator", [n1.create_token("PLUS", "+")]),
)
# Technically, the grammar only supports binary expressions, but
# the compiler and engine can handle n-ary expressions, so we can
# directly flatten the tree and add all additional nodes as extra
# expressions
for n2 in other_nodes:
base_tree.children.append(n2)
return base_tree
def visit_dot_expression(self, node, block, parent):
"""
Visit dot expression and lower them into mutation fragments.
"""
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'block':
# only generate a fake_block once for every line
# node: block in which the fake assignments should be inserted
block = self.fake_tree(node)
if node.data == 'dot_expression':
dot_expr = node
expression = Tree(
'mutation',
[
parent.primary_expression,
Tree(
'mutation_fragment',
[
dot_expr.child(0), # name
dot_expr.arguments
])
])
path = block.add_assignment(expression,
original_line=parent.line())
parent.children = [
Tree('primary_expression', [Tree('entity', [path])])
]
for c in node.children:
self.visit_dot_expression(c, block, parent=node)
def visit_function_dot(self, node, block):
"""
Visit function call with more than one path and lower
them into mutations.
"""
if not hasattr(node, "children") or len(node.children) == 0:
return
if node.data == "call_expression":
call_expr = node
if len(call_expr.path.children) > 1:
path_fragments = call_expr.path.children
call_expr.children = [
Tree(
"expression",
[
Tree(
"entity",
[Tree("path", call_expr.path.children[:-1])],
)
],
),
Tree(
"mutation_fragment",
[
path_fragments[-1].children[0],
*call_expr.children[1:],
],
),
]
call_expr.data = "mutation"
for c in node.children:
self.visit_function_dot(c, block)
def make_full_tree_from_cmp(expr):
"""
Builds a full tree from a cmp_expression node.
"""
return Tree('expression', [
Tree('or_expression',
[Tree('and_expression', [Tree('cmp_expression', [expr])])])
])
def build_string_value(cls, orig_node, text):
"""
Returns the AST for a plain string AST node with 'text'
Uses a `orig_node` to determine the line and column of the new Token.
"""
return Tree(
'values',
[Tree('string', [orig_node.create_token('DOUBLE_QUOTED', text)])])
def create_entity(path):
"""
Create an entity expression
"""
return Tree('expression', [
Tree('entity', [
path
])
])
def visit_assignment(self, node, block, parent):
"""
Visit assignments and lower destructors
"""
if not hasattr(node, "children") or len(node.children) == 0:
return
if node.data == "block":
# only generate a fake_block once for every line
# node: block in which the fake assignments should be inserted
block = self.fake_tree(node)
if node.data == "assignment":
c = node.children[0]
if c.data == "path":
# a path assignment -> no processing required
pass
else:
assert c.data == "assignment_destructoring"
line = node.line()
base_expr = node.assignment_fragment.base_expression
eq_tok = node.assignment_fragment.child(0)
orig_node = Tree("base_expression", base_expr.children)
orig_obj = block.add_assignment(orig_node, original_line=line)
for i, n in enumerate(c.children):
new_line = block.line()
n.expect(
len(n.children) == 1,
"object_destructoring_invalid_path",
)
name = n.child(0)
name.line = new_line # update token's line info
# <n> = <val>
val = self.create_entity(
Tree(
"path",
[
orig_obj.child(0),
Tree(
"path_fragment", [Tree("string", [name])]
),
],
)
)
a = block.assignment_path(n, val, new_line, eq_tok=eq_tok)
block.insert_node(a, name.line)
return True
else:
for c in node.children:
performed_destructuring = self.visit_assignment(
c, block, parent=node
)
if performed_destructuring:
parent.children.remove(node)
def process_concise_block(self, node, fake_tree):
"""
Creates a service_block around a concise_when_block and fixes up its
line numbers with the fake_tree.
"""
line = fake_tree.line()
# create token from the "new" line
name = Token("NAME", node.child(0).value, line=line)
path_token = Token("NAME", node.child(1).value, line=line)
t = Tree(
"service_block",
[
Tree(
"service",
[
Tree("path", [name]),
Tree(
"service_fragment",
[
Tree("command", [path_token]),
Tree("output", [path_token]),
],
),
],
),
Tree("nested_block", [Tree("block", [node.when_block])]),
],
)
return t
def visit_assignment(self, node, block, parent):
"""
Visit assignments and lower destructors
"""
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'block':
# only generate a fake_block once for every line
# node: block in which the fake assignments should be inserted
block = self.fake_tree(node)
if node.data == 'assignment':
c = node.children[0]
if c.data == 'path':
# a path assignment -> no processing required
pass
else:
assert c.data == 'assignment_destructoring'
line = node.line()
base_expr = node.assignment_fragment.base_expression
orig_node = Tree('base_expression', base_expr.children)
orig_obj = block.add_assignment(orig_node, original_line=line)
for i, n in enumerate(c.children):
new_line = block.line()
n.expect(len(n.children) == 1,
'object_destructoring_invalid_path')
name = n.child(0)
name.line = new_line # update token's line info
# <n> = <val>
val = self.create_entity(Tree('path', [
orig_obj.child(0),
Tree('path_fragment', [
Tree('string', [name])
])
]))
if i + 1 == len(c.children):
# for the last entry, we can recycle the existing node
node.children[0] = n
node.assignment_fragment.base_expression.children = \
[val]
else:
# insert new fake line
a = block.assignment_path(n, val, new_line)
parent.children.insert(0, a)
else:
for c in node.children:
self.visit_assignment(c, block, parent=node)
def visit_as_expr(self, node, block):
"""
Visit assignments and move 'as' up the tree if required
"""
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'foreach_block':
block = node.foreach_statement
assert block is not None
elif node.data == 'service_block' or node.data == 'when_block':
block = node.service.service_fragment
block = node.service.service_fragment
assert block is not None
if node.data == 'pow_expression':
as_op = node.as_operator
if as_op is not None and as_op.output_names is not None:
output = Tree('output', as_op.output_names.children)
node.expect(block is not None, 'service_no_inline_output')
block.children.append(output)
node.children = [node.children[0]]
for c in node.children:
self.visit_as_expr(c, block)
def add_strings(n1, *other_nodes):
"""
Create an AST for concating two or more nodes.
"""
# concatenation is currently defined as
# arith_expression = arith_expression arith_operator mul_expression
base_tree = Tree('arith_expression', [
Tree('arith_expression',
[Tree('mul_expression', [Tree('unary_expression', [n1])])])
])
# if we only got one node, no concatenation is required. return
# directly
if len(other_nodes) == 0:
return base_tree.children[0]
base_tree.children.append(Tree('arith_operator',
[Token('PLUS', '+')]), )
# Technically, the grammar only supports binary expressions, but
# the compiler and engine can handle n-ary expressions, so we can
# directly flatten the tree and add all additional nodes as extra
# mul_expressions
for n2 in other_nodes:
base_tree.children.append(
Tree('mul_expression', [Tree('unary_expression', [n2])]))
return base_tree
def visit(cls, node, block, entity, pred, fun, parent):
if not hasattr(node, 'children') or len(node.children) == 0:
return
if node.data == 'block':
# only generate a fake_block once for every line
# node: block in which the fake assignments should be inserted
block = cls.fake_tree(node)
elif node.data == 'entity' or node.data == 'key_value':
# set the parent where the inline_expression path should be
# inserted
entity = node
elif node.data == 'service' and node.child(0).data == 'path':
entity = node
# create fake lines for base_expressions too, but only when required:
# 1) `expressions` are already allowed to be nested
# 2) `assignment_fragments` are ignored to avoid two lines for simple
# service/mutation assignments (`a = my_service command`)
if node.data == 'base_expression' and \
node.child(0).data != 'expression' and \
parent.data != 'assignment_fragment':
# replace base_expression too
fun(node, block, node)
node.children = [Tree('path', node.children)]
for c in node.children:
cls.visit(c, block, entity, pred, fun, parent=node)
if pred(node):
assert entity is not None
assert block is not None
fake_tree = block
if not isinstance(fake_tree, FakeTree):
fake_tree = cls.fake_tree(block)
# Evaluate from leaf to the top
fun(node, fake_tree, entity.path)
# split services into service calls and mutations
if entity.data == 'service':
entity.entity = Tree('entity', [entity.path])
service_to_mutation(entity)
def rewrite_cmp_expr(cls, node):
cmp_op = node.cmp_operator
cmp_tok = cmp_op.child(0)
if cmp_tok.type == "NOT_EQUAL":
cmp_tok = cmp_op.create_token("EQUAL", "==")
elif cmp_tok.type == "GREATER_EQUAL":
cmp_tok = cmp_op.create_token("LESSER", "<")
cmp_op.children.reverse()
else:
assert cmp_tok.type == "GREATER"
cmp_tok = cmp_op.create_token("LESSER_EQUAL", "<=")
cmp_op.children.reverse()
# replace comparison token
cmp_op.children = [cmp_tok]
# create new comparison tree with 'NOT'
node.kind = "unary_expression"
node.children = [
Tree("unary_operator", [node.create_token("NOT", "!")]),
Tree("expression", node.children),
]
def rewrite_cmp_expr(cls, node):
cmp_op = node.cmp_operator
cmp_tok = cmp_op.child(0)
if cmp_tok.type == 'NOT_EQUAL':
cmp_tok = cmp_op.create_token('EQUAL', '==')
elif cmp_tok.type == 'GREATER_EQUAL':
cmp_tok = cmp_op.create_token('LESSER', '<')
cmp_op.children.reverse()
else:
assert cmp_tok.type == 'GREATER'
cmp_tok = cmp_op.create_token('LESSER_EQUAL', '<=')
cmp_op.children.reverse()
# replace comparison token
cmp_op.children = [cmp_tok]
# create new comparison tree with 'NOT'
unary_op = cls.create_unary_operation(
Tree('or_expression', [
Tree('and_expression', [Tree('cmp_expression', node.children)])
]))
node.children = [unary_op]
def create_entity(path):
"""
Create an entity expression
"""
return Tree('expression', [
Tree('or_expression', [
Tree('and_expression', [
Tree('cmp_expression', [
Tree('arith_expression', [
Tree('mul_expression', [
Tree('unary_expression', [
Tree('pow_expression', [
Tree('primary_expression',
[Tree('entity', [path])])
])
])
])
])
])
])
])
])
def rewrite_cmp_expr(cls, node):
cmp_op = node.cmp_operator
cmp_tok = cmp_op.child(0)
if cmp_tok.type == 'NOT_EQUAL':
cmp_tok = cmp_op.create_token('EQUAL', '==')
elif cmp_tok.type == 'GREATER_EQUAL':
cmp_tok = cmp_op.create_token('LESSER', '<')
cmp_op.children.reverse()
else:
assert cmp_tok.type == 'GREATER'
cmp_tok = cmp_op.create_token('LESSER_EQUAL', '<=')
cmp_op.children.reverse()
# replace comparison token
cmp_op.children = [cmp_tok]
# create new comparison tree with 'NOT'
node.kind = 'unary_expression'
node.children = [
Tree('unary_operator', [node.create_token('NOT', '!')]),
Tree('expression', node.children),
]
def create_unary_operation(child):
op = Tree('unary_operator', [child.create_token('NOT', '!')])
return Tree('arith_expression', [
Tree('mul_expression', [
Tree('unary_expression', [
op,
Tree('unary_expression', [
Tree('pow_expression',
[Tree('primary_expression', [child])])
])
])
])
])
def concat_string_templates(self, fake_tree, orig_node, string_objs):
"""
Concatenes the to-be-inserted string templates.
For example, a string template like "a{exp}b" gets flatten to:
"a" + fake_path_to_exp as string + "b"
Strings can be inserted directly, but string templates must be
evaluated to new AST nodes and the reference to their fake_node
assignment should be used instead.
"""
ks = []
for s in string_objs:
if s['$OBJECT'] == 'string':
# plain string -> insert directly
str_node = self.build_string_value(s['string'])
string_tree = Tree(
'pow_expression',
[Tree('primary_expression', [Tree('entity', [str_node])])])
ks.append(string_tree)
else:
assert s['$OBJECT'] == 'code'
# string template -> eval
# ignore newlines in string interpolation
code = ''.join(s['code'].split('\n'))
evaled_node = self.eval(orig_node, code, fake_tree)
# cast to string (`as string`)
base_type = orig_node.create_token('STRING_TYPE', 'string')
as_operator = Tree(
'as_operator',
[Tree('types', [Tree('base_type', [base_type])])])
as_tree = Tree('pow_expression', [
Tree('primary_expression', [
Tree('entity', [evaled_node]),
]), as_operator
])
ks.append(as_tree)
return ks
def process_concise_block(self, node, fake_tree):
"""
Creates a service_block around a concise_when_block and fixes up its
line numbers with the fake_tree.
"""
line = fake_tree.line()
# create token from the "new" line
name = Token('NAME', node.child(0).value, line=line)
path_token = Token('NAME', node.child(1).value, line=line)
t = Tree('service_block', [
Tree('service', [
Tree('path', [name]),
Tree('service_fragment', [
Tree('command', [path_token]),
Tree('output', [path_token]),
])
]),
Tree('nested_block', [Tree('block', [node.when_block])])
])
return t
def visit(cls, node, block, entity, pred, fun, parent):
if not hasattr(node, "children") or len(node.children) == 0:
return
if node.data == "block":
# only generate a fake_block once for every line
# node: block in which the fake assignments should be inserted
block = cls.fake_tree(node)
elif node.data == "entity" or node.data == "key_value":
# set the parent where the inline_expression path should be
# inserted
entity = node
elif node.data == "service" and node.child(0).data == "path":
entity = node
for c in node.children:
cls.visit(c, block, entity, pred, fun, parent=node)
# create fake lines for base_expressions too, but only when required:
# 1) `expressions` are already allowed to be nested
# 2) `assignment_fragments` are ignored to avoid two lines for simple
# service/mutation assignments (`a = my_service command`)
if (
node.data == "base_expression"
and node.child(0).data != "expression"
and parent.data != "assignment_fragment"
):
# replace base_expression too
fun(node, block, node)
node.children = [Tree("path", node.children)]
if pred(node):
assert entity is not None
assert block is not None
fake_tree = block
if not isinstance(fake_tree, FakeTree):
fake_tree = cls.fake_tree(block)
# Evaluate from leaf to the top
fun(node, fake_tree, entity.path)
entity.path.expect(
entity.data != "service", "service_name_not_inline_service"
)
def eval(self, orig_node, code_string, fake_tree):
"""
Evaluates a string by parsing it to its AST representation.
Inserts the AST expression as fake_node and returns the path
reference to the inserted fake_node.
"""
line = orig_node.line()
column = int(orig_node.column()) + 1
# add whitespace as padding to fixup the column location of the
# resulting tokens.
from storyscript.Story import Story
story = Story(" " * column + code_string, features=self.features)
story.parse(self.parser, allow_single_quotes=True)
new_node = story.tree
new_node = new_node.block
orig_node.expect(new_node, "string_templates_no_assignment")
# go to the actual node -> jump into block.rules or block.service
for i in range(2):
orig_node.expect(
len(new_node.children) == 1, "string_templates_no_assignment"
)
new_node = new_node.children[0]
# for now only expressions or service_blocks are allowed inside string
# templates
if (
new_node.data == "service_block"
and new_node.service_fragment is None
):
# it was a plain-old path initially
name = Token("NAME", code_string.strip(), line=line, column=column)
name.end_column = int(orig_node.end_column()) - 1
return Tree("path", [name])
if new_node.data == "absolute_expression":
new_node = new_node.children[0]
else:
orig_node.expect(
new_node.data == "service", "string_templates_no_assignment"
)
# the new assignment should be inserted at the top of the current block
return fake_tree.add_assignment(new_node, original_line=line)
def build_string_value(cls, text):
"""
Returns the AST for a plain string AST node with 'text'
"""
return Tree('values', [Tree('string', [Token('DOUBLE_QUOTED', text)])])
def create_entity(path):
"""
Create an entity expression
"""
return Tree("expression", [Tree("entity", [path])])
def concat_string_templates(self, fake_tree, orig_node, string_objs):
"""
Concatenates the to-be-inserted string templates.
For example, a string template like "a{exp}b" gets flatten to:
"a" + fake_path_to_exp as string + "b"
Strings can be inserted directly, but string templates must be
evaluated to new AST nodes and the reference to their fake_node
assignment should be used instead.
"""
ks = []
for s in string_objs:
if s["$OBJECT"] == "string":
# plain string -> insert directly
str_node = self.build_string_value(
orig_node=orig_node, text=s["string"]
)
string_tree = Tree("expression", [Tree("entity", [str_node])])
string_tree.kind = "primary_expression"
ks.append(string_tree)
else:
assert s["$OBJECT"] == "code"
# string template -> eval
# ignore newlines in string interpolation
code = "".join(s["code"].split("\n"))
evaled_node = self.eval(orig_node, code, fake_tree)
# cast to string (`as string`)
base_type = orig_node.create_token("STRING_TYPE", "string")
to_operator = Tree(
"to_operator",
[Tree("types", [Tree("base_type", [base_type])])],
)
as_tree = Tree(
"expression",
[
Tree("expression", [Tree("entity", [evaled_node]),]),
to_operator,
],
)
as_tree.kind = "pow_expression"
as_tree.child(0).kind = "primary_expression"
ks.append(as_tree)
return ks
