def visit_function_dot(self, node, block): """ Visit function call with more than one path anme and lower them into mutations. """ if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'call_expression': call_expr = node if len(call_expr.path.children) > 1: path_fragments = call_expr.path.children if len(path_fragments) == 2: # don't rewrite s.length.max() yet call_expr.children = [ Tree('primary_expression', [ Tree('entity', [Tree('path', [call_expr.path.children[0]])]) ]), Tree('mutation_fragment', [ path_fragments[-1].children[0], *call_expr.children[1:] ]) ] call_expr.data = 'mutation' for c in node.children: self.visit_function_dot(c, block)
def visit_expr_values(self, node, fake_tree): """ Transforms a value with a direct index path into two lines. """ if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'expression': values = node.follow(['entity', 'values']) if values is not None and len(values.children) > 1: value_type = values.child(0).data assert value_type == 'list' or value_type == 'map' path_fragments = values.children[1:] # insert the list as a new temporary reference values.children = [values.children[0]] obj = Tree('expression', node.children) path = fake_tree.add_assignment(obj, original_line='1') # insert all path fragments to it path.children.extend(path_fragments) # replace assignment with new path node.children = [Tree('entity', [path])] if node.data == 'block': fake_tree = self.fake_tree(node) for c in node.children: self.visit_expr_values(c, fake_tree)
def add_strings(n1, *other_nodes): """ Create an AST for concating two or more nodes. """ # concatenation is currently defined as # expression = expression arith_operator expression base_tree = Tree("expression", [n1]) base_tree.kind = "arith_expression" # if we only got one node, no concatenation is required. return # directly if len(other_nodes) == 0: return base_tree.children[0] base_tree.children.append( Tree("arith_operator", [n1.create_token("PLUS", "+")]), ) # Technically, the grammar only supports binary expressions, but # the compiler and engine can handle n-ary expressions, so we can # directly flatten the tree and add all additional nodes as extra # expressions for n2 in other_nodes: base_tree.children.append(n2) return base_tree
def visit_dot_expression(self, node, block, parent): """ Visit dot expression and lower them into mutation fragments. """ if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'block': # only generate a fake_block once for every line # node: block in which the fake assignments should be inserted block = self.fake_tree(node) if node.data == 'dot_expression': dot_expr = node expression = Tree( 'mutation', [ parent.primary_expression, Tree( 'mutation_fragment', [ dot_expr.child(0), # name dot_expr.arguments ]) ]) path = block.add_assignment(expression, original_line=parent.line()) parent.children = [ Tree('primary_expression', [Tree('entity', [path])]) ] for c in node.children: self.visit_dot_expression(c, block, parent=node)
def visit_function_dot(self, node, block): """ Visit function call with more than one path and lower them into mutations. """ if not hasattr(node, "children") or len(node.children) == 0: return if node.data == "call_expression": call_expr = node if len(call_expr.path.children) > 1: path_fragments = call_expr.path.children call_expr.children = [ Tree( "expression", [ Tree( "entity", [Tree("path", call_expr.path.children[:-1])], ) ], ), Tree( "mutation_fragment", [ path_fragments[-1].children[0], *call_expr.children[1:], ], ), ] call_expr.data = "mutation" for c in node.children: self.visit_function_dot(c, block)
def make_full_tree_from_cmp(expr): """ Builds a full tree from a cmp_expression node. """ return Tree('expression', [ Tree('or_expression', [Tree('and_expression', [Tree('cmp_expression', [expr])])]) ])
def build_string_value(cls, orig_node, text): """ Returns the AST for a plain string AST node with 'text' Uses a `orig_node` to determine the line and column of the new Token. """ return Tree( 'values', [Tree('string', [orig_node.create_token('DOUBLE_QUOTED', text)])])
def create_entity(path): """ Create an entity expression """ return Tree('expression', [ Tree('entity', [ path ]) ])
def visit_assignment(self, node, block, parent): """ Visit assignments and lower destructors """ if not hasattr(node, "children") or len(node.children) == 0: return if node.data == "block": # only generate a fake_block once for every line # node: block in which the fake assignments should be inserted block = self.fake_tree(node) if node.data == "assignment": c = node.children[0] if c.data == "path": # a path assignment -> no processing required pass else: assert c.data == "assignment_destructoring" line = node.line() base_expr = node.assignment_fragment.base_expression eq_tok = node.assignment_fragment.child(0) orig_node = Tree("base_expression", base_expr.children) orig_obj = block.add_assignment(orig_node, original_line=line) for i, n in enumerate(c.children): new_line = block.line() n.expect( len(n.children) == 1, "object_destructoring_invalid_path", ) name = n.child(0) name.line = new_line # update token's line info # <n> = <val> val = self.create_entity( Tree( "path", [ orig_obj.child(0), Tree( "path_fragment", [Tree("string", [name])] ), ], ) ) a = block.assignment_path(n, val, new_line, eq_tok=eq_tok) block.insert_node(a, name.line) return True else: for c in node.children: performed_destructuring = self.visit_assignment( c, block, parent=node ) if performed_destructuring: parent.children.remove(node)
def process_concise_block(self, node, fake_tree): """ Creates a service_block around a concise_when_block and fixes up its line numbers with the fake_tree. """ line = fake_tree.line() # create token from the "new" line name = Token("NAME", node.child(0).value, line=line) path_token = Token("NAME", node.child(1).value, line=line) t = Tree( "service_block", [ Tree( "service", [ Tree("path", [name]), Tree( "service_fragment", [ Tree("command", [path_token]), Tree("output", [path_token]), ], ), ], ), Tree("nested_block", [Tree("block", [node.when_block])]), ], ) return t
def visit_assignment(self, node, block, parent): """ Visit assignments and lower destructors """ if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'block': # only generate a fake_block once for every line # node: block in which the fake assignments should be inserted block = self.fake_tree(node) if node.data == 'assignment': c = node.children[0] if c.data == 'path': # a path assignment -> no processing required pass else: assert c.data == 'assignment_destructoring' line = node.line() base_expr = node.assignment_fragment.base_expression orig_node = Tree('base_expression', base_expr.children) orig_obj = block.add_assignment(orig_node, original_line=line) for i, n in enumerate(c.children): new_line = block.line() n.expect(len(n.children) == 1, 'object_destructoring_invalid_path') name = n.child(0) name.line = new_line # update token's line info # <n> = <val> val = self.create_entity(Tree('path', [ orig_obj.child(0), Tree('path_fragment', [ Tree('string', [name]) ]) ])) if i + 1 == len(c.children): # for the last entry, we can recycle the existing node node.children[0] = n node.assignment_fragment.base_expression.children = \ [val] else: # insert new fake line a = block.assignment_path(n, val, new_line) parent.children.insert(0, a) else: for c in node.children: self.visit_assignment(c, block, parent=node)
def visit_as_expr(self, node, block): """ Visit assignments and move 'as' up the tree if required """ if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'foreach_block': block = node.foreach_statement assert block is not None elif node.data == 'service_block' or node.data == 'when_block': block = node.service.service_fragment block = node.service.service_fragment assert block is not None if node.data == 'pow_expression': as_op = node.as_operator if as_op is not None and as_op.output_names is not None: output = Tree('output', as_op.output_names.children) node.expect(block is not None, 'service_no_inline_output') block.children.append(output) node.children = [node.children[0]] for c in node.children: self.visit_as_expr(c, block)
def add_strings(n1, *other_nodes): """ Create an AST for concating two or more nodes. """ # concatenation is currently defined as # arith_expression = arith_expression arith_operator mul_expression base_tree = Tree('arith_expression', [ Tree('arith_expression', [Tree('mul_expression', [Tree('unary_expression', [n1])])]) ]) # if we only got one node, no concatenation is required. return # directly if len(other_nodes) == 0: return base_tree.children[0] base_tree.children.append(Tree('arith_operator', [Token('PLUS', '+')]), ) # Technically, the grammar only supports binary expressions, but # the compiler and engine can handle n-ary expressions, so we can # directly flatten the tree and add all additional nodes as extra # mul_expressions for n2 in other_nodes: base_tree.children.append( Tree('mul_expression', [Tree('unary_expression', [n2])])) return base_tree
def visit(cls, node, block, entity, pred, fun, parent): if not hasattr(node, 'children') or len(node.children) == 0: return if node.data == 'block': # only generate a fake_block once for every line # node: block in which the fake assignments should be inserted block = cls.fake_tree(node) elif node.data == 'entity' or node.data == 'key_value': # set the parent where the inline_expression path should be # inserted entity = node elif node.data == 'service' and node.child(0).data == 'path': entity = node # create fake lines for base_expressions too, but only when required: # 1) `expressions` are already allowed to be nested # 2) `assignment_fragments` are ignored to avoid two lines for simple # service/mutation assignments (`a = my_service command`) if node.data == 'base_expression' and \ node.child(0).data != 'expression' and \ parent.data != 'assignment_fragment': # replace base_expression too fun(node, block, node) node.children = [Tree('path', node.children)] for c in node.children: cls.visit(c, block, entity, pred, fun, parent=node) if pred(node): assert entity is not None assert block is not None fake_tree = block if not isinstance(fake_tree, FakeTree): fake_tree = cls.fake_tree(block) # Evaluate from leaf to the top fun(node, fake_tree, entity.path) # split services into service calls and mutations if entity.data == 'service': entity.entity = Tree('entity', [entity.path]) service_to_mutation(entity)
def rewrite_cmp_expr(cls, node): cmp_op = node.cmp_operator cmp_tok = cmp_op.child(0) if cmp_tok.type == "NOT_EQUAL": cmp_tok = cmp_op.create_token("EQUAL", "==") elif cmp_tok.type == "GREATER_EQUAL": cmp_tok = cmp_op.create_token("LESSER", "<") cmp_op.children.reverse() else: assert cmp_tok.type == "GREATER" cmp_tok = cmp_op.create_token("LESSER_EQUAL", "<=") cmp_op.children.reverse() # replace comparison token cmp_op.children = [cmp_tok] # create new comparison tree with 'NOT' node.kind = "unary_expression" node.children = [ Tree("unary_operator", [node.create_token("NOT", "!")]), Tree("expression", node.children), ]
def rewrite_cmp_expr(cls, node): cmp_op = node.cmp_operator cmp_tok = cmp_op.child(0) if cmp_tok.type == 'NOT_EQUAL': cmp_tok = cmp_op.create_token('EQUAL', '==') elif cmp_tok.type == 'GREATER_EQUAL': cmp_tok = cmp_op.create_token('LESSER', '<') cmp_op.children.reverse() else: assert cmp_tok.type == 'GREATER' cmp_tok = cmp_op.create_token('LESSER_EQUAL', '<=') cmp_op.children.reverse() # replace comparison token cmp_op.children = [cmp_tok] # create new comparison tree with 'NOT' unary_op = cls.create_unary_operation( Tree('or_expression', [ Tree('and_expression', [Tree('cmp_expression', node.children)]) ])) node.children = [unary_op]
def create_entity(path): """ Create an entity expression """ return Tree('expression', [ Tree('or_expression', [ Tree('and_expression', [ Tree('cmp_expression', [ Tree('arith_expression', [ Tree('mul_expression', [ Tree('unary_expression', [ Tree('pow_expression', [ Tree('primary_expression', [Tree('entity', [path])]) ]) ]) ]) ]) ]) ]) ]) ])
def rewrite_cmp_expr(cls, node): cmp_op = node.cmp_operator cmp_tok = cmp_op.child(0) if cmp_tok.type == 'NOT_EQUAL': cmp_tok = cmp_op.create_token('EQUAL', '==') elif cmp_tok.type == 'GREATER_EQUAL': cmp_tok = cmp_op.create_token('LESSER', '<') cmp_op.children.reverse() else: assert cmp_tok.type == 'GREATER' cmp_tok = cmp_op.create_token('LESSER_EQUAL', '<=') cmp_op.children.reverse() # replace comparison token cmp_op.children = [cmp_tok] # create new comparison tree with 'NOT' node.kind = 'unary_expression' node.children = [ Tree('unary_operator', [node.create_token('NOT', '!')]), Tree('expression', node.children), ]
def create_unary_operation(child): op = Tree('unary_operator', [child.create_token('NOT', '!')]) return Tree('arith_expression', [ Tree('mul_expression', [ Tree('unary_expression', [ op, Tree('unary_expression', [ Tree('pow_expression', [Tree('primary_expression', [child])]) ]) ]) ]) ])
def concat_string_templates(self, fake_tree, orig_node, string_objs): """ Concatenes the to-be-inserted string templates. For example, a string template like "a{exp}b" gets flatten to: "a" + fake_path_to_exp as string + "b" Strings can be inserted directly, but string templates must be evaluated to new AST nodes and the reference to their fake_node assignment should be used instead. """ ks = [] for s in string_objs: if s['$OBJECT'] == 'string': # plain string -> insert directly str_node = self.build_string_value(s['string']) string_tree = Tree( 'pow_expression', [Tree('primary_expression', [Tree('entity', [str_node])])]) ks.append(string_tree) else: assert s['$OBJECT'] == 'code' # string template -> eval # ignore newlines in string interpolation code = ''.join(s['code'].split('\n')) evaled_node = self.eval(orig_node, code, fake_tree) # cast to string (`as string`) base_type = orig_node.create_token('STRING_TYPE', 'string') as_operator = Tree( 'as_operator', [Tree('types', [Tree('base_type', [base_type])])]) as_tree = Tree('pow_expression', [ Tree('primary_expression', [ Tree('entity', [evaled_node]), ]), as_operator ]) ks.append(as_tree) return ks
def process_concise_block(self, node, fake_tree): """ Creates a service_block around a concise_when_block and fixes up its line numbers with the fake_tree. """ line = fake_tree.line() # create token from the "new" line name = Token('NAME', node.child(0).value, line=line) path_token = Token('NAME', node.child(1).value, line=line) t = Tree('service_block', [ Tree('service', [ Tree('path', [name]), Tree('service_fragment', [ Tree('command', [path_token]), Tree('output', [path_token]), ]) ]), Tree('nested_block', [Tree('block', [node.when_block])]) ]) return t
def visit(cls, node, block, entity, pred, fun, parent): if not hasattr(node, "children") or len(node.children) == 0: return if node.data == "block": # only generate a fake_block once for every line # node: block in which the fake assignments should be inserted block = cls.fake_tree(node) elif node.data == "entity" or node.data == "key_value": # set the parent where the inline_expression path should be # inserted entity = node elif node.data == "service" and node.child(0).data == "path": entity = node for c in node.children: cls.visit(c, block, entity, pred, fun, parent=node) # create fake lines for base_expressions too, but only when required: # 1) `expressions` are already allowed to be nested # 2) `assignment_fragments` are ignored to avoid two lines for simple # service/mutation assignments (`a = my_service command`) if ( node.data == "base_expression" and node.child(0).data != "expression" and parent.data != "assignment_fragment" ): # replace base_expression too fun(node, block, node) node.children = [Tree("path", node.children)] if pred(node): assert entity is not None assert block is not None fake_tree = block if not isinstance(fake_tree, FakeTree): fake_tree = cls.fake_tree(block) # Evaluate from leaf to the top fun(node, fake_tree, entity.path) entity.path.expect( entity.data != "service", "service_name_not_inline_service" )
def eval(self, orig_node, code_string, fake_tree): """ Evaluates a string by parsing it to its AST representation. Inserts the AST expression as fake_node and returns the path reference to the inserted fake_node. """ line = orig_node.line() column = int(orig_node.column()) + 1 # add whitespace as padding to fixup the column location of the # resulting tokens. from storyscript.Story import Story story = Story(" " * column + code_string, features=self.features) story.parse(self.parser, allow_single_quotes=True) new_node = story.tree new_node = new_node.block orig_node.expect(new_node, "string_templates_no_assignment") # go to the actual node -> jump into block.rules or block.service for i in range(2): orig_node.expect( len(new_node.children) == 1, "string_templates_no_assignment" ) new_node = new_node.children[0] # for now only expressions or service_blocks are allowed inside string # templates if ( new_node.data == "service_block" and new_node.service_fragment is None ): # it was a plain-old path initially name = Token("NAME", code_string.strip(), line=line, column=column) name.end_column = int(orig_node.end_column()) - 1 return Tree("path", [name]) if new_node.data == "absolute_expression": new_node = new_node.children[0] else: orig_node.expect( new_node.data == "service", "string_templates_no_assignment" ) # the new assignment should be inserted at the top of the current block return fake_tree.add_assignment(new_node, original_line=line)
def build_string_value(cls, text): """ Returns the AST for a plain string AST node with 'text' """ return Tree('values', [Tree('string', [Token('DOUBLE_QUOTED', text)])])
def create_entity(path): """ Create an entity expression """ return Tree("expression", [Tree("entity", [path])])
def concat_string_templates(self, fake_tree, orig_node, string_objs): """ Concatenates the to-be-inserted string templates. For example, a string template like "a{exp}b" gets flatten to: "a" + fake_path_to_exp as string + "b" Strings can be inserted directly, but string templates must be evaluated to new AST nodes and the reference to their fake_node assignment should be used instead. """ ks = [] for s in string_objs: if s["$OBJECT"] == "string": # plain string -> insert directly str_node = self.build_string_value( orig_node=orig_node, text=s["string"] ) string_tree = Tree("expression", [Tree("entity", [str_node])]) string_tree.kind = "primary_expression" ks.append(string_tree) else: assert s["$OBJECT"] == "code" # string template -> eval # ignore newlines in string interpolation code = "".join(s["code"].split("\n")) evaled_node = self.eval(orig_node, code, fake_tree) # cast to string (`as string`) base_type = orig_node.create_token("STRING_TYPE", "string") to_operator = Tree( "to_operator", [Tree("types", [Tree("base_type", [base_type])])], ) as_tree = Tree( "expression", [ Tree("expression", [Tree("entity", [evaled_node]),]), to_operator, ], ) as_tree.kind = "pow_expression" as_tree.child(0).kind = "primary_expression" ks.append(as_tree) return ks