def recode(self, ast, eval): """Iterate over items in this ast. For each sub-ast, decode it; other items are taken verbatim. Form a new item list and use it to recreate an ASTNode of the original type. """ #self.logger.debug("RECODE: tag=%s" % ast.tag) merge_ok = (ast.tag in ('block', 'block_merge', 'cmdlist')) newitems = [] for item in ast.items: if type(item) == ASTNode: new_ast = self.decode(item, eval) if isinstance(new_ast, ASTNode) and \ (new_ast.tag == 'block_merge') and merge_ok: newitems.extend(new_ast.items) else: newitems.append(new_ast) else: newitems.append(item) # if new item is a code block of some kind, but contains no # statements, then change it to a block_merge to be merged # into the parent ast node if (ast.tag in ('async', 'sync', 'block', 'cmdlist')) and \ (len(newitems) == 0): #return self.nop return ASTNode('block_merge') return ASTNode(ast.tag, *newitems, **ast.attributes)
def decode_id_ref(self, ast, eval): """Decode a identifier reference. """ assert ast.tag == 'id_ref', ASTerr(ast) var = ast.items[0] # If this is a variable capturing the result of a DD command # then don't decode it as it will be used at execution time if re.match(r'^RET\d?$', var): return ast #print "Getting VAR=%s" % var res = eval.getAST(var) #print "RES= %s" % str(res) if isinstance(res, int) or isinstance(res, float): return ASTNode('number', res) if isinstance(res, str): return ASTNode('string', res) # NOP? #print "RES= %s" % str(res) ## if type(res) != ASTNode: ## raise DecodeError("Unexpected eval result in decoding type: %s='%s' (%s)" % ( ## var, str(res), str(type(res)))) if type(res) != Closure: raise DecodeError( "Unexpected eval result in decoding type: %s='%s' (%s)" % (var, str(res), str(type(res)))) # <-- res is an ASTNode return res
def merge(self, tag, astlist, **astattrs): newitems = [] for item in astlist: if type(item) != ASTNode: newitems.append(item) else: if item.tag == 'block_merge': newitems.extend(item.items) else: if (item.tag in ('async', 'sync', 'block', 'block_merge', 'cmdlist')) and (len(item.items) == 0): pass else: newitems.append(item) # if new item is a code block of some kind, but contains no # statements, then change it to a block_merge to be merged # into the parent ast node if (tag in ('async', 'sync', 'block', 'cmdlist')) and \ (len(newitems) == 0): return ASTNode('block_merge') return ASTNode(tag, *newitems, **astattrs)
def p_factor6(p): """ factor : ID | AND | OR | IN """ p[0] = ASTNode('string', p[1])
def decode_star_set(self, ast, eval): """Decode a *SET statement. *SET <param_list> Evaluate the parameter list and set the variables within the current decoder environment. """ assert ast.tag == 'star_set', ASTerr(ast) assert len(ast.items) == 2, ASTerr(ast) params_ast = ast.items[0] close = (ast.items[1] != None) # Decode parameter list--should be no vars left afterward params = self._decode_params(params_ast, eval) #res = eval.set_params(params_ast, close=delay_eval) for var, val_ast in params.items(): # if they specified any -ALIASOFF we'll delay evaluation if close: val = make_closure(val_ast, eval) else: val = eval.eval(val_ast) params[var] = val eval.set_vars(params) # See Decoder class Note [1] #return self.nop return ASTNode('block_merge')
def p_command_abs(self, p): """abs_command : STAR_SUB factor param_list""" p[0] = ASTNode( 'star_sub', p[2], p[3], )
def p_abscmd(self, p): """abs_cmd : factor param_list""" p[0] = ASTNode( 'abscmd', p[1], p[2], )
def parse(self, buf, startline=1): # Initialize module level error variables self.reset(lineno=startline) try: ast = self.parser.parse(buf, lexer=self.lexer) #print("errors=%d, AST=%s" % (self.errors, ast)) ## # !!! HACK !!! MUST FIX PARSER!!! ## try: ## print(self.errors, "errors") ## self.errinfo.pop() ## self.errors -= 1 ## ast = self.result ## except IndexError: ## pass ## print(ast) except Exception as e: # capture traceback? Yacc tracebacks aren't that useful errstr = 'ERROR: %s' % (str(e)) ast = ASTNode(errstr) # verify errors>0 ??? #assert(self.errors > 0) if self.errors == 0: self.errors += 1 self.errinfo.append( Bunch.Bunch(lineno=self.lexer.lexer.lineno, errstr=errstr, token=None)) self.logger.error(errstr) return (self.errors, ast, self.errinfo)
def p_command_list3(self, p): """command_list : async | sync | abs_command | special_form """ p[0] = ASTNode('cmdlist', p[1])
def parse_params(self, buf): """Hack routine to parse a bare parameter list. """ # TODO: need separate lexer? parser? # Initialize module level error variables self.reset(lineno=1) try: ast = self.param_parser.parse(buf, lexer=self.lexer) except Exception as e: # capture traceback? Yacc tracebacks aren't that useful ast = ASTNode('ERROR: %s' % str(e)) # verify errors>0 #assert(self.errors > 0) try: assert (ast.tag == 'param_list') except AssertionError: # ?? We're being silent like normal parsing pass return (self.errors, ast, self.errinfo)
def parse_skbuf(self, buf): # Get the constituent parts of a skeleton file: # header, parameter list, command part (hdrbuf, prmbuf, cmdbuf, startline) = sk_common.get_skparts(buf) # print("header", hdrbuf) # print("params", prmbuf) # print("commands", cmdbuf) # Get the header params try: header, _2, _3 = collect_params(hdrbuf) except Exception as e: # don't let parsing errors of the header hold us back # header is not really used for anything important header = {} # Make a buffer of the default params in an easily parsable form params, param_lst, patterns = collect_params(prmbuf) parambuf = ' '.join(param_lst) #print(parambuf) # Parse default params into an ast. (errors, ast_params, errinfo) = self.parse_params(parambuf) #print("ast_params:", ast_params.printAST()) # This will hold the results res = Bunch.Bunch(errors=errors, errinfo=errinfo, header=header) # make readable errors if errors > 0: #print("ERRINFO = ", errinfo) for errbnch in errinfo: errbnch.verbose = sk_common.mk_error(parambuf, errbnch, 1) # parse the command part (errors, ast_cmds, errinfo) = self.parse(cmdbuf, startline=startline) # Append errinfo together res.errors += errors res.errinfo.extend(errinfo) # make readable errors for errbnch in errinfo: errbnch.verbose = sk_common.mk_error(cmdbuf, errbnch, 10) res.params = params res.patterns = patterns # Finally, glue the params AST and the commands AST together to make # "skeleton" node res.ast = ASTNode("skeleton", ast_params, ast_cmds) # return a bundle of these objects return res
def p_sync(self, p): """sync : exec_command SEMICOLON | abs_command SEMICOLON | command_block SEMICOLON | proc_call SEMICOLON | while_loop SEMICOLON | let_stmnt SEMICOLON | catch SEMICOLON """ p[0] = ASTNode('sync', p[1])
def p_async(self, p): """async : exec_command COMMA | abs_command COMMA | command_block COMMA | proc_call COMMA | while_loop COMMA | let_stmnt COMMA | catch COMMA """ p[0] = ASTNode('async', p[1])
def _decode_merge(self, body_ast, eval): new_ast = self.decode(body_ast, eval) # if result is a command list, then mark it for merging into # one large result if isinstance(new_ast, ASTNode) and \ new_ast.tag == 'cmdlist': new_ast = ASTNode('block_merge', *new_ast.items) return new_ast
def __init__(self, evaluator, sk_bank, logger): """Decoder constructor. (params) is a dict of the initial environment (variables & values) of the decoder. (sk_bank) is used to lookup sk file ASTs and default parameters. """ # Evaluator for all external entities self.eval = evaluator # Object that lets me look up parsed abstract commands self.sk_bank = sk_bank self.logger = logger self.nop = ASTNode('nop') super(Decoder, self).__init__()
def p_special_form(self, p): """special_form : if_list | star_if_list | star_for_loop | while_loop | catch | raise | return | star_set_stmnt | let_stmnt | set_stmnt | proc_defn | import_stmnt """ p[0] = ASTNode('cmdlist', p[1])
def p_dyad1(p): """dyad : expression MUL expression | expression DIV expression | expression ADD expression | expression SUB expression | expression LT expression | expression GT expression | expression LE expression | expression GE expression | expression EQ expression | expression NE expression | expression AND expression | expression OR expression """ p[0] = ASTNode('dyad', p[1], p[2], p[3])
def decode_frame_id_ref(self, ast, eval): """Decode a frame allocation reference. """ assert ast.tag == 'frame_id_ref', ASTerr(ast) # In SOSS, frame id allocations are done in decoding! # uncomment this to reserve allocations until execution time ## res = eval.eval_string_interpolate(ast.items[0], vars_only=True) ## if isinstance(res, str): ## return ASTNode('frame_id_ref', res) # comment this to reserve allocations until execution time res = eval.eval(ast) if isinstance(res, str): return ASTNode('string', res) name = ast.items[0] raise DecodeError("Error decoding frame id reference: %s='%s' (%s)" % (name, str(res), str(type(res))))
def _decode_string(self, ast, eval): """Decode a string interpolation. """ # cls = make_closure(ast, eval.clone()) # ast = ASTNode(ast.tag, ast.items[0]) # ast.cls = cls # return ast # Only expand variable references in strings in the decoding stage res = eval.eval_string_interpolate(ast.items[0], vars_only=True) if isinstance(res, str): return ASTNode(ast.tag, res) name = ast.items[0] raise DecodeError("Don't know how to decode type: %s='%s' (%s)" % (name, str(res), str(type(res))))
def decode_star_if(self, ast, eval): """Decode a *IF statement. *IF <pred-exp> <then-clause> *ELIF <pred-exp> <then-clause> ... *ELSE <else-clause> *ENDIF AST is a variable number of 'cond' ribs (if-exp, then-exp). Else clause is represented by a final (True, then-exp) cond rib. """ assert ast.tag == 'star_if', ASTerr(ast) # Iterate over the set of cond ribs, decode the first then-part # for whose predicate evaluates to true. for cond_ast in ast.items: assert cond_ast.tag == 'cond', ASTerr(cond_ast) assert len(cond_ast.items) == 2, ASTerr(cond_ast) (pred_ast, then_ast) = cond_ast.items if pred_ast == True: # ELSE clause return self._decode_merge(then_ast, eval) # No longer. Could be dyad or monad... #assert pred_ast.tag == 'expression', ASTerr(pred_ast) res = eval.eval(pred_ast) if eval.isTrue(res): return self._decode_merge(then_ast, eval) # See Note [1] #return self.nop return ASTNode('block_merge')
def parse_opecmd(self, buf, startline=1): # Initialize module level error variables self.reset(lineno=startline) try: ast = self.ope_parser.parse(buf, lexer=self.lexer) except Exception as e: # capture traceback? Yacc tracebacks aren't that useful ast = ASTNode('ERROR: %s' % str(e)) # verify errors>0 assert (self.errors > 0) try: assert (ast.tag == 'cmdlist') except AssertionError: # ?? We're being silent like normal parsing pass return (self.errors, ast, self.errinfo)
def p_proc_defn1(self, p): """proc_defn : DEF ID LPAREN varlist RPAREN command_block""" p[0] = ASTNode('proc', p[2], p[4], p[6])
def p_raise(self, p): """raise : RAISE expression""" p[0] = ASTNode('raise', p[2])
def p_import(self, p): """import_stmnt : FROM QSTR IMPORT varlist""" p[0] = ASTNode('import', p[2], p[4])
def p_ddcmd(self, p): """dd_cmd : EXEC factor factor param_list""" p[0] = ASTNode('exec', p[2], p[3], p[4], None)
def p_varlist1(self, p): """varlist : ID""" p[0] = ASTNode('varlist', p[1])
def p_idlist1(self, p): """idlist : expressions""" p[0] = ASTNode('idlist', p[1])
def p_return2(self, p): """return : RETURN""" p[0] = ASTNode('return')
def p_return(self, p): """return : RETURN expression""" p[0] = ASTNode('return', p[2])
def p_catch1(self, p): """catch : CATCH ID command_block""" p[0] = ASTNode('catch', p[2], p[3])