def recode(self, ast, eval): """Iterate over items in this ast. For each sub-ast, decode it; other items are taken verbatim. Form a new item list and use it to recreate an ASTNode of the original type. """ #self.logger.debug("RECODE: tag=%s" % ast.tag) merge_ok = (ast.tag in ('block', 'block_merge', 'cmdlist')) newitems = [] for item in ast.items: if type(item) == ASTNode: new_ast = self.decode(item, eval) if isinstance(new_ast, ASTNode) and \ (new_ast.tag == 'block_merge') and merge_ok: newitems.extend(new_ast.items) else: newitems.append(new_ast) else: newitems.append(item) # if new item is a code block of some kind, but contains no # statements, then change it to a block_merge to be merged # into the parent ast node if (ast.tag in ('async', 'sync', 'block', 'cmdlist')) and \ (len(newitems) == 0): #return self.nop return ASTNode('block_merge') return ASTNode(ast.tag, *newitems, **ast.attributes)
def decode_id_ref(self, ast, eval): """Decode a identifier reference. """ assert ast.tag == 'id_ref', ASTerr(ast) var = ast.items[0] # If this is a variable capturing the result of a DD command # then don't decode it as it will be used at execution time if re.match(r'^RET\d?$', var): return ast #print "Getting VAR=%s" % var res = eval.getAST(var) #print "RES= %s" % str(res) if isinstance(res, int) or isinstance(res, float): return ASTNode('number', res) if isinstance(res, str): return ASTNode('string', res) # NOP? #print "RES= %s" % str(res) ## if type(res) != ASTNode: ## raise DecodeError("Unexpected eval result in decoding type: %s='%s' (%s)" % ( ## var, str(res), str(type(res)))) if type(res) != Closure: raise DecodeError( "Unexpected eval result in decoding type: %s='%s' (%s)" % (var, str(res), str(type(res)))) # <-- res is an ASTNode return res
def merge(self, tag, astlist, **astattrs): newitems = [] for item in astlist: if type(item) != ASTNode: newitems.append(item) else: if item.tag == 'block_merge': newitems.extend(item.items) else: if (item.tag in ('async', 'sync', 'block', 'block_merge', 'cmdlist')) and (len(item.items) == 0): pass else: newitems.append(item) # if new item is a code block of some kind, but contains no # statements, then change it to a block_merge to be merged # into the parent ast node if (tag in ('async', 'sync', 'block', 'cmdlist')) and \ (len(newitems) == 0): return ASTNode('block_merge') return ASTNode(tag, *newitems, **astattrs)
def decode_star_set(self, ast, eval): """Decode a *SET statement. *SET <param_list> Evaluate the parameter list and set the variables within the current decoder environment. """ assert ast.tag == 'star_set', ASTerr(ast) assert len(ast.items) == 2, ASTerr(ast) params_ast = ast.items[0] close = (ast.items[1] != None) # Decode parameter list--should be no vars left afterward params = self._decode_params(params_ast, eval) #res = eval.set_params(params_ast, close=delay_eval) for var, val_ast in params.items(): # if they specified any -ALIASOFF we'll delay evaluation if close: val = make_closure(val_ast, eval) else: val = eval.eval(val_ast) params[var] = val eval.set_vars(params) # See Decoder class Note [1] #return self.nop return ASTNode('block_merge')
def p_factor6(p): """ factor : ID | AND | OR | IN """ p[0] = ASTNode('string', p[1])
def p_command_abs(p): """abs_command : factor param_list""" p[0] = ASTNode( 'abscmd', p[1], p[2], )
def parse(self, buf, startline=1): # Initialize module level error variables self.reset(lineno=startline) try: ast = self.parser.parse(buf, lexer=self.lexer) #print "errors=%d, AST=%s" % (self.errors, ast) # !!! HACK !!! MUST FIX PARSER!!! ## try: ## self.errinfo.pop() ## self.errors -= 1 ## ast = self.result ## except IndexError: ## pass #print ast #print errors, "errors" except Exception, e: # capture traceback? Yacc tracebacks aren't that useful ast = ASTNode('ERROR: %s' % str(e)) # verify errors>0 ??? #assert(self.errors > 0) if self.errors == 0: self.errors += 1
def p_abscmd(self, p): """abscmd : factor param_list""" p[0] = ASTNode( 'abscmd', p[1], p[2], )
def p_command_abs(self, p): """abs_command : STAR_SUB factor param_list""" p[0] = ASTNode( 'star_sub', p[2], p[3], )
def p_sync(self, p): """sync : exec_command SEMICOLON | abs_command SEMICOLON | command_block SEMICOLON | proc_call SEMICOLON """ p[0] = ASTNode('sync', p[1])
def p_async(self, p): """async : exec_command COMMA | abs_command COMMA | command_block COMMA | proc_call COMMA """ p[0] = ASTNode('async', p[1])
def p_command_list3(self, p): """command_list : async | sync | abs_command | special_form """ p[0] = ASTNode('cmdlist', p[1])
def _decode_merge(self, body_ast, eval): new_ast = self.decode(body_ast, eval) # if result is a command list, then mark it for merging into # one large result if isinstance(new_ast, ASTNode) and \ new_ast.tag == 'cmdlist': new_ast = ASTNode('block_merge', *new_ast.items) return new_ast
def p_special_form(self, p): """special_form : if_list | star_if_list | star_for_loop | while_loop | raise | star_set_stmnt | let_stmnt | set_stmnt | proc_defn """ p[0] = ASTNode('cmdlist', p[1])
def parse_opecmd(self, buf, startline=1): # Initialize module level error variables self.reset(lineno=startline) try: ast = self.ope_parser.parse(buf, lexer=self.lexer) except Exception, e: # capture traceback? Yacc tracebacks aren't that useful ast = ASTNode('ERROR: %s' % str(e)) # verify errors>0 assert (self.errors > 0)
def parse_params(self, buf): """Hack routine to parse a bare parameter list. """ # TODO: need separate lexer? parser? # Initialize module level error variables self.reset(lineno=1) try: ast = self.param_parser.parse(buf, lexer=self.lexer) except Exception, e: # capture traceback? Yacc tracebacks aren't that useful ast = ASTNode('ERROR: %s' % str(e))
def p_dyad1(p): """dyad : expression MUL expression | expression DIV expression | expression ADD expression | expression SUB expression | expression LT expression | expression GT expression | expression LE expression | expression GE expression | expression EQ expression | expression NE expression | expression AND expression | expression OR expression """ p[0] = ASTNode('dyad', p[1], p[2], p[3])
def __init__(self, evaluator, sk_bank, logger): """Decoder constructor. (params) is a dict of the initial environment (variables & values) of the decoder. (sk_bank) is used to lookup sk file ASTs and default parameters. """ # Evaluator for all external entities self.eval = evaluator # Object that lets me look up parsed abstract commands self.sk_bank = sk_bank self.logger = logger self.nop = ASTNode('nop') super(Decoder, self).__init__()
def _decode_string(self, ast, eval): """Decode a string interpolation. """ # cls = make_closure(ast, eval.clone()) # ast = ASTNode(ast.tag, ast.items[0]) # ast.cls = cls # return ast # Only expand variable references in strings in the decoding stage res = eval.eval_string_interpolate(ast.items[0], vars_only=True) if isinstance(res, str): return ASTNode(ast.tag, res) name = ast.items[0] raise DecodeError("Don't know how to decode type: %s='%s' (%s)" % (name, str(res), str(type(res))))
def decode_frame_id_ref(self, ast, eval): """Decode a frame allocation reference. """ assert ast.tag == 'frame_id_ref', ASTerr(ast) # In SOSS, frame id allocations are done in decoding! # uncomment this to reserve allocations until execution time ## res = eval.eval_string_interpolate(ast.items[0], vars_only=True) ## if isinstance(res, str): ## return ASTNode('frame_id_ref', res) # comment this to reserve allocations until execution time res = eval.eval(ast) if isinstance(res, str): return ASTNode('string', res) name = ast.items[0] raise DecodeError("Error decoding frame id reference: %s='%s' (%s)" % (name, str(res), str(type(res))))
def decode_star_if(self, ast, eval): """Decode a *IF statement. *IF <pred-exp> <then-clause> *ELIF <pred-exp> <then-clause> ... *ELSE <else-clause> *ENDIF AST is a variable number of 'cond' ribs (if-exp, then-exp). Else clause is represented by a final (True, then-exp) cond rib. """ assert ast.tag == 'star_if', ASTerr(ast) # Iterate over the set of cond ribs, decode the first then-part # for whose predicate evaluates to true. for cond_ast in ast.items: assert cond_ast.tag == 'cond', ASTerr(cond_ast) assert len(cond_ast.items) == 2, ASTerr(cond_ast) (pred_ast, then_ast) = cond_ast.items if pred_ast == True: # ELSE clause return self._decode_merge(then_ast, eval) # No longer. Could be dyad or monad... #assert pred_ast.tag == 'expression', ASTerr(pred_ast) res = eval.eval(pred_ast) if eval.isTrue(res): return self._decode_merge(then_ast, eval) # See Note [1] #return self.nop return ASTNode('block_merge')
def p_func_call(p): """func_call : ID LPAREN expression_list RPAREN""" p[0] = ASTNode('func_call', p[1], p[3])
def p_command_list(p): """command_list : exec_command | abs_command """ p[0] = ASTNode('cmdlist', p[1])
def p_frame_id_acquisition(p): """frame_id_ref : GET_F_NO LSTR""" p[0] = ASTNode('frame_id_ref', p[2])
def p_asnum1(p): """asnum : LPAREN expression RPAREN""" p[0] = ASTNode('asnum', p[2])
def p_expression_list1(p): """expression_list : expression""" p[0] = ASTNode('expression_list', p[1])
def p_command_exec(p): """exec_command : EXEC factor factor param_list""" p[0] = ASTNode('exec', p[2], p[3], p[4], None)
def p_param_list1(p): """ param_list : empty""" p[0] = ASTNode('param_list')
cmdstr = cmdstr.strip() res = sk_bank.parser.parse_opecmd(cmdstr) if res[0]: raise DecodeError("Error parsing command '%s': %s" % (cmdstr, res[2])) ast = res[1] assert ast.tag == 'cmdlist', ASTerr(ast) ast = ast.items[0] assert ast.tag == 'abscmd', ASTerr(ast) assert len(ast.items) == 2, ASTerr(ast) (ast_cmd_exp, ast_params) = ast.items # Make a *SUB ast and decode it ast = ASTNode('star_sub', ast_cmd_exp, ast_params) decoder = Decoder(eval, sk_bank, logger) newast = decoder.decode(ast, eval) print newast.AST2str() if options.verbose: print newast.printAST() return 0 def main(options, args):
def decode_star_sub(self, ast, eval): """Decode a *SUB statement. *SUB <cmd_exp> <param_list> Evaluate the <cmd_exp> to find the name of the abstract command being expanded. Evaluate <param_list> to find the initial values for the parameters and decode it. """ assert ast.tag == 'star_sub', ASTerr(ast) assert len(ast.items) == 2, ASTerr(ast) (ast_cmd_exp, ast_params) = ast.items # evaluate cmd_exp --> abs command name cmdname = eval.eval(ast_cmd_exp) assert type( cmdname ) == types.StringType, "command name does not evaluate to a string" # Decode actual parameters to abstract command call # should be no vars left after this actuals = self._decode_params(ast_params, eval) # Check for presence of special OBE_ID and OBE_MODE parameters try: obe_id_ast = actuals['obe_id'] except KeyError: raise DecodeError("No parameter set: OBE_ID") try: obe_mode_ast = actuals['obe_mode'] except KeyError: raise DecodeError("No parameter set: OBE_MODE") # Remove these two from the actuals (? is this correct?) del actuals['obe_id'] del actuals['obe_mode'] # Evaluate OBE_ID and OBE_MODE obe_id = eval.eval(obe_id_ast) assert type( obe_id) == types.StringType, "OBE_ID does not evaluate to a string" obe_mode = eval.eval(obe_mode_ast) assert type( obe_mode ) == types.StringType, "OBE_MODE does not evaluate to a string" # Make closures of the actuals #cur_eval = eval.clone() cur_eval = eval for (var, form) in actuals.items(): actuals[var] = make_closure(form, cur_eval) # Look up the skeleton file from our sk_bank skbunch = self.sk_bank.lookup(obe_id, obe_mode, cmdname) if skbunch.errors > 0: # TODO: include verbose error string in this error raise DecodeError("%d errors parsing referent skeleton file '%s'" % (skbunch.errors, skbunch.filepath)) ast = skbunch.ast assert ast.tag == 'skeleton', ASTerr(ast) assert len(ast.items) == 2, ASTerr(ast) (ast_default_params, ast_body) = ast.items # Get new evaluator with empty variables new_eval = eval.clone(inherit_vars=False) # Decode default parameters for abstract command # should be no vars left after this params = self._decode_params(ast_default_params, new_eval) # Make closures of them for (var, form) in params.items(): params[var] = make_closure(form, new_eval) #print "Defaults:", params # Now substitute actuals overriding defaults as necessary #print "Substituting:", actuals.keys() params.update(actuals) #print "Final:", params # Store decoded forms into new evaluator new_eval.set_vars(params) # Decode the skeleton file body in the new environment new_ast_body = self.decode(ast_body, new_eval) assert new_ast_body.tag == 'command_section', ASTerr(new_ast_body) assert len(new_ast_body.items) == 3, ASTerr(new_ast_body) # Extract the preprocessing ASTs from the decoded skeleton file # ast. (pre_ast, main_ast, post_ast) = new_ast_body.items # return our decoded command block. Each principal section is # processed synchronously within the command block. #res_ast = ASTNode('block_merge', res_ast = ASTNode('block', ASTNode('sync', ASTNode('block', pre_ast)), ASTNode('sync', ASTNode('block', main_ast)), ASTNode('sync', ASTNode('block', post_ast))) # Ugh..Because AST constructor was not designed to set name res_ast.name = '%s_%s_%s' % (obe_id.upper(), obe_mode.upper(), cmdname.upper()) return res_ast