def recode(self, ast, eval):
"""Iterate over items in this ast. For each sub-ast, decode it;
other items are taken verbatim. Form a new item list and use
it to recreate an ASTNode of the original type.
"""
#self.logger.debug("RECODE: tag=%s" % ast.tag)
merge_ok = (ast.tag in ('block', 'block_merge', 'cmdlist'))
newitems = []
for item in ast.items:
if type(item) == ASTNode:
new_ast = self.decode(item, eval)
if isinstance(new_ast, ASTNode) and \
(new_ast.tag == 'block_merge') and merge_ok:
newitems.extend(new_ast.items)
else:
newitems.append(new_ast)
else:
newitems.append(item)
# if new item is a code block of some kind, but contains no
# statements, then change it to a block_merge to be merged
# into the parent ast node
if (ast.tag in ('async', 'sync', 'block', 'cmdlist')) and \
(len(newitems) == 0):
#return self.nop
return ASTNode('block_merge')
return ASTNode(ast.tag, *newitems, **ast.attributes)
def decode_id_ref(self, ast, eval):
"""Decode a identifier reference.
"""
assert ast.tag == 'id_ref', ASTerr(ast)
var = ast.items[0]
# If this is a variable capturing the result of a DD command
# then don't decode it as it will be used at execution time
if re.match(r'^RET\d?$', var):
return ast
#print "Getting VAR=%s" % var
res = eval.getAST(var)
#print "RES= %s" % str(res)
if isinstance(res, int) or isinstance(res, float):
return ASTNode('number', res)
if isinstance(res, str):
return ASTNode('string', res)
# NOP?
#print "RES= %s" % str(res)
## if type(res) != ASTNode:
## raise DecodeError("Unexpected eval result in decoding type: %s='%s' (%s)" % (
## var, str(res), str(type(res))))
if type(res) != Closure:
raise DecodeError(
"Unexpected eval result in decoding type: %s='%s' (%s)" %
(var, str(res), str(type(res))))
# <-- res is an ASTNode
return res
def merge(self, tag, astlist, **astattrs):
newitems = []
for item in astlist:
if type(item) != ASTNode:
newitems.append(item)
else:
if item.tag == 'block_merge':
newitems.extend(item.items)
else:
if (item.tag in ('async', 'sync', 'block', 'block_merge',
'cmdlist')) and (len(item.items) == 0):
pass
else:
newitems.append(item)
# if new item is a code block of some kind, but contains no
# statements, then change it to a block_merge to be merged
# into the parent ast node
if (tag in ('async', 'sync', 'block', 'cmdlist')) and \
(len(newitems) == 0):
return ASTNode('block_merge')
return ASTNode(tag, *newitems, **astattrs)
def p_factor6(p):
""" factor : ID
| AND
| OR
| IN
"""
p[0] = ASTNode('string', p[1])
def decode_star_set(self, ast, eval):
"""Decode a *SET statement.
*SET <param_list>
Evaluate the parameter list and set the variables within the
current decoder environment.
"""
assert ast.tag == 'star_set', ASTerr(ast)
assert len(ast.items) == 2, ASTerr(ast)
params_ast = ast.items[0]
close = (ast.items[1] != None)
# Decode parameter list--should be no vars left afterward
params = self._decode_params(params_ast, eval)
#res = eval.set_params(params_ast, close=delay_eval)
for var, val_ast in params.items():
# if they specified any -ALIASOFF we'll delay evaluation
if close:
val = make_closure(val_ast, eval)
else:
val = eval.eval(val_ast)
params[var] = val
eval.set_vars(params)
# See Decoder class Note [1]
#return self.nop
return ASTNode('block_merge')
def p_command_abs(self, p):
"""abs_command : STAR_SUB factor param_list"""
p[0] = ASTNode(
'star_sub',
p[2],
p[3],
)
def p_abscmd(self, p):
"""abs_cmd : factor param_list"""
p[0] = ASTNode(
'abscmd',
p[1],
p[2],
)
def parse(self, buf, startline=1):
# Initialize module level error variables
self.reset(lineno=startline)
try:
ast = self.parser.parse(buf, lexer=self.lexer)
#print("errors=%d, AST=%s" % (self.errors, ast))
## # !!! HACK !!! MUST FIX PARSER!!!
## try:
## print(self.errors, "errors")
## self.errinfo.pop()
## self.errors -= 1
## ast = self.result
## except IndexError:
## pass
## print(ast)
except Exception as e:
# capture traceback? Yacc tracebacks aren't that useful
errstr = 'ERROR: %s' % (str(e))
ast = ASTNode(errstr)
# verify errors>0 ???
#assert(self.errors > 0)
if self.errors == 0:
self.errors += 1
self.errinfo.append(
Bunch.Bunch(lineno=self.lexer.lexer.lineno,
errstr=errstr,
token=None))
self.logger.error(errstr)
return (self.errors, ast, self.errinfo)
def p_command_list3(self, p):
"""command_list : async
| sync
| abs_command
| special_form
"""
p[0] = ASTNode('cmdlist', p[1])
def parse_params(self, buf):
"""Hack routine to parse a bare parameter list.
"""
# TODO: need separate lexer? parser?
# Initialize module level error variables
self.reset(lineno=1)
try:
ast = self.param_parser.parse(buf, lexer=self.lexer)
except Exception as e:
# capture traceback? Yacc tracebacks aren't that useful
ast = ASTNode('ERROR: %s' % str(e))
# verify errors>0
#assert(self.errors > 0)
try:
assert (ast.tag == 'param_list')
except AssertionError:
# ?? We're being silent like normal parsing
pass
return (self.errors, ast, self.errinfo)
def parse_skbuf(self, buf):
# Get the constituent parts of a skeleton file:
# header, parameter list, command part
(hdrbuf, prmbuf, cmdbuf, startline) = sk_common.get_skparts(buf)
# print("header", hdrbuf)
# print("params", prmbuf)
# print("commands", cmdbuf)
# Get the header params
try:
header, _2, _3 = collect_params(hdrbuf)
except Exception as e:
# don't let parsing errors of the header hold us back
# header is not really used for anything important
header = {}
# Make a buffer of the default params in an easily parsable form
params, param_lst, patterns = collect_params(prmbuf)
parambuf = ' '.join(param_lst)
#print(parambuf)
# Parse default params into an ast.
(errors, ast_params, errinfo) = self.parse_params(parambuf)
#print("ast_params:", ast_params.printAST())
# This will hold the results
res = Bunch.Bunch(errors=errors, errinfo=errinfo, header=header)
# make readable errors
if errors > 0:
#print("ERRINFO = ", errinfo)
for errbnch in errinfo:
errbnch.verbose = sk_common.mk_error(parambuf, errbnch, 1)
# parse the command part
(errors, ast_cmds, errinfo) = self.parse(cmdbuf, startline=startline)
# Append errinfo together
res.errors += errors
res.errinfo.extend(errinfo)
# make readable errors
for errbnch in errinfo:
errbnch.verbose = sk_common.mk_error(cmdbuf, errbnch, 10)
res.params = params
res.patterns = patterns
# Finally, glue the params AST and the commands AST together to make
# "skeleton" node
res.ast = ASTNode("skeleton", ast_params, ast_cmds)
# return a bundle of these objects
return res
def p_sync(self, p):
"""sync : exec_command SEMICOLON
| abs_command SEMICOLON
| command_block SEMICOLON
| proc_call SEMICOLON
| while_loop SEMICOLON
| let_stmnt SEMICOLON
| catch SEMICOLON
"""
p[0] = ASTNode('sync', p[1])
def p_async(self, p):
"""async : exec_command COMMA
| abs_command COMMA
| command_block COMMA
| proc_call COMMA
| while_loop COMMA
| let_stmnt COMMA
| catch COMMA
"""
p[0] = ASTNode('async', p[1])
def _decode_merge(self, body_ast, eval):
new_ast = self.decode(body_ast, eval)
# if result is a command list, then mark it for merging into
# one large result
if isinstance(new_ast, ASTNode) and \
new_ast.tag == 'cmdlist':
new_ast = ASTNode('block_merge', *new_ast.items)
return new_ast
def __init__(self, evaluator, sk_bank, logger):
"""Decoder constructor. (params) is a dict of the initial
environment (variables & values) of the decoder. (sk_bank) is
used to lookup sk file ASTs and default parameters.
"""
# Evaluator for all external entities
self.eval = evaluator
# Object that lets me look up parsed abstract commands
self.sk_bank = sk_bank
self.logger = logger
self.nop = ASTNode('nop')
super(Decoder, self).__init__()
def p_special_form(self, p):
"""special_form : if_list
| star_if_list
| star_for_loop
| while_loop
| catch
| raise
| return
| star_set_stmnt
| let_stmnt
| set_stmnt
| proc_defn
| import_stmnt
"""
p[0] = ASTNode('cmdlist', p[1])
def p_dyad1(p):
"""dyad : expression MUL expression
| expression DIV expression
| expression ADD expression
| expression SUB expression
| expression LT expression
| expression GT expression
| expression LE expression
| expression GE expression
| expression EQ expression
| expression NE expression
| expression AND expression
| expression OR expression
"""
p[0] = ASTNode('dyad', p[1], p[2], p[3])
def decode_frame_id_ref(self, ast, eval):
"""Decode a frame allocation reference.
"""
assert ast.tag == 'frame_id_ref', ASTerr(ast)
# In SOSS, frame id allocations are done in decoding!
# uncomment this to reserve allocations until execution time
## res = eval.eval_string_interpolate(ast.items[0], vars_only=True)
## if isinstance(res, str):
## return ASTNode('frame_id_ref', res)
# comment this to reserve allocations until execution time
res = eval.eval(ast)
if isinstance(res, str):
return ASTNode('string', res)
name = ast.items[0]
raise DecodeError("Error decoding frame id reference: %s='%s' (%s)" %
(name, str(res), str(type(res))))
def _decode_string(self, ast, eval):
"""Decode a string interpolation.
"""
# cls = make_closure(ast, eval.clone())
# ast = ASTNode(ast.tag, ast.items[0])
# ast.cls = cls
# return ast
# Only expand variable references in strings in the decoding stage
res = eval.eval_string_interpolate(ast.items[0], vars_only=True)
if isinstance(res, str):
return ASTNode(ast.tag, res)
name = ast.items[0]
raise DecodeError("Don't know how to decode type: %s='%s' (%s)" %
(name, str(res), str(type(res))))
def decode_star_if(self, ast, eval):
"""Decode a *IF statement.
*IF <pred-exp>
<then-clause>
*ELIF <pred-exp>
<then-clause>
...
*ELSE
<else-clause>
*ENDIF
AST is a variable number of 'cond' ribs (if-exp, then-exp). Else
clause is represented by a final (True, then-exp) cond rib.
"""
assert ast.tag == 'star_if', ASTerr(ast)
# Iterate over the set of cond ribs, decode the first then-part
# for whose predicate evaluates to true.
for cond_ast in ast.items:
assert cond_ast.tag == 'cond', ASTerr(cond_ast)
assert len(cond_ast.items) == 2, ASTerr(cond_ast)
(pred_ast, then_ast) = cond_ast.items
if pred_ast == True:
# ELSE clause
return self._decode_merge(then_ast, eval)
# No longer. Could be dyad or monad...
#assert pred_ast.tag == 'expression', ASTerr(pred_ast)
res = eval.eval(pred_ast)
if eval.isTrue(res):
return self._decode_merge(then_ast, eval)
# See Note [1]
#return self.nop
return ASTNode('block_merge')
def parse_opecmd(self, buf, startline=1):
# Initialize module level error variables
self.reset(lineno=startline)
try:
ast = self.ope_parser.parse(buf, lexer=self.lexer)
except Exception as e:
# capture traceback? Yacc tracebacks aren't that useful
ast = ASTNode('ERROR: %s' % str(e))
# verify errors>0
assert (self.errors > 0)
try:
assert (ast.tag == 'cmdlist')
except AssertionError:
# ?? We're being silent like normal parsing
pass
return (self.errors, ast, self.errinfo)
def p_proc_defn1(self, p):
"""proc_defn : DEF ID LPAREN varlist RPAREN command_block"""
p[0] = ASTNode('proc', p[2], p[4], p[6])
def p_raise(self, p):
"""raise : RAISE expression"""
p[0] = ASTNode('raise', p[2])
def p_import(self, p):
"""import_stmnt : FROM QSTR IMPORT varlist"""
p[0] = ASTNode('import', p[2], p[4])
def p_ddcmd(self, p):
"""dd_cmd : EXEC factor factor param_list"""
p[0] = ASTNode('exec', p[2], p[3], p[4], None)
def p_varlist1(self, p):
"""varlist : ID"""
p[0] = ASTNode('varlist', p[1])
def p_idlist1(self, p):
"""idlist : expressions"""
p[0] = ASTNode('idlist', p[1])
def p_return2(self, p):
"""return : RETURN"""
p[0] = ASTNode('return')
def p_return(self, p):
"""return : RETURN expression"""
p[0] = ASTNode('return', p[2])
def p_catch1(self, p):
"""catch : CATCH ID command_block"""
p[0] = ASTNode('catch', p[2], p[3])
