def require(self,tok,msg=''): ''' Like skip but prints a warning if said token wasn't found. msg can detail why that particular token is needed. ''' if self.skip(tok): return True else: print('{0}: WARNING, {1} expected {3}, got {2}.'.format(self.position(), tokens.prettyTok(tok), tokens.prettyTok(self.peek()),msg)) return False
def readfile(fn,context): included.append(fn) print('Reading {0}...'.format(fn)) try: tokenstream = iter(scanner.tokenstream(fn)) while True: readcode(tokenstream,context) if tokenstream.peek() != (tokens.EndOfFile,): print('UNEXPECTED {0}'.format(tokens.prettyTok(next(tokenstream)))) else: break except IOError: print('WARNING: problem reading {0}, giving up'.format(fn)) return
def readexpr(tokstream,required): ''' Parses an expression from the given token stream, an instance of tokenstream. Returns None if the current point doesn't look like an expression. Set required to true if absence of expression should fail dramatically, otherwise you'll just get None if the stream doesn't start like an expression. ''' tok = tokstream.peek() if not tok: if required: print('Expression expected, got End of file') return None if tok[0] == tokens.OpeningBracket: next(tokstream) # Check if this is a cast tok = tokstream.peek() if tok and tok[0] == tokens.Type: t = next(tokstream) tokstream.require((tokens.ClosingBracket,),msg='to close cast brackets') currentexpr = (t,readexpr(tokstream,required=True)) else: currentexpr = readexpr(tokstream,required=True) tokstream.require((tokens.ClosingBracket,),msg='after expression {0}'.format(prettyexpr(currentexpr))) elif tok[0] in tokens.atoms: next(tokstream) currentexpr = (tok,) elif tok[0] in [tokens.BuiltinFunction, tokens.FunctionName]: next(tokstream) currentexpr = [(tokens.FunctionCall,), tok] # a list, to permit adding parameters one at a time # Having brackets and a parameter list is optional (because tok could be a *constant*, but now we can't make the difference between f() and f. Should really have a [list] if (tokstream.skip((tokens.OpeningBracket,))): currentexpr.extend(readexprseq(tokstream)) tokstream.require((tokens.ClosingBracket,),msg='to close {0} function call'.format(tok[1])) # else: # print('Warning: is {0} a constant or a keyword?'.format(tok[1])) currentexpr = tuple(currentexpr) elif tok[0] in tokens.unaryoperators: currentexpr = (next(tokstream),readexpr(tokstream,required=True)) else: if required: print('{0}: WARNING: Expression expected, got {1}.'.format(tokstream.position(),tokens.prettyTok(tok))) return None while True: # currentexpr contains something, see if it is the first element of a binary/ternary expression... tok = tokstream.peek() if tok[0] == tokens.OpeningSquareBracket: next(tokstream) t = [(tokens.ArrayAccess,),currentexpr] t.extend(readexprseq(tokstream)) currentexpr = tuple(t) tokstream.require((tokens.ClosingSquareBracket,)) elif tok[0] == tokens.OpeningCurly: # As in: $string{characternumber} next(tokstream) t = [(tokens.ArrayAccess,),currentexpr] t.extend(readexprseq(tokstream)) currentexpr = tuple(t) tokstream.require((tokens.ClosingCurly,),msg='for closing string character reference') elif tok[0] in tokens.postfixoperators: currentexpr = (next(tokstream), currentexpr) elif tok[0] in tokens.binaryoperators: currentexpr = (next(tokstream), currentexpr, readexpr(tokstream,required=True)) elif tok[0] == tokens.Question: next(tokstream) t = readexpr(tokstream,required=True) tokstream.require((tokens.Colon,)) currentexpr = (tok, currentexpr, t, readexpr(tokstream,required=True)) else: return currentexpr