def get_next_token(self): while self.current_char is not None: if self.current_char.isspace(): self.skip_whitespace() continue if self.current_char.isalpha(): return self._id() if self.current_char.isdigit(): return Token(TokenType.INTEGER, self.integer(), lineno=self.lineno, column=self.column) try: token_type = TokenType(self.current_char) except ValueError: self.error() else: token = Token(token_type=token_type, value=token_type.value, lineno=self.lineno, column=self.column) self.advance() print(token) return token return Token(TokenType.EOF, None)
class TestParser(unittest.TestCase): def test_empty(self): takens =[] node = Parser(tokens).parse() self.assertEqual(node,None) def test_numbers(self): takens=[Token(TokenType.NUMBER, 51.2)] node= Parser(tokens).parse() self.assertEqual(node, NumberNode(51.2)) def test_individual_operations(self): takens= [ Token(TokenType.NUMBER, 27), Token(TokenType.PLUS), Token(TokenType.NUMBER, 14), ] node = Parser(tokens).parse() self.assertEqual(node, AddNode(NumberNode(27),NumberNode(14))) takens= [ Token(TokenType.NUMBER, 27), Token(TokenType.MINUS), Token(TokenType.NUMBER, 14), ] node = Parser(tokens).parse() self.assertEqual(node, SubtractNode(NumberNode(27),NumberNode(14))) takens= [ Token(TokenType.NUMBER, 27), Token(TokenType.MULTIPLY), Token(TokenType.NUMBER, 14), ]
def test_InteriorNode_valueTest3(self): leaf1 = LeafNode(5) leaf2 = LeafNode(10) node1 = InteriorNode(Token('+'), leaf1, leaf2) node2 = InteriorNode(Token('*'), leaf1, node1) node3 = InteriorNode(Token('-'), node2, leaf2) self.assertEqual(node3.value(), 65)
def buildQuote(self): end_found = False # Create token for the open quote token = Token('QUOTE', self.code[self.cur_pos], self.line, self.col) self.__tokens.append(token) self.logToken(token) self.cur_pos += 1 self.col += 1 while not end_found: # get next char char = self.code[self.cur_pos] # Found closing quote if re.match(r'\"', char): token = Token('QUOTE', char, self.line, self.col) self.__tokens.append(token) self.logToken(token) end_found = True else: # Check foor invalid types! otherwise make a token for the valid chars if not re.match(r'^[a-z\s]$', char) or re.match(r'\n', char): self.logError( f'Character list contains invalid character: [ {repr(char)} ]. It can only contain lowercase letters and spaces.', self.line, self.col) end_found = True else: token = Token('CHAR', char, self.line, self.col) self.__tokens.append(token) self.logToken(token) self.cur_pos += 1 self.col += 1
def check_auth(username, password): """This function is called to check if a username / password combination is valid. """ if username == 'demouser': if password == 'demopass': request.user = username logging.info( 'Authenticated demouser with password {}'.format(password)) return True logging.info('Failed to authenticate demouser with password {}'.format( password)) elif username == 'PASSTOKEN': try: token = Token('password') decoded_token = token.decode_token(password) request.user = decoded_token['sub'] logging.info('Valid password token from {}'.format(request.user)) return True except jose.exceptions.JOSEError as e: logging.info('Password token error {}'.format(repr(e))) elif username == 'NOTIFICATION': if password == 'notipass': # if password == 'notipassword': request.user = username logging.info( 'Authenticated NOTIFICATION with password {}'.format(password)) return True logging.info( 'Failed to authenticate NOTIFICATION with password {}'.format( password)) return False
def wordTokenizer(text): foundedTokens = [] errorChars = [] pos = 0 tmp, remainText = textSpliter(text) while tmp: # print(f"'{tmp}'",pos) # input() for tokName, tokRegex in tokensMap.items(): if fullmatch(tokRegex, tmp): foundedTokens.append( Token(tokName, sub(persionSounds, "", tmp), tokRegex, pos=(pos, pos + (len(tmp))))) pos += len(tmp) tmp, remainText = textSpliter(remainText) break else: if len(tmp) != 1: try: remainText = tmp[-1] + remainText except: print("ERR", tmp) tmp = tmp[:-1] else: errorChars.append( Token("ERROR", tmp, "NO-REGEX-MATCHES", pos=(pos, pos + (len(tmp))))) pos += len(tmp) tmp, remainText = textSpliter(remainText) # continue return foundedTokens, errorChars
def __tokenize(self, string: str) -> Token: line_number = 1 position = 0 line_start = 0 match = get_token(string) while match is not None: match_type = match.lastgroup if match_type == TokenType.NEWLINE: line_start = position line_number += 1 elif match_type != TokenType.SKIP: value = match.group(match_type) if match_type == TokenType.ID and value in KEYWORDS: match_type = value elif match_type == TokenType.ID and value == '-': match_type = TokenType.LI elif match_type == TokenType.ID and value == ':': match_type = TokenType.ASSIGN yield Token(match_type, value, line_number, match.start() - line_start) position = match.end() match = get_token(string, position) if position != len(string): raise UnexpectedChar( f'Unexpected character {string[position]} in line {line_number}' ) yield Token(TokenType.EOF, '', line_number, str())
def parse_condition_expression(tokens): if len(tokens) == 0: return if tokens[0] == Token(TokenType.WORD, Word.ODD): expression = Parser.parse_expression(tokens[1:]) if not expression: raise TokenError(tokens.get(1, None)) return Condition(ConditionType.UNARY, (Word.ODD, expression)) condition_operators = [ BinaryOperator.EQUAL, BinaryOperator.HASHTAG, BinaryOperator.LESS, BinaryOperator.LESSEQUAL, BinaryOperator.GREATER, BinaryOperator.GREATEREQUAL ] separated_tokens = Parser.separate_tokens_with_operators( tokens, condition_operators) if separated_tokens: previous_tokens, operator, after_tokens = separated_tokens previous = Parser.parse_expression(previous_tokens) after = Parser.parse_expression(after_tokens) if not previous or not after: raise TokenError(Token(TokenType.OPERATOR, operator)) return Condition(ConditionType.BINARY, (previous, operator, after)) raise TokenError(tokens[1])
def test_double_equals(self): """Test tokenizing double equals.""" self.assertEqual(lexer.tokenize("a == 10", ""), [ Token(token_kinds.identifier, "a"), Token(token_kinds.twoequals), Token(token_kinds.number, "10") ])
def next_token(self): while self.ch!=0 and (self.ch.isspace() or is_comment_tag(self.ch)): if is_comment_tag(self.ch): self.read_comment() else: self.read_char() if self.ch!=0 and is_alpha(self.ch): literal = self.read_identifier() tok = tokens.token_alpha_dict.get(literal, Token(tokens.IDENT, literal)) elif self.ch!=0 and self.ch.isdigit(): literal = self.read_number() tok = Token(tokens.INT, literal) elif self.ch!=0 and self.ch == '"': tok = Token(tokens.STRING, self.read_string()) self.read_char() else: if self.ch in tokens.peeking: next_ch = self.peek_char() tok = tokens.token_ch2_dict.get(self.ch+str(next_ch)) if tok is not None: self.read_char() self.read_char() return tok tok = tokens.token_ch_dict.get(self.ch, Token(tokens.ILLEGAL, self.ch)) self.read_char() return tok
def test_numbers(self): tokens = [Token(TokenType.INT, 4)] node = Parser(tokens).parse() self.assertEqual(node, IntNode(4)) tokens = [Token(TokenType.FLOAT, 3.0)] node = Parser(tokens).parse() self.assertEqual(node, FloatNode(3.0))
def test_symbol_splits_keywords(self): """Test that the lexer splits on symbols.""" self.assertEqual(lexer.tokenize("ident1+ident2", ""), [ Token(token_kinds.identifier, "ident1"), Token(token_kinds.plus), Token(token_kinds.identifier, "ident2") ])
def add_block(block, tokens): """Convert block into a token if possible and add to tokens. If block is non-empty but cannot be made into a token, this function records a compiler error. We don't need to check for symbol kind tokens here because they are converted before they are shifted into the block. block - block to convert into a token, as list of Tagged characters. tokens (List[Token]) - List of the tokens so fat parsed. """ if block: range_ = Range(block[0].p, block[-1].p) keyword_kind = match_keyword_kind(block) if keyword_kind: tokens.append(Token(keyword_kind, r=range_)) return number_string = match_number_string(block) if number_string: tokens.append(Token(token_kinds.number, number_string, r=range_)) return identifier_name = match_identifier_name(block) if identifier_name: tokens.append(Token( token_kinds.identifier, identifier_name, r=range_)) return descr = f"unrecognized token at '{block_to_str(block)}'" raise CompilerError(descr, range_)
def test_full_expression(self): tokens =[ Token(TokenType.NUMBER, 27), Token(TokenType.PLUS), Token(TokenType.LPAREN), Token(TokenType.NUMBER, 43), Token(TokenType.DIVIDE), Token(TokenType.NUMBER, 36), Token(TokenType.MINUS), Token(TokenType.NUMBER, 48), Token(TokenType.MULTIPLY), Token(TokenType.NUMBER, 51), ] node = Parser(tokens).parse() self.assertEqual(node, AddNode( NumberNode(27), MultiplyNode( SubtractNode( DivideNode( NumberNode(43), NumberNode(36), ), NumberNode(48), ), NumberNode(51), ) ))
def __init__(self, lexer): self.lexer = lexer self.cur_token = Token() self.peek_token = Token() self.next_token() self.next_token() self.errors = [] self.prefix_parse_fns = {} self.infix_parse_fns = {} self.register_prefix(TokenType.Ident, self.parse_identifier) self.register_prefix(TokenType.Num, self.parse_integer_literal) self.register_prefix(TokenType.Bang, self.parse_prefix_expression) self.register_prefix(TokenType.Minus, self.parse_prefix_expression) self.register_prefix(TokenType.TRUE, self.parse_boolean) self.register_prefix(TokenType.FALSE, self.parse_boolean) self.register_prefix(TokenType.LParen, self.parse_grouped_expression) self.register_prefix(TokenType.If, self.parse_if_expression) self.register_prefix(TokenType.Fn, self.parse_function_literal) self.register_prefix(TokenType.String, self.parse_string_literal) self.register_prefix(TokenType.LBracket, self.parse_array_literal) self.register_prefix(TokenType.LBrace, self.parse_hash_literal) self.register_infix(TokenType.Plus, self.parse_infix_expression) self.register_infix(TokenType.Minus, self.parse_infix_expression) self.register_infix(TokenType.Star, self.parse_infix_expression) self.register_infix(TokenType.Slash, self.parse_infix_expression) self.register_infix(TokenType.Eq, self.parse_infix_expression) self.register_infix(TokenType.NotEq, self.parse_infix_expression) self.register_infix(TokenType.Less, self.parse_infix_expression) self.register_infix(TokenType.Greater, self.parse_infix_expression) self.register_infix(TokenType.LParen, self.parse_call_expression) self.register_infix(TokenType.LBracket, self.parse_index_expression)
def lexer(self): while self.current is not None: if self.current in (' ', '\r', '\t', '\b', '\n'): self.next() elif self.current.isdigit(): num = "" while (self.current is not None) and (self.current.isdigit() or self.current == "."): num += self.current self.next() yield Token("NUMBER", float(num)) elif self.current in ("+", "-", "*", "/"): operator = self.current self.next() yield Token(types[operator], operator) elif self.current in ("(", ")"): parenthesis = self.current self.next() yield Token(types[parenthesis], parenthesis) else: raise SyntaxError("Invalid Syntax")
def test_tokens(): lexer = Lexer("3+5") assert lexer.get_next_token() == Token(TokenType.INTEGER_CONST, 3) assert lexer.get_next_token() == Token(TokenType.PLUS, "+") assert lexer.get_next_token() == Token(TokenType.INTEGER_CONST, 5) assert lexer.get_next_token() == Token(TokenType.EOF, TokenType.EOF.value)
def test_nums(self): tokens = list(Lexer("111 235 2.1").generateTokens()) self.assertEqual(tokens, [ Token(TokenType.INT, 111), Token(TokenType.INT, 235), Token(TokenType.FLOAT, 2.1) ])
def evaluate(self): while self._scanner.hasNext(): currentToken = self._scanner.next() self._expressionSoFar += str(currentToken) + " " if currentToken.getType() == Token.INT: self._operandStack.push(currentToken) elif currentToken.isOperator(): if len(self._operandStack) < 2: raise Exception("Too few operands on the stack") t2 = self._operandStack.pop() t1 = self._operandStack.pop() try: result = Token( self._computeValue(currentToken, t1.getValue(), t2.getValue())) except Exception as error: return error else: self._operandStack.push(result) else: raise Exception("Unknown token type") if len(self._operandStack) > 1: raise Exception("Too many operands on the stack") result = self._operandStack.pop() return result.getValue()
def parse_expression(tokens): expressions = [] operators = [] def peak(stack): return stack[-1] if stack else None def apply_operator(): operator = operators.pop() right = expressions.pop() left = expressions.pop() expressions.append( Expression(ExpressionType.BINARY, (left, operator, right))) precedences = { BinaryOperator.PLUS: 1, BinaryOperator.MINUS: 1, BinaryOperator.TIMES: 2, BinaryOperator.SLASH: 2 } last_token_type = TokenType.OPERATOR for token in tokens: if last_token_type == token.type: raise TokenError(token) last_token_type = token.type if token.type == TokenType.NUMBER: expressions.append( Expression(ExpressionType.NUMBER, token.object)) elif token.type == TokenType.IDENTIFIER: expressions.append( Expression(ExpressionType.IDENTIFIER, token.object)) elif token == Token(TokenType.SIGN, Sign.LEFTPAREN): operators.append(token.object) elif token == Token(TokenType.SIGN, Sign.RIGHTPAREN): top = peak(operators) while top is not None and top != Sign.LEFTPAREN: apply_operator() top = peak(operators) operators.pop() else: if token.object not in { BinaryOperator.PLUS, BinaryOperator.MINUS, BinaryOperator.TIMES, BinaryOperator.SLASH }: raise TokenError(token, TokenType.OPERATOR) top = peak(operators) while top is not None and top not in {Sign.LEFTPAREN, Sign.RIGHTPAREN} and precedences[top] >= \ precedences[token.object]: apply_operator() top = peak(operators) operators.append(token.object) while peak(operators) is not None: apply_operator() return expressions[0] if expressions else None
def test_operators(): tokens = list(Lexer("+-*/").generate_tokens()) assert tokens == [ Token(TokenType.PLUS), Token(TokenType.MINUS), Token(TokenType.MULTIPLY), Token(TokenType.DIVIDE), ]
def get_token(cookie): """ Extracts the security token from the cookie """ token = Token.decode(cookie) if Token.is_valid(token): return token return None
def make_equ(self): self.advance() if self.current_char == '=' and self.current_char != None: self.advance() return self.tokens.append(Token(TYPE_EQUAL)) else: self.advance() return self.tokens.append(Token(TYPE_COLON))
def test_InteriorNode_prefixTest2(self): leaf1 = LeafNode(72) leaf2 = LeafNode(14) node1 = InteriorNode(Token('+'), leaf1, leaf2) node2 = InteriorNode(Token('*'), leaf1, node1) node3 = InteriorNode(Token('/'), node2, leaf2) s = "/ * 72 + 72 14 14" self.assertEqual(node3.prefix(), s)
def test_InteriorNode_postfixTest2(self): leaf1 = LeafNode(72) leaf2 = LeafNode(14) node1 = InteriorNode(Token('+'), leaf1, leaf2) node2 = InteriorNode(Token('*'), leaf1, node1) node3 = InteriorNode(Token('/'), node2, leaf2) s = "72 72 14 + * 14 /" self.assertEqual(node3.postfix(), s)
def final_html(subst_ast): """ Final html string is computed by lookin up the token,terminal reverse hash map. """ token = Token() #Init Token html_map = token.inverseMap() # Get Inverse mapping of tokens and html terminals html_final = html_Tree(subst_ast, html_map) # Compute Final html tree in a string type return html_final
def test_InteriorNode_infixTest1(self): leaf1 = LeafNode(5) leaf2 = LeafNode(10) node1 = InteriorNode(Token('+'), leaf1, leaf2) node2 = InteriorNode(Token('*'), leaf1, node1) node3 = InteriorNode(Token('-'), node2, leaf2) s = "((5 * (5 + 10)) - 10)" self.assertEqual(node3.infix(), s)
def test_operators(self): tokens = list(Lexer("+-*/").generate_tokens()) self.assertEqual(tokens, [ Token(TokenType.PLUS), Token(TokenType.MINUS), Token(TokenType.MULTIPLY), Token(TokenType.DIVIDE), ])
def test_operatorler(self): tokens = list(Lexer("+-*/").token_olustur()) self.assertEqual(tokens, [ Token(TokenTip.ARTİ), Token(TokenTip.EKSİ), Token(TokenTip.CARP), Token(TokenTip.BOL), ])
def _get_next_token(self): self._skip_white_space() if self._current_char.isdigit(): self._current_token = Token(self._getInteger()) elif self._current_char == Scanner.EOE: self._current_token = None else: self._current_token = Token(self._current_char) self._next_char()
def test_numbers(self): tokens = list(Lexer("123 123.456 123. .456 .").generate_tokens()) self.assertEqual(tokens, [ Token(TokenType.NUMBER, 123.000), Token(TokenType.NUMBER, 123.456), Token(TokenType.NUMBER, 123.000), Token(TokenType.NUMBER, 000.456), Token(TokenType.NUMBER, 000.000), ])
def computeTokens(self, stream, seen, tokens): """Returns list of tokens, by iteratively walking over the input stream and checking for delimeters.""" myToken = Token() for char in stream: if self.isValidTag(char, seen): continue #ignore white space characters seen += char if myToken.isTag(seen): token = myToken.getToken(seen) tokens.append(token) seen = "" return tokens
def evaluate(self): """Returns the value of the postfix expression.""" for currentToken in self._scanner: if currentToken.getType() == Token.INT: self._operandStack.push(currentToken) elif currentToken.isOperator(): right = self._operandStack.pop() left = self._operandStack.pop() result = Token(self._computeValue(currentToken, left.getValue(), right.getValue())) self._operandStack.push(result) result = self._operandStack.pop() return result.getValue();
def tokens(): service = request.args.get('service') scope = request.args.get('scope') if not scope: typ = '' name = '' actions = [] else: params = scope.split(':') if len(params) != 3: return jsonify(error='Invalid scope parameter'), 400 typ = params[0] name = params[1] actions = params[2].split(',') authorized_actions = get_allowed_actions(request.user, actions) token = Token(service, typ, name, authorized_actions) encoded_token = token.encode_token() return jsonify(token=encoded_token)
def read_identifier(self): t = Token(kind='identifier', value='') while not self.strm.eof(): c = self.strm.get() if not self.is_identifier_char(c): self.strm.prev() break else: t.value += c if not len(t.value): raise Tokenizer.ModuleError('not found identifier') # impossible if t.value == 'import': t.kind = 'import' elif t.value == 'if': t.kind = 'if' elif t.value == 'elif': t.kind = 'elif' elif t.value == 'else': t.kind = 'else' elif t.value == 'end': t.kind = 'end' self.tokens.append(t)
def evaluate(self): while self._scanner.hasNext(): currentToken = self._scanner.next() self._expressionSoFar += str(currentToken) + " " if currentToken.getType() == Token.INT: self._operandStack.push(currentToken) elif currentToken.isOperator(): if len(self._operandStack) < 2: raise AttributeError("Too few operands on the stack") t2 = self._operandStack.pop() t1 = self._operandStack.pop() result = Token(self._computeValue(currentToken, t1.getValue(), t2.getValue())) self._operandStack.push(result) else: raise AttributeError("Unknown token type") if len(self._operandStack) > 1: raise AttributeError("Too many operands on the stack") result = self._operandStack.pop() return result.getValue();