def get_next_token(self):
while self.current_char is not None:
if self.current_char.isspace():
self.skip_whitespace()
continue
if self.current_char.isalpha():
return self._id()
if self.current_char.isdigit():
return Token(TokenType.INTEGER,
self.integer(),
lineno=self.lineno,
column=self.column)
try:
token_type = TokenType(self.current_char)
except ValueError:
self.error()
else:
token = Token(token_type=token_type,
value=token_type.value,
lineno=self.lineno,
column=self.column)
self.advance()
print(token)
return token
return Token(TokenType.EOF, None)
class TestParser(unittest.TestCase):
def test_empty(self):
takens =[]
node = Parser(tokens).parse()
self.assertEqual(node,None)
def test_numbers(self):
takens=[Token(TokenType.NUMBER, 51.2)]
node= Parser(tokens).parse()
self.assertEqual(node, NumberNode(51.2))
def test_individual_operations(self):
takens= [
Token(TokenType.NUMBER, 27),
Token(TokenType.PLUS),
Token(TokenType.NUMBER, 14),
]
node = Parser(tokens).parse()
self.assertEqual(node, AddNode(NumberNode(27),NumberNode(14)))
takens= [
Token(TokenType.NUMBER, 27),
Token(TokenType.MINUS),
Token(TokenType.NUMBER, 14),
]
node = Parser(tokens).parse()
self.assertEqual(node, SubtractNode(NumberNode(27),NumberNode(14)))
takens= [
Token(TokenType.NUMBER, 27),
Token(TokenType.MULTIPLY),
Token(TokenType.NUMBER, 14),
]
def test_InteriorNode_valueTest3(self):
leaf1 = LeafNode(5)
leaf2 = LeafNode(10)
node1 = InteriorNode(Token('+'), leaf1, leaf2)
node2 = InteriorNode(Token('*'), leaf1, node1)
node3 = InteriorNode(Token('-'), node2, leaf2)
self.assertEqual(node3.value(), 65)
def buildQuote(self):
end_found = False
# Create token for the open quote
token = Token('QUOTE', self.code[self.cur_pos], self.line, self.col)
self.__tokens.append(token)
self.logToken(token)
self.cur_pos += 1
self.col += 1
while not end_found:
# get next char
char = self.code[self.cur_pos]
# Found closing quote
if re.match(r'\"', char):
token = Token('QUOTE', char, self.line, self.col)
self.__tokens.append(token)
self.logToken(token)
end_found = True
else:
# Check foor invalid types! otherwise make a token for the valid chars
if not re.match(r'^[a-z\s]$', char) or re.match(r'\n', char):
self.logError(
f'Character list contains invalid character: [ {repr(char)} ]. It can only contain lowercase letters and spaces.',
self.line, self.col)
end_found = True
else:
token = Token('CHAR', char, self.line, self.col)
self.__tokens.append(token)
self.logToken(token)
self.cur_pos += 1
self.col += 1
def check_auth(username, password):
"""This function is called to check if a username /
password combination is valid.
"""
if username == 'demouser':
if password == 'demopass':
request.user = username
logging.info(
'Authenticated demouser with password {}'.format(password))
return True
logging.info('Failed to authenticate demouser with password {}'.format(
password))
elif username == 'PASSTOKEN':
try:
token = Token('password')
decoded_token = token.decode_token(password)
request.user = decoded_token['sub']
logging.info('Valid password token from {}'.format(request.user))
return True
except jose.exceptions.JOSEError as e:
logging.info('Password token error {}'.format(repr(e)))
elif username == 'NOTIFICATION':
if password == 'notipass':
# if password == 'notipassword':
request.user = username
logging.info(
'Authenticated NOTIFICATION with password {}'.format(password))
return True
logging.info(
'Failed to authenticate NOTIFICATION with password {}'.format(
password))
return False
def wordTokenizer(text):
foundedTokens = []
errorChars = []
pos = 0
tmp, remainText = textSpliter(text)
while tmp:
# print(f"'{tmp}'",pos)
# input()
for tokName, tokRegex in tokensMap.items():
if fullmatch(tokRegex, tmp):
foundedTokens.append(
Token(tokName,
sub(persionSounds, "", tmp),
tokRegex,
pos=(pos, pos + (len(tmp)))))
pos += len(tmp)
tmp, remainText = textSpliter(remainText)
break
else:
if len(tmp) != 1:
try:
remainText = tmp[-1] + remainText
except:
print("ERR", tmp)
tmp = tmp[:-1]
else:
errorChars.append(
Token("ERROR",
tmp,
"NO-REGEX-MATCHES",
pos=(pos, pos + (len(tmp)))))
pos += len(tmp)
tmp, remainText = textSpliter(remainText)
# continue
return foundedTokens, errorChars
def __tokenize(self, string: str) -> Token:
line_number = 1
position = 0
line_start = 0
match = get_token(string)
while match is not None:
match_type = match.lastgroup
if match_type == TokenType.NEWLINE:
line_start = position
line_number += 1
elif match_type != TokenType.SKIP:
value = match.group(match_type)
if match_type == TokenType.ID and value in KEYWORDS:
match_type = value
elif match_type == TokenType.ID and value == '-':
match_type = TokenType.LI
elif match_type == TokenType.ID and value == ':':
match_type = TokenType.ASSIGN
yield Token(match_type, value, line_number,
match.start() - line_start)
position = match.end()
match = get_token(string, position)
if position != len(string):
raise UnexpectedChar(
f'Unexpected character {string[position]} in line {line_number}'
)
yield Token(TokenType.EOF, '', line_number, str())
def parse_condition_expression(tokens):
if len(tokens) == 0:
return
if tokens[0] == Token(TokenType.WORD, Word.ODD):
expression = Parser.parse_expression(tokens[1:])
if not expression:
raise TokenError(tokens.get(1, None))
return Condition(ConditionType.UNARY, (Word.ODD, expression))
condition_operators = [
BinaryOperator.EQUAL, BinaryOperator.HASHTAG, BinaryOperator.LESS,
BinaryOperator.LESSEQUAL, BinaryOperator.GREATER,
BinaryOperator.GREATEREQUAL
]
separated_tokens = Parser.separate_tokens_with_operators(
tokens, condition_operators)
if separated_tokens:
previous_tokens, operator, after_tokens = separated_tokens
previous = Parser.parse_expression(previous_tokens)
after = Parser.parse_expression(after_tokens)
if not previous or not after:
raise TokenError(Token(TokenType.OPERATOR, operator))
return Condition(ConditionType.BINARY, (previous, operator, after))
raise TokenError(tokens[1])
def test_double_equals(self):
"""Test tokenizing double equals."""
self.assertEqual(lexer.tokenize("a == 10", ""), [
Token(token_kinds.identifier, "a"),
Token(token_kinds.twoequals),
Token(token_kinds.number, "10")
])
def next_token(self):
while self.ch!=0 and (self.ch.isspace() or is_comment_tag(self.ch)):
if is_comment_tag(self.ch):
self.read_comment()
else:
self.read_char()
if self.ch!=0 and is_alpha(self.ch):
literal = self.read_identifier()
tok = tokens.token_alpha_dict.get(literal, Token(tokens.IDENT, literal))
elif self.ch!=0 and self.ch.isdigit():
literal = self.read_number()
tok = Token(tokens.INT, literal)
elif self.ch!=0 and self.ch == '"':
tok = Token(tokens.STRING, self.read_string())
self.read_char()
else:
if self.ch in tokens.peeking:
next_ch = self.peek_char()
tok = tokens.token_ch2_dict.get(self.ch+str(next_ch))
if tok is not None:
self.read_char()
self.read_char()
return tok
tok = tokens.token_ch_dict.get(self.ch, Token(tokens.ILLEGAL, self.ch))
self.read_char()
return tok
def test_numbers(self):
tokens = [Token(TokenType.INT, 4)]
node = Parser(tokens).parse()
self.assertEqual(node, IntNode(4))
tokens = [Token(TokenType.FLOAT, 3.0)]
node = Parser(tokens).parse()
self.assertEqual(node, FloatNode(3.0))
def test_symbol_splits_keywords(self):
"""Test that the lexer splits on symbols."""
self.assertEqual(lexer.tokenize("ident1+ident2", ""), [
Token(token_kinds.identifier, "ident1"),
Token(token_kinds.plus),
Token(token_kinds.identifier, "ident2")
])
def add_block(block, tokens):
"""Convert block into a token if possible and add to tokens. If block is non-empty but cannot be made into a token,
this function records a compiler error. We don't need to check for symbol kind tokens here because they are
converted before they are shifted into the block.
block - block to convert into a token, as list of Tagged characters.
tokens (List[Token]) - List of the tokens so fat parsed.
"""
if block:
range_ = Range(block[0].p, block[-1].p)
keyword_kind = match_keyword_kind(block)
if keyword_kind:
tokens.append(Token(keyword_kind, r=range_))
return
number_string = match_number_string(block)
if number_string:
tokens.append(Token(token_kinds.number, number_string, r=range_))
return
identifier_name = match_identifier_name(block)
if identifier_name:
tokens.append(Token(
token_kinds.identifier, identifier_name, r=range_))
return
descr = f"unrecognized token at '{block_to_str(block)}'"
raise CompilerError(descr, range_)
def test_full_expression(self):
tokens =[
Token(TokenType.NUMBER, 27),
Token(TokenType.PLUS),
Token(TokenType.LPAREN),
Token(TokenType.NUMBER, 43),
Token(TokenType.DIVIDE),
Token(TokenType.NUMBER, 36),
Token(TokenType.MINUS),
Token(TokenType.NUMBER, 48),
Token(TokenType.MULTIPLY),
Token(TokenType.NUMBER, 51),
]
node = Parser(tokens).parse()
self.assertEqual(node, AddNode(
NumberNode(27),
MultiplyNode(
SubtractNode(
DivideNode(
NumberNode(43),
NumberNode(36),
),
NumberNode(48),
),
NumberNode(51),
)
))
def __init__(self, lexer):
self.lexer = lexer
self.cur_token = Token()
self.peek_token = Token()
self.next_token()
self.next_token()
self.errors = []
self.prefix_parse_fns = {}
self.infix_parse_fns = {}
self.register_prefix(TokenType.Ident, self.parse_identifier)
self.register_prefix(TokenType.Num, self.parse_integer_literal)
self.register_prefix(TokenType.Bang, self.parse_prefix_expression)
self.register_prefix(TokenType.Minus, self.parse_prefix_expression)
self.register_prefix(TokenType.TRUE, self.parse_boolean)
self.register_prefix(TokenType.FALSE, self.parse_boolean)
self.register_prefix(TokenType.LParen, self.parse_grouped_expression)
self.register_prefix(TokenType.If, self.parse_if_expression)
self.register_prefix(TokenType.Fn, self.parse_function_literal)
self.register_prefix(TokenType.String, self.parse_string_literal)
self.register_prefix(TokenType.LBracket, self.parse_array_literal)
self.register_prefix(TokenType.LBrace, self.parse_hash_literal)
self.register_infix(TokenType.Plus, self.parse_infix_expression)
self.register_infix(TokenType.Minus, self.parse_infix_expression)
self.register_infix(TokenType.Star, self.parse_infix_expression)
self.register_infix(TokenType.Slash, self.parse_infix_expression)
self.register_infix(TokenType.Eq, self.parse_infix_expression)
self.register_infix(TokenType.NotEq, self.parse_infix_expression)
self.register_infix(TokenType.Less, self.parse_infix_expression)
self.register_infix(TokenType.Greater, self.parse_infix_expression)
self.register_infix(TokenType.LParen, self.parse_call_expression)
self.register_infix(TokenType.LBracket, self.parse_index_expression)
def lexer(self):
while self.current is not None:
if self.current in (' ', '\r', '\t', '\b', '\n'):
self.next()
elif self.current.isdigit():
num = ""
while (self.current is not None) and (self.current.isdigit()
or self.current == "."):
num += self.current
self.next()
yield Token("NUMBER", float(num))
elif self.current in ("+", "-", "*", "/"):
operator = self.current
self.next()
yield Token(types[operator], operator)
elif self.current in ("(", ")"):
parenthesis = self.current
self.next()
yield Token(types[parenthesis], parenthesis)
else:
raise SyntaxError("Invalid Syntax")
def test_tokens():
lexer = Lexer("3+5")
assert lexer.get_next_token() == Token(TokenType.INTEGER_CONST, 3)
assert lexer.get_next_token() == Token(TokenType.PLUS, "+")
assert lexer.get_next_token() == Token(TokenType.INTEGER_CONST, 5)
assert lexer.get_next_token() == Token(TokenType.EOF, TokenType.EOF.value)
def test_nums(self):
tokens = list(Lexer("111 235 2.1").generateTokens())
self.assertEqual(tokens, [
Token(TokenType.INT, 111),
Token(TokenType.INT, 235),
Token(TokenType.FLOAT, 2.1)
])
def evaluate(self):
while self._scanner.hasNext():
currentToken = self._scanner.next()
self._expressionSoFar += str(currentToken) + " "
if currentToken.getType() == Token.INT:
self._operandStack.push(currentToken)
elif currentToken.isOperator():
if len(self._operandStack) < 2:
raise Exception("Too few operands on the stack")
t2 = self._operandStack.pop()
t1 = self._operandStack.pop()
try:
result = Token(
self._computeValue(currentToken, t1.getValue(),
t2.getValue()))
except Exception as error:
return error
else:
self._operandStack.push(result)
else:
raise Exception("Unknown token type")
if len(self._operandStack) > 1:
raise Exception("Too many operands on the stack")
result = self._operandStack.pop()
return result.getValue()
def parse_expression(tokens):
expressions = []
operators = []
def peak(stack):
return stack[-1] if stack else None
def apply_operator():
operator = operators.pop()
right = expressions.pop()
left = expressions.pop()
expressions.append(
Expression(ExpressionType.BINARY, (left, operator, right)))
precedences = {
BinaryOperator.PLUS: 1,
BinaryOperator.MINUS: 1,
BinaryOperator.TIMES: 2,
BinaryOperator.SLASH: 2
}
last_token_type = TokenType.OPERATOR
for token in tokens:
if last_token_type == token.type:
raise TokenError(token)
last_token_type = token.type
if token.type == TokenType.NUMBER:
expressions.append(
Expression(ExpressionType.NUMBER, token.object))
elif token.type == TokenType.IDENTIFIER:
expressions.append(
Expression(ExpressionType.IDENTIFIER, token.object))
elif token == Token(TokenType.SIGN, Sign.LEFTPAREN):
operators.append(token.object)
elif token == Token(TokenType.SIGN, Sign.RIGHTPAREN):
top = peak(operators)
while top is not None and top != Sign.LEFTPAREN:
apply_operator()
top = peak(operators)
operators.pop()
else:
if token.object not in {
BinaryOperator.PLUS, BinaryOperator.MINUS,
BinaryOperator.TIMES, BinaryOperator.SLASH
}:
raise TokenError(token, TokenType.OPERATOR)
top = peak(operators)
while top is not None and top not in {Sign.LEFTPAREN, Sign.RIGHTPAREN} and precedences[top] >= \
precedences[token.object]:
apply_operator()
top = peak(operators)
operators.append(token.object)
while peak(operators) is not None:
apply_operator()
return expressions[0] if expressions else None
def test_operators():
tokens = list(Lexer("+-*/").generate_tokens())
assert tokens == [
Token(TokenType.PLUS),
Token(TokenType.MINUS),
Token(TokenType.MULTIPLY),
Token(TokenType.DIVIDE),
]
def get_token(cookie):
"""
Extracts the security token from the cookie
"""
token = Token.decode(cookie)
if Token.is_valid(token):
return token
return None
def make_equ(self):
self.advance()
if self.current_char == '=' and self.current_char != None:
self.advance()
return self.tokens.append(Token(TYPE_EQUAL))
else:
self.advance()
return self.tokens.append(Token(TYPE_COLON))
def test_InteriorNode_prefixTest2(self):
leaf1 = LeafNode(72)
leaf2 = LeafNode(14)
node1 = InteriorNode(Token('+'), leaf1, leaf2)
node2 = InteriorNode(Token('*'), leaf1, node1)
node3 = InteriorNode(Token('/'), node2, leaf2)
s = "/ * 72 + 72 14 14"
self.assertEqual(node3.prefix(), s)
def test_InteriorNode_postfixTest2(self):
leaf1 = LeafNode(72)
leaf2 = LeafNode(14)
node1 = InteriorNode(Token('+'), leaf1, leaf2)
node2 = InteriorNode(Token('*'), leaf1, node1)
node3 = InteriorNode(Token('/'), node2, leaf2)
s = "72 72 14 + * 14 /"
self.assertEqual(node3.postfix(), s)
def final_html(subst_ast):
""" Final html string is computed by lookin up the
token,terminal reverse hash map.
"""
token = Token() #Init Token
html_map = token.inverseMap() # Get Inverse mapping of tokens and html terminals
html_final = html_Tree(subst_ast, html_map) # Compute Final html tree in a string type
return html_final
def test_InteriorNode_infixTest1(self):
leaf1 = LeafNode(5)
leaf2 = LeafNode(10)
node1 = InteriorNode(Token('+'), leaf1, leaf2)
node2 = InteriorNode(Token('*'), leaf1, node1)
node3 = InteriorNode(Token('-'), node2, leaf2)
s = "((5 * (5 + 10)) - 10)"
self.assertEqual(node3.infix(), s)
def test_operators(self):
tokens = list(Lexer("+-*/").generate_tokens())
self.assertEqual(tokens, [
Token(TokenType.PLUS),
Token(TokenType.MINUS),
Token(TokenType.MULTIPLY),
Token(TokenType.DIVIDE),
])
def test_operatorler(self):
tokens = list(Lexer("+-*/").token_olustur())
self.assertEqual(tokens, [
Token(TokenTip.ARTİ),
Token(TokenTip.EKSİ),
Token(TokenTip.CARP),
Token(TokenTip.BOL),
])
def _get_next_token(self):
self._skip_white_space()
if self._current_char.isdigit():
self._current_token = Token(self._getInteger())
elif self._current_char == Scanner.EOE:
self._current_token = None
else:
self._current_token = Token(self._current_char)
self._next_char()
def test_numbers(self):
tokens = list(Lexer("123 123.456 123. .456 .").generate_tokens())
self.assertEqual(tokens, [
Token(TokenType.NUMBER, 123.000),
Token(TokenType.NUMBER, 123.456),
Token(TokenType.NUMBER, 123.000),
Token(TokenType.NUMBER, 000.456),
Token(TokenType.NUMBER, 000.000),
])
def computeTokens(self, stream, seen, tokens):
"""Returns list of tokens, by iteratively walking over the
input stream and checking for delimeters."""
myToken = Token()
for char in stream:
if self.isValidTag(char, seen):
continue #ignore white space characters
seen += char
if myToken.isTag(seen):
token = myToken.getToken(seen)
tokens.append(token)
seen = ""
return tokens
def evaluate(self):
"""Returns the value of the postfix expression."""
for currentToken in self._scanner:
if currentToken.getType() == Token.INT:
self._operandStack.push(currentToken)
elif currentToken.isOperator():
right = self._operandStack.pop()
left = self._operandStack.pop()
result = Token(self._computeValue(currentToken,
left.getValue(),
right.getValue()))
self._operandStack.push(result)
result = self._operandStack.pop()
return result.getValue();
def tokens():
service = request.args.get('service')
scope = request.args.get('scope')
if not scope:
typ = ''
name = ''
actions = []
else:
params = scope.split(':')
if len(params) != 3:
return jsonify(error='Invalid scope parameter'), 400
typ = params[0]
name = params[1]
actions = params[2].split(',')
authorized_actions = get_allowed_actions(request.user, actions)
token = Token(service, typ, name, authorized_actions)
encoded_token = token.encode_token()
return jsonify(token=encoded_token)
def read_identifier(self):
t = Token(kind='identifier', value='')
while not self.strm.eof():
c = self.strm.get()
if not self.is_identifier_char(c):
self.strm.prev()
break
else:
t.value += c
if not len(t.value):
raise Tokenizer.ModuleError('not found identifier') # impossible
if t.value == 'import':
t.kind = 'import'
elif t.value == 'if':
t.kind = 'if'
elif t.value == 'elif':
t.kind = 'elif'
elif t.value == 'else':
t.kind = 'else'
elif t.value == 'end':
t.kind = 'end'
self.tokens.append(t)
def evaluate(self):
while self._scanner.hasNext():
currentToken = self._scanner.next()
self._expressionSoFar += str(currentToken) + " "
if currentToken.getType() == Token.INT:
self._operandStack.push(currentToken)
elif currentToken.isOperator():
if len(self._operandStack) < 2:
raise AttributeError("Too few operands on the stack")
t2 = self._operandStack.pop()
t1 = self._operandStack.pop()
result = Token(self._computeValue(currentToken,
t1.getValue(),
t2.getValue()))
self._operandStack.push(result)
else:
raise AttributeError("Unknown token type")
if len(self._operandStack) > 1:
raise AttributeError("Too many operands on the stack")
result = self._operandStack.pop()
return result.getValue();
