def test_vector_transposition(self):
lexer = Lexer('(1)**T')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENP, '('))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.TPOSE, '**T'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_real_no_leading_zeros(self):
lexer = Lexer('0.00 + 00.0')
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, 0.00))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 0))
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, 0.0))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_no_leading_zeros(self):
lexer = Lexer('0 + 00')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 0))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 0))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 0))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_set_cardinality(self):
lexer = Lexer('#{1}')
self.assertEqual(lexer.get_next_token(), Token(Type.CARD, '#'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_real_must_have_fraction_again(self):
lexer = Lexer('1.0 + 1.')
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, 1.0))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.PERIOD, '.'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_set_xor(self):
lexer = Lexer('{1} ^ {1}')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.XOR, '^'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_set_and(self):
lexer = Lexer('{1} & {1}')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.AND, '&'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_set_union(self):
lexer = Lexer('{1} + {1}')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENC, '{'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEC, '}'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_list_union(self):
lexer = Lexer('[1] + [1]')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENB, '['))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEB, ']'))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENB, '['))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEB, ']'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_tuple_union(self):
lexer = Lexer('(1) + (1)')
self.assertEqual(lexer.get_next_token(), Token(Type.OPENP, '('))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENP, '('))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_assignment_expression(self):
lexer = Lexer('let x of Z := 5.')
self.assertEqual(lexer.get_next_token(), Token(Type.LET, 'let'))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'x'))
self.assertEqual(lexer.get_next_token(), Token(Type.OF, 'of'))
self.assertEqual(lexer.get_next_token(), Token(Type.Z, 'Z'))
self.assertEqual(lexer.get_next_token(), Token(Type.DEFAS, ':='))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 5))
self.assertEqual(lexer.get_next_token(), Token(Type.PERIOD, '.'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_vector_norm(self):
lexer = Lexer('|| (1) ||')
self.assertEqual(lexer.get_next_token(), Token(Type.NORM, '||'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENP, '('))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.NORM, '||'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_reserved_values(self):
lexer = Lexer('e false null pi true')
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, math.e))
self.assertEqual(lexer.get_next_token(), Token(Type.BOOL, False))
self.assertEqual(lexer.get_next_token(), Token(Type.NULL, None))
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, math.pi))
self.assertEqual(lexer.get_next_token(), Token(Type.BOOL, True))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_absolute_value(self):
lexer = Lexer('| 1 |')
self.assertEqual(lexer.get_next_token(), Token(Type.PIPE, '|'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.PIPE, '|'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_difference(self):
lexer = Lexer('1 - 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.DIFF, '-'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_fact(self):
lexer = Lexer('1!')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.FACT, '!'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_union(self):
lexer = Lexer('1 + 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_modulus(self):
lexer = Lexer('1 % 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.MOD, '%'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_less_than_equal(self):
lexer = Lexer('1 <= 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.LTEQ, '<='))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_step_interval(self):
lexer = Lexer('forall x in [1,..1..,#X)')
self.assertEqual(lexer.get_next_token(), Token(Type.FORALL, 'forall'))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'x'))
self.assertEqual(lexer.get_next_token(), Token(Type.IN, 'in'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENB, '['))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.COMMA, ','))
self.assertEqual(lexer.get_next_token(), Token(Type.RANGE, '..'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.RANGE, '..'))
self.assertEqual(lexer.get_next_token(), Token(Type.COMMA, ','))
self.assertEqual(lexer.get_next_token(), Token(Type.CARD, '#'))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'X'))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_function_declaration(self):
lexer = Lexer('let f: Z -> Z, f(x) := 5x.')
self.assertEqual(lexer.get_next_token(), Token(Type.LET, 'let'))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'f'))
self.assertEqual(lexer.get_next_token(), Token(Type.COLON, ':'))
self.assertEqual(lexer.get_next_token(), Token(Type.Z, 'Z'))
self.assertEqual(lexer.get_next_token(), Token(Type.TO, '->'))
self.assertEqual(lexer.get_next_token(), Token(Type.Z, 'Z'))
self.assertEqual(lexer.get_next_token(), Token(Type.COMMA, ','))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'f'))
self.assertEqual(lexer.get_next_token(), Token(Type.OPENP, '('))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'x'))
self.assertEqual(lexer.get_next_token(), Token(Type.CLOSEP, ')'))
self.assertEqual(lexer.get_next_token(), Token(Type.DEFAS, ':='))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 5))
self.assertEqual(lexer.get_next_token(), Token(Type.ID, 'x'))
self.assertEqual(lexer.get_next_token(), Token(Type.PERIOD, '.'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_pow(self):
lexer = Lexer('1**1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.POW, '**'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_reserved_types(self):
lexer = Lexer('B Boolean Integer N Natural Q Rational R Real S String U Universal Z')
self.assertEqual(lexer.get_next_token(), Token(Type.B, 'B'))
self.assertEqual(lexer.get_next_token(), Token(Type.B, 'B'))
self.assertEqual(lexer.get_next_token(), Token(Type.Z, 'Z'))
self.assertEqual(lexer.get_next_token(), Token(Type.N, 'N'))
self.assertEqual(lexer.get_next_token(), Token(Type.N, 'N'))
self.assertEqual(lexer.get_next_token(), Token(Type.Q, 'Q'))
self.assertEqual(lexer.get_next_token(), Token(Type.Q, 'Q'))
self.assertEqual(lexer.get_next_token(), Token(Type.R, 'R'))
self.assertEqual(lexer.get_next_token(), Token(Type.R, 'R'))
self.assertEqual(lexer.get_next_token(), Token(Type.S, 'S'))
self.assertEqual(lexer.get_next_token(), Token(Type.S, 'S'))
self.assertEqual(lexer.get_next_token(), Token(Type.U, 'U'))
self.assertEqual(lexer.get_next_token(), Token(Type.U, 'U'))
self.assertEqual(lexer.get_next_token(), Token(Type.Z, 'Z'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_greater_than_equal(self):
lexer = Lexer('1 >= 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.GTEQ, '>='))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_real_union(self):
lexer = Lexer('3.0 + 2.5')
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, 3.0))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.REAL, 2.5))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_composition(self):
lexer = Lexer('1 * 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.COMPOSE, '*'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_reserved_operators(self):
lexer = Lexer('and diff iff inter mod not onlyif or to xor union')
self.assertEqual(lexer.get_next_token(), Token(Type.AND, '&'))
self.assertEqual(lexer.get_next_token(), Token(Type.DIFF, '-'))
self.assertEqual(lexer.get_next_token(), Token(Type.IFF, '<=>'))
self.assertEqual(lexer.get_next_token(), Token(Type.AND, '&'))
self.assertEqual(lexer.get_next_token(), Token(Type.MOD, '%'))
self.assertEqual(lexer.get_next_token(), Token(Type.NOT, '~'))
self.assertEqual(lexer.get_next_token(), Token(Type.IMPL, '=>'))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.TO, '->'))
self.assertEqual(lexer.get_next_token(), Token(Type.XOR, '^'))
self.assertEqual(lexer.get_next_token(), Token(Type.UNION, '+'))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def test_integer_division(self):
lexer = Lexer('1 / 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.DIV, '/'))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))
def get_next_token(self):
while self.pos < len(self.text) and self.text[self.pos].isspace():
self.pos += 1
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '//':
self.pos += 2
while self.pos < len(self.text) and self.text[self.pos] != '\n':
self.pos += 1
while self.pos < len(self.text) and self.text[self.pos].isspace():
self.pos += 1
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '/*':
self.pos += 2
while self.pos + 1 < len(
self.text) and self.text[self.pos:self.pos + 2] != '*/':
self.pos += 1
if self.pos + 1 == len(self.text):
raise
self.pos += 2
while self.pos < len(self.text) and self.text[self.pos].isspace():
self.pos += 1
if self.pos == len(self.text):
return Token(Type.EOF, '')
if self.text[self.pos].isalpha() and not self.text[self.pos].isdigit():
i = self.pos
while self.pos < len(self.text) and self.text[
self.pos].isalpha() and not self.text[self.pos].isdigit():
self.pos += 1
while self.pos < len(self.text) and self.text[self.pos] == '`':
self.pos += 1
lexeme = self.text[i:self.pos]
if lexeme in self.reserved.words:
return self.reserved.words[lexeme]
return Token(Type.ID, self.text[i:self.pos])
if self.text[self.pos] == '"':
self.pos += 1
i = self.pos
while self.pos < len(self.text) and self.text[self.pos] != '"':
self.pos += 1
if self.pos == len(self.text):
raise
j = self.pos
self.pos += 1
return Token(Type.STRING, self.text[i:j])
if self.text[self.pos] == "'":
self.pos += 1
i = self.pos
while self.pos < len(self.text) and self.text[self.pos] != "'":
self.pos += 1
if self.pos == len(self.text):
raise
j = self.pos
self.pos += 1
return Token(Type.STRING, self.text[i:j])
if self.text[self.pos] == '0':
i = self.pos
self.pos += 1
if not (self.pos < len(self.text) and self.text[self.pos] == '.'):
return Token(Type.INT, int(self.text[i]))
if not (self.pos + 1 < len(self.text)
and self.text[self.pos + 1] >= '0'
and self.text[self.pos + 1] <= '9'):
return Token(Type.INT, int(self.text[i]))
self.pos += 1
while self.pos < len(self.text) and self.text[
self.pos] >= '0' and self.text[self.pos] <= '9':
self.pos += 1
return Token(Type.REAL, float(self.text[i:self.pos]))
if self.text[self.pos] >= '1' and self.text[self.pos] <= '9':
i = self.pos
while self.pos < len(self.text) and self.text[
self.pos] >= '0' and self.text[self.pos] <= '9':
self.pos += 1
if not (self.pos < len(self.text) and self.text[self.pos] == '.'):
return Token(Type.INT, int(self.text[i:self.pos]))
if not (self.pos + 1 < len(self.text)
and self.text[self.pos + 1] >= '0'
and self.text[self.pos + 1] <= '9'):
return Token(Type.INT, int(self.text[i:self.pos]))
self.pos += 1
while self.pos < len(self.text) and self.text[
self.pos] >= '0' and self.text[self.pos] <= '9':
self.pos += 1
return Token(Type.REAL, float(self.text[i:self.pos]))
if self.text[self.pos] == '_':
self.pos += 1
return Token(Type.UNDER, '_')
if self.pos + 2 < len(self.text) and self.text[self.pos:self.pos +
3] == '**T':
self.pos += 3
return Token(Type.TPOSE, '**T')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '**':
self.pos += 2
return Token(Type.POW, '**')
if self.text[self.pos] == '*':
self.pos += 1
return Token(Type.COMPOSE, '*')
if self.text[self.pos] == '/':
self.pos += 1
return Token(Type.DIV, '/')
if self.text[self.pos] == '%':
self.pos += 1
return Token(Type.MOD, '%')
if self.text[self.pos] == '^':
self.pos += 1
return Token(Type.XOR, '^')
if self.text[self.pos] == '&':
self.pos += 1
return Token(Type.AND, '&')
if self.text[self.pos] == '+':
self.pos += 1
return Token(Type.UNION, '+')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '->':
self.pos += 2
return Token(Type.TO, '->')
if self.text[self.pos] == '-':
self.pos += 1
return Token(Type.DIFF, '-')
if self.text[self.pos] == '#':
self.pos += 1
return Token(Type.CARD, '#')
if self.text[self.pos] == '!':
self.pos += 1
return Token(Type.FACT, '!')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '~=':
self.pos += 2
return Token(Type.NEQ, '~=')
if self.text[self.pos] == '~':
self.pos += 1
return Token(Type.NOT, '~')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '>=':
self.pos += 2
return Token(Type.GTEQ, '>=')
if self.text[self.pos] == '>':
self.pos += 1
return Token(Type.GT, '>')
if self.pos + 2 < len(self.text) and self.text[self.pos:self.pos +
3] == '<=>':
self.pos += 3
return Token(Type.IFF, '<=>')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '<=':
self.pos += 2
return Token(Type.LTEQ, '<=')
if self.text[self.pos] == '<':
self.pos += 1
return Token(Type.LT, '<')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '=>':
self.pos += 2
return Token(Type.IMPL, '=>')
if self.text[self.pos] == '=':
self.pos += 1
return Token(Type.EQ, '=')
if self.text[self.pos] == '(':
self.pos += 1
return Token(Type.OPENP, '(')
if self.text[self.pos] == ')':
self.pos += 1
return Token(Type.CLOSEP, ')')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '||':
self.pos += 2
return Token(Type.NORM, '||')
if self.text[self.pos] == '|':
self.pos += 1
return Token(Type.PIPE, '|')
if self.text[self.pos] == '[':
self.pos += 1
return Token(Type.OPENB, '[')
if self.text[self.pos] == ']':
self.pos += 1
return Token(Type.CLOSEB, ']')
if self.text[self.pos] == '{':
self.pos += 1
return Token(Type.OPENC, '{')
if self.text[self.pos] == '}':
self.pos += 1
return Token(Type.CLOSEC, '}')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == ':=':
self.pos += 2
return Token(Type.DEFAS, ':=')
if self.text[self.pos] == ':':
self.pos += 1
return Token(Type.COLON, ':')
if self.pos + 1 < len(self.text) and self.text[self.pos:self.pos +
2] == '..':
self.pos += 2
return Token(Type.RANGE, '..')
if self.text[self.pos] == '.':
self.pos += 1
return Token(Type.PERIOD, '.')
if self.text[self.pos] == ',':
self.pos += 1
return Token(Type.COMMA, ',')
if self.text[self.pos] == ';':
self.pos += 1
return Token(Type.SEMIC, ';')
if self.text[self.pos] == '?':
self.pos += 1
return Token(Type.QUEST, '?')
raise
def test_integer_not_equal(self):
lexer = Lexer('1 ~= 1')
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.NEQ, '~='))
self.assertEqual(lexer.get_next_token(), Token(Type.INT, 1))
self.assertEqual(lexer.get_next_token(), Token(Type.EOF, ''))