def _handle_word(self, state):
word = ""
start_pos = state.column
while state.column < len(state.line):
current_symbol = state.line[state.column]
if not is_word(current_symbol):
break
word += current_symbol
state.column += 1
if word in Tokens.Keyword.value:
self._token.append(
Token(token_type=Tokens.Keyword,
row=state.row,
column=start_pos,
spec=word))
return
if word in Tokens.ConstLiteral.value:
self._token.append(
Token(token_type=Tokens.ConstLiteral,
row=state.row,
column=start_pos,
spec=word))
return
self._add_correct_token(token_type=Tokens.Identifier,
row=state.row,
column=start_pos,
lit_value=word)
def _interpolate(self, state):
self._token.append(
Token(token_type=Tokens.InterpolationStart,
row=state.row,
column=state.column))
state.column += 2
while state.row < len(state.all_lines):
state.line = state.all_lines[state.row]
while state.column < len(state.line):
if self._get_next_token_interpolate(state):
return
state.column = 0
state.row += 1
state.row -= 1
state.column = len(state.all_lines[-1]) - 1
self._check_multi_line_mode(state)
self._token.append(
Token(token_type=Tokens.InterpolationEnd,
row=state.row,
column=state.column))
self._invalid_token.append(
WrongToken(message="There was no end for interpolation",
token=self._token[-1]))
def _handle_punctuation(self, state):
current_symbol = state.line[state.column]
self._token.append(
Token(token_type=Tokens.Punctuation,
row=state.row,
column=state.column,
spec=current_symbol))
state.column += 1
def _add_wrong_token(self, token_type, row, column, lit_value,
error_message):
self._token.append(
Token(token_type=token_type,
row=row,
column=column,
index=len(self._symbol_table)))
self._symbol_table.append(lit_value)
self._invalid_token.append(
WrongToken(message=error_message, token=self._token[-1]))
def _get_next_token_interpolate(self, state):
current_symbol = state.line[state.column]
if not state.multi_line_mode and current_symbol == Tokens.InterpolationEnd.value:
self._token.append(
Token(token_type=Tokens.InterpolationEnd,
row=state.row,
column=state.column))
return True
else:
self._get_next_token(state)
return False
def _handle_whitespaces(self, state):
while state.column < len(state.line):
current_symbol = state.line[state.column]
if current_symbol == ' ':
self._token.append(
Token(token_type=Tokens.Space,
row=state.row,
column=state.column))
elif current_symbol == '\t':
self._token.append(
Token(token_type=Tokens.Tab,
row=state.row,
column=state.column))
elif current_symbol == '\n':
self._token.append(
Token(token_type=Tokens.Enter,
row=state.row,
column=state.column))
else:
break
state.column += 1
def _init_template_string(self, state):
self._token.append(
Token(token_type=Tokens.StartTemplateString,
row=state.row,
column=state.column))
self._add_correct_token(token_type=Tokens.StringLiteral,
row=state.row,
column=state.column,
lit_value="",
token_spec=state.line[state.column])
state.token = self._token[-1]
state.multi_line_mode = True
state.column += 1
def _add_correct_token(self,
token_type,
row,
column,
lit_value,
token_spec=None):
self._token.append(
Token(token_type=token_type,
row=row,
column=column,
index=len(self._symbol_table),
spec=token_spec))
self._symbol_table.append(lit_value)
def _handle_template_string(self, state):
st = state.column
ending = state.token.spec
while st < len(state.line):
temp_s = state.line[st:]
pos = temp_s.find(ending)
if pos == -1:
if self._check_interpolation(state, state.column,
len(state.line)):
return
if len(state.line) >= 2 and state.line[-2] == '\\':
slash_pos = len(state.line) - 2
while slash_pos >= state.column and state.line[
slash_pos] == '\\':
slash_pos -= 1
if (len(state.line) - 2 - slash_pos) % 2 == 1:
string_lit = state.line[state.column:]
self._symbol_table[state.token.index] += string_lit
state.column = len(state.line)
return
string_lit = state.line[state.column:]
self._symbol_table[state.token.index] += string_lit
state.column = len(state.line)
return
else:
st += pos
if self._check_interpolation(state, state.column, st):
return
if st - 1 >= state.column and state.line[st - 1] == '\\':
slash_pos = st - 1
while slash_pos >= state.column and state.line[
slash_pos] == '\\':
slash_pos -= 1
if (st - 1 - slash_pos) % 2 == 1:
st += 1
continue
string_lit = state.line[state.column:st]
self._symbol_table[state.token.index] += string_lit
self._token.append(
Token(token_type=Tokens.EndTemplateString,
row=state.row,
column=st))
state.column = st + 1
state.token = None
state.multi_line_mode = False
return
def _check_multi_line_mode(self, state):
if state.multi_line_mode:
if state.token.type == Tokens.MultiLineComment:
self._invalid_token.append(
WrongToken(message="Multi line comment requires end",
token=state.token))
if state.token.type == Tokens.StringLiteral and state.token.spec != '`':
self._invalid_token.append(
WrongToken(message="String literal requires the end",
token=state.token))
if state.token.type == Tokens.StringLiteral and state.token.spec == '`':
self._token.append(
Token(token_type=Tokens.EndTemplateString,
row=state.row,
column=state.column))
self._invalid_token.append(
WrongToken(message="String literal requires the end",
token=self._token[-1]))
def _handle_operation(self, state):
op = ""
start_pos = state.column
while state.column < len(state.line):
temp_op = op + state.line[state.column]
if temp_op in Tokens.Operators.value:
op = temp_op
else:
break
state.column += 1
if op == '/':
self._check_regex(state)
else:
self._token.append(
Token(token_type=Tokens.Operators,
row=state.row,
column=start_pos,
spec=op))
def _check_regex(self, state):
start_pos = state.column - 1
i = len(self._token) - 1
from TokenChecks import is_whitespace_token
while i >= 0 and is_whitespace_token(self._token[i].type):
i -= 1
if i < 0 or self._token[i].type != Tokens.Identifier \
and self._token[i].type != Tokens.NumberLiteral:
regex = '/'
while state.column < len(state.line):
if state.line[state.column] != '/':
regex += state.line[state.column]
state.column += 1
else:
break
if state.column < len(state.line):
regex += '/'
state.column += 1
if state.column >= len(state.line):
if state.line[-1] == '\n':
state.column = len(state.line) - 1
regex = regex[:-1]
regex_token = Token(token_type=Tokens.Regex,
row=state.row,
column=start_pos,
index=len(self._symbol_table))
self._invalid_token.append(
WrongToken(message="Regex has no end", token=regex_token))
self._symbol_table.append(regex)
self._token.append(regex_token)
else:
was_error = False
while state.column < len(state.line):
c = state.line[state.column]
if not is_symbol(c) or c == '_':
if is_number(c) or c == '_':
was_error = True
regex += c
break
regex += c
state.column += 1
if not was_error:
regex_token = Token(token_type=Tokens.Regex,
row=state.row,
column=start_pos,
index=len(self._symbol_table))
self._symbol_table.append(regex)
self._token.append(regex_token)
else:
regex = regex[:]
regex_token = Token(token_type=Tokens.Regex,
row=state.row,
column=start_pos,
index=len(self._symbol_table))
self._invalid_token.append(
WrongToken(message="Regex incorrect ending",
token=regex_token))
self._symbol_table.append(regex)
self._token.append(regex_token)
state.column += 1
else:
self._token.append(
Token(token_type=Tokens.Operators,
row=state.row,
column=start_pos,
spec='/'))
def _handle_number(self, state):
if self._check_nonnormal_numbers(state):
return
current_symbol = state.line[state.column]
if is_dot(current_symbol):
if state.column + 1 >= len(state.line) or not is_number(
state.line[state.column + 1]):
if is_dot(state.line[state.column + 1]) and \
state.column + 2 < len(state.line) and is_dot(state.line[state.column + 2]):
self._token.append(
Token(token_type=Tokens.Operators,
spec='...',
row=state.row,
column=state.column))
state.column += 3
else:
self._token.append(
Token(token_type=Tokens.Punctuation,
row=state.row,
column=state.column,
spec=current_symbol))
state.column += 1
return
if current_symbol == '0' and state.column + 1 < len(state.line) and \
not is_correct_after_number(state.line[state.column + 1]) and \
not is_dot(state.line[state.column + 1]):
state.column += 1
self._add_wrong_token(token_type=Tokens.NumberLiteral,
row=state.row,
column=state.column - 1,
lit_value=current_symbol +
state.line[state.column],
error_message="Not allowed start from zero")
state.column += 1
return
new_number = ""
was_dot, was_e, was_sign, was_dash, last_dash = False, False, False, False, False
start_pos = state.column
unknown = False
while state.column < len(state.line):
current_symbol = state.line[state.column]
if is_number(current_symbol):
new_number += current_symbol
last_dash = False
elif is_dot(current_symbol):
new_number += current_symbol
if last_dash:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Unexpected symbol in the number end")
state.column += 1
return
was_dot = True
break
elif is_e(current_symbol):
new_number += current_symbol
if last_dash:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Unexpected symbol in the number end")
state.column += 1
return
was_e = True
break
elif is_down_dash(current_symbol):
last_dash = True
new_number += current_symbol
else:
unknown = True
break
state.column += 1
if was_dot:
state.column += 1
if state.column < len(state.line) and \
(is_number(state.line[state.column]) or is_e(state.line[state.column])):
while state.column < len(state.line):
current_symbol = state.line[state.column]
if is_number(current_symbol):
new_number += current_symbol
last_dash = False
elif is_e(current_symbol):
new_number += current_symbol
if last_dash:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message=
"Unexpected symbol in the number end")
state.column += 1
return
was_e = True
break
elif is_down_dash(current_symbol):
last_dash = True
new_number += current_symbol
else:
unknown = True
break
state.column += 1
if was_e:
state.column += 1
if state.column < len(state.line) and \
(is_number(state.line[state.column]) or state.line[state.column] in '+-'):
if state.line[state.column] in '+-':
new_number += state.line[state.column]
state.column += 1
if state.column >= len(state.line) or not is_number(
state.line[state.column]):
new_number = new_number[:-1]
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Unexpected symbol in the number end"
)
state.column -= 1
return
while state.column < len(state.line):
current_symbol = state.line[state.column]
if is_number(current_symbol):
new_number += current_symbol
last_dash = False
elif is_down_dash(current_symbol):
last_dash = True
new_number += current_symbol
else:
unknown = True
break
state.column += 1
else:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Unexpected symbol in the number end")
return
if last_dash:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Unexpected symbol in the number end")
state.column += 1
return
if unknown:
if is_correct_after_number(state.line[state.column]):
cur_spec = Tokens.NumberLiteral.value[1]
if was_dot or was_e:
cur_spec = Tokens.NumberLiteral.value[0]
self._add_correct_token(token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
token_spec=cur_spec)
elif state.line[state.column] == 'n':
new_number += 'n'
if was_dot or was_e:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
error_message="Invalid biginteger literal")
else:
self._add_correct_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
token_spec=Tokens.NumberLiteral.value[5])
state.column += 1
else:
self._add_wrong_token(
token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number + state.line[state.column],
error_message="Unexpected symbol in the number end")
state.column += 1
else:
cur_spec = Tokens.NumberLiteral.value[1]
if was_dot or was_e:
cur_spec = Tokens.NumberLiteral.value[0]
self._add_correct_token(token_type=Tokens.NumberLiteral,
row=state.row,
column=start_pos,
lit_value=new_number,
token_spec=cur_spec)
def __init__(self):
self.column = 0
self.row = 0
self.line = ""
self.token = Token(token_type=Tokens.Invalid)
self.multi_line_mode = False