def test_tokenize_trigonometrics(self):
expression = "sin(5) + cos(5) + tan(5) + ctan(5)"
token_list = [
tokens.SinFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(5),
tokens.CloseParenthesisToken(),
tokens.PlusOperatorToken(),
tokens.CosFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(5),
tokens.CloseParenthesisToken(),
tokens.PlusOperatorToken(),
tokens.TanFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(5),
tokens.CloseParenthesisToken(),
tokens.PlusOperatorToken(),
tokens.CtanFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(5),
tokens.CloseParenthesisToken()
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_division(self):
expr = "5 / 2"
token_list = [
tokens.OperandToken(5),
tokens.DivisionOperatorToken(),
tokens.OperandToken(2)
]
self.assertListEqual(token_list, tokenize(expr))
def test_tokenize_logarithm_with_custom_base(self):
expr = "Log100(10)"
token_list = [
tokens.LogFunctionToken(has_custom_base=True),
tokens.OperandToken(100),
tokens.OpenParenthesisToken(),
tokens.OperandToken(10),
tokens.CloseParenthesisToken()
]
self.assertListEqual(token_list, tokenize(expr))
def test_tokenize_negative(self):
expression = "-5 + 2"
token_list = [
tokens.OperandToken(-5),
tokens.PlusOperatorToken(),
tokens.OperandToken(2)
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_simple_operator(self):
expression = "2 + 1"
computed_token_list = tokenize(expression)
postfix_token_list = infix_to_postfix(computed_token_list)
token_list = [
tokens.OperandToken(2),
tokens.OperandToken(1),
tokens.PlusOperatorToken(),
]
self.assertListEqual(postfix_token_list, token_list)
def test_tokenize_implicit_product_variable_and_parentheses(self):
expr = "x(2-1)"
token_list = [
tokens.VariableToken('x'),
tokens.ProductOperatorToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(2),
tokens.MinusOperatorToken(),
tokens.OperandToken(1),
tokens.CloseParenthesisToken()
]
self.assertListEqual(token_list, tokenize(expr))
def test_parenthesis(self):
expression = "2 * (1 + 5)"
computed_token_list = tokenize(expression)
postfix_token_list = infix_to_postfix(computed_token_list)
token_list = [
tokens.OperandToken(2),
tokens.OperandToken(1),
tokens.OperandToken(5),
tokens.PlusOperatorToken(),
tokens.ProductOperatorToken()
]
self.assertListEqual(postfix_token_list, token_list)
def test_multiple_operators_reversed(self):
expression = "2 * 1 + 5"
computed_token_list = tokenize(expression)
postfix_token_list = infix_to_postfix(computed_token_list)
token_list = [
tokens.OperandToken(2),
tokens.OperandToken(1),
tokens.ProductOperatorToken(),
tokens.OperandToken(5),
tokens.PlusOperatorToken(),
]
self.assertListEqual(postfix_token_list, token_list)
def test_tokenize_negative_within_parenthesis(self):
expression = "(-5 + 2)"
token_list = [
tokens.OpenParenthesisToken(),
tokens.OperandToken(-5),
tokens.PlusOperatorToken(),
tokens.OperandToken(2),
tokens.CloseParenthesisToken()
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_2(self):
expression = "2 * var + 0.5 = 1"
token_list = [
tokens.OperandToken(2),
tokens.ProductOperatorToken(),
tokens.VariableToken('var'),
tokens.PlusOperatorToken(),
tokens.OperandToken(0.5),
tokens.EqualSignToken(),
tokens.OperandToken(1)
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_implicit_product_operand_and_constant(self):
expr = "5 pi"
token_list = [
tokens.OperandToken(5),
tokens.ProductOperatorToken(),
tokens.PiConstantToken()
]
self.assertListEqual(token_list, tokenize(expr))
def test_tokenize_5(self):
expression = "Log10"
token_list = [
tokens.LogFunctionToken(),
tokens.OperandToken(10),
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_1(self):
expression = "(3 + (4 - 1)) * 5"
token_list = [
tokens.OpenParenthesisToken(),
tokens.OperandToken(3),
tokens.PlusOperatorToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(4),
tokens.MinusOperatorToken(),
tokens.OperandToken(1),
tokens.CloseParenthesisToken(),
tokens.CloseParenthesisToken(),
tokens.ProductOperatorToken(),
tokens.OperandToken(5)
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_simple_function(self):
expression = "sin 5"
computed_token_list = tokenize(expression)
postfix_token_list = infix_to_postfix(computed_token_list)
token_list = [
tokens.OperandToken(5),
tokens.SinFunctionToken(),
]
self.assertListEqual(postfix_token_list, token_list)
def test_tokenize_4(self):
expression = "Log(10)"
token_list = [
tokens.LogFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(10),
tokens.CloseParenthesisToken()
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_3(self):
expression = "2x + 1 = 2(1-x)"
token_list = [
tokens.OperandToken(2),
tokens.ProductOperatorToken(),
tokens.VariableToken('x'),
tokens.PlusOperatorToken(),
tokens.OperandToken(1),
tokens.EqualSignToken(),
tokens.OperandToken(2),
tokens.ProductOperatorToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(1),
tokens.MinusOperatorToken(),
tokens.VariableToken('x'),
tokens.CloseParenthesisToken()
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_tokenize_7(self):
expression = "sin(1.5*pi)"
token_list = [
tokens.SinFunctionToken(),
tokens.OpenParenthesisToken(),
tokens.OperandToken(1.5),
tokens.ProductOperatorToken(),
tokens.PiConstantToken(),
tokens.CloseParenthesisToken()
]
computed_token_list = tokenize(expression)
self.assertListEqual(computed_token_list, token_list)
def test_print_token(self):
token = tokens.PlusOperatorToken()
self.assertEqual(str(token), "Token: PlusOperatorToken")
self.assertEqual(repr(token), "Token: PlusOperatorToken")
token = tokens.LogFunctionToken(has_custom_base=False)
self.assertEqual(str(token), "Token: LogFunctionToken (10-base)")
token = tokens.LogFunctionToken(has_custom_base=True)
self.assertEqual(str(token), "Token: LogFunctionToken (Custom base)")
token = tokens.OperandToken(5)
self.assertEqual(str(token), "Token: Operand (5)")
token = tokens.VariableToken('x')
self.assertEqual(str(token), "Token: Variable (x)")
def tokenize(input: str):
input = input.lower()
token_list = []
i = 0
while i < len(input):
char = input[i]
if char == '(':
token_list.append(tokens.OpenParenthesisToken())
elif char == ')':
token_list.append(tokens.CloseParenthesisToken())
elif char == '+':
token_list.append(tokens.PlusOperatorToken())
# Only compute the '-' char as the minus operator if it's not part of a negative operand
elif char == '-' and token_list and not isinstance(token_list[-1], tokens.OpenParenthesisToken):
token_list.append(tokens.MinusOperatorToken())
elif char == '*':
token_list.append(tokens.ProductOperatorToken())
elif char == '/':
token_list.append(tokens.DivisionOperatorToken())
elif char == '=':
token_list.append(tokens.EqualSignToken())
elif char.isalpha():
token_strings = {
'sin': tokens.SinFunctionToken,
'cos': tokens.CosFunctionToken,
'tan': tokens.TanFunctionToken,
'ctan': tokens.CtanFunctionToken,
'pi': tokens.PiConstantToken,
'e': tokens.EulerConstantToken,
'log': tokens.LogFunctionToken,
'ln': tokens.LnFunctionToken
}
for key in token_strings:
if input[i:i + len(key)] == key:
token_list.append(token_strings[key]())
i += len(key) - 1
break
# The character was not part of any special token
else:
token_components = [char]
j = i + 1
while j < len(input) and input[j].isalpha():
token_components.append(input[j])
j += 1
i = j - 1
token_list.append(tokens.VariableToken(''.join(token_components)))
elif char.isdecimal() or char == '-':
token_components = [char]
# Keep consuming decimal characters
j = i + 1
while j < len(input) and (input[j].isdecimal() or input[j] == '.'):
token_components.append(input[j])
j += 1
i = j - 1
token_list.append(tokens.OperandToken(float(''.join(token_components))))
i += 1
# Token List postprocessing, in particular:
# - Add product operator between operand and variable
# - Add product operator between operand and constant
# - Add product operator between operand and open parenthesis (except for the Log function)
# - Add product operator between variable and open parenthesis
# - Mark LogFunctionToken to have a custom base if followed by two ConstantTokens
processed_token_list = []
for i, tok in enumerate(token_list):
processed_token_list.append(tok)
if i == len(token_list) - 1:
break
if is_operand(tok):
# - Add product operator between operand and variable
if is_variable(token_list[i + 1]):
logger.debug("Adding implicit product operator between operand and variable")
processed_token_list.append(tokens.ProductOperatorToken())
elif is_constant(token_list[i + 1]):
logger.debug("Adding implicit product operator between operand and constant")
processed_token_list.append(tokens.ProductOperatorToken())
# - Add product operator between operand and open parenthesis (except for the Log function)
elif is_left_paren(token_list[i + 1]) and not isinstance(token_list[i - 1], tokens.LogFunctionToken):
logger.debug("Adding implicit product operator between operand and open parenthesis")
processed_token_list.append(tokens.ProductOperatorToken())
elif is_variable(tok):
# - Add product operator between variable and open parenthesis
if is_left_paren(token_list[i + 1]):
logger.debug("Adding implicit product operator between variable and open parenthesis")
processed_token_list.append(tokens.ProductOperatorToken())
# - Mark LogFunctionToken to have a custom base if followed by two ConstantTokens or a ConstantToken and open parenthesis
elif isinstance(tok, tokens.LogFunctionToken) and i < len(token_list) - 2 and is_operand(
token_list[i + 1]) and (is_constant(token_list[i + 2]) or is_left_paren(token_list[i + 2])):
tok.has_custom_base = True
return processed_token_list
