def tokenize(line):
"""
Convert a string into a list of tokens
@param line: the string to convert
@return a list of tokens
"""
tokens = []
token = ''
# Identify tokens
parseFloatState = 0
for char in line:
# Handle float-like tokens
token, parseFloatState = tokenizeFloat(token, tokens, char,
parseFloatState)
# Handle non-float tokens
if parseFloatState: token += char
elif UC_Utils.isWhitespace(char):
if token: tokens.append(token)
token = ""
elif UC_Utils.isSpecialChar(char):
if token: tokens.append(token)
token = ""
tokens.append(char)
else:
token += char
token, parseFloatState = tokenizeFloat(token, tokens, "", parseFloatState)
if token: tokens.append(token)
return tokens
def aggregateSignHelper(tokens, updatedTokens):
while tokens:
token = tokens.pop(0)
if token == UC_Common.BRACKET_OPEN:
updatedTokens.append(token)
aggregateSignHelper(tokens, updatedTokens)
elif token == UC_Common.BRACKET_SHUT:
updatedTokens.append(token)
return updatedTokens
elif ((token == UC_Common.OPERATOR_ADD
or token == UC_Common.OPERATOR_SUB)
and (not updatedTokens
or UC_Utils.isSpecialChar(updatedTokens[-1]))):
if tokens and UC_Utils.isFloat(tokens[0]):
updatedTokens.append(f"{token}{tokens.pop(0)}")
elif not updatedTokens or updatedTokens[
-1] != UC_Common.BRACKET_SHUT:
updatedTokens.extend([
UC_Common.BRACKET_OPEN, f"{token}1",
UC_Common.OPERATOR_MUL
])
aggregateSignHelper(tokens, updatedTokens)
updatedTokens.append(UC_Common.BRACKET_SHUT)
else:
updatedTokens.append(token)
else:
updatedTokens.append(token)
def appendUnitTokens(aggregatedTokens, unitTokens, token=None):
# Append unit tokens to list of aggregated tokens
if unitTokens:
if unitTokens[-1] == UC_Common.OPERATOR_MUL or unitTokens[
-1] == UC_Common.OPERATOR_DIV:
operator = unitTokens.pop()
aggregatedTokens.append(unitTokens)
aggregatedTokens.append(operator)
else:
aggregatedTokens.append(unitTokens)
if token is not None:
# Inject multiplication if needed
if ((aggregatedTokens)
and (not UC_Utils.isSpecialChar(aggregatedTokens[-1]))
and (token == UC_Common.BRACKET_OPEN)):
aggregatedTokens.append(UC_Common.OPERATOR_MUL)
aggregatedTokens.append(token)
return []
def aggregateQuantities(tokens):
"""
Combine tokens which constitute a quantity
@param tokens: a list of tokens
"""
aggregatedTokens = []
needsValue = True
while tokens:
if UC_Utils.isOperator(tokens[0]):
if needsValue:
raise UC_Common.UnitError(
f"Expected float; received '{tokens[0]}'")
aggregatedTokens.append(tokens.pop(0))
needsValue = True
elif UC_Utils.isSpecialChar(tokens[0]):
aggregatedTokens.append(tokens.pop(0))
else:
needsValue = False
# Get value
quantity = '1'
try:
float(tokens[0])
quantity = tokens.pop(0)
except:
# Inject multiplication where needed
if aggregatedTokens and aggregatedTokens[
-1] == UC_Common.BRACKET_SHUT:
aggregatedTokens.append(UC_Common.OPERATOR_MUL)
# Get unit
unit = []
if tokens and isinstance(tokens[0], list):
unit = tokens.pop(0)
aggregatedTokens.append((quantity, unit))
if needsValue and aggregatedTokens:
raise UC_Common.UnitError(f"Expected float; no tokens received")
return aggregatedTokens