def get_lexeme(self):
if self.buf in self.keywordsTable:
keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(keyword_lexeme)
elif self.buf in self.delTable:
delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
self.line_number,
self.column_number - len(self.buf) + 1)
self.lexemesTable.append(delim_lexeme)
elif self.buf.isdigit():
u_int_lexeme = self.uIntsTable.add(
int(self.buf), self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(u_int_lexeme)
else:
idn_lexeme = self.idnTable.add(self.buf, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(idn_lexeme)
self.buf = ''
def insert(self, variable, value, env_list): """ Inserts a new variable with the specified value into the specified environment. Parameters: variable -- A VARIABLE type Lexeme object with the variable's name as its value. value -- A NUMBER, BOOLEAN, or STRING type Lexeme object with the variable's value as its value. env_list -- The Environment list into which to insert the variable. """ if self._debug: print( "Inserting variable [" + str(variable) + "]") if self._debug: print( " with value [" + str(value) + "].") var_glue = Lexeme(token_type="GLUE", left=variable) val_glue = Lexeme(token_type="GLUE", left=value) head = env_list.left var_glue.right = head.left val_glue.right = head.right head.left = var_glue head.right = val_glue
def get_lexeme(self):
if self.buf in self.keywordsTable:
keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(keyword_lexeme)
elif self.buf in self.delTable:
delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
self.line_number,
self.column_number - len(self.buf) + 1)
self.lexemesTable.append(delim_lexeme)
elif self.buf.isdigit() and self.buf2 == '':
u_int_lexeme = self.uIntsTable.add(
int(self.buf), self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(u_int_lexeme)
elif self.buf.isdigit() and self.buf2 == ":" and self.buf3.isdigit():
if int(self.buf) <= 24 and int(self.buf3) <= 60:
time_lexeme = self.timeTablle.add(
self.buf + self.buf2 + self.buf3, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(time_lexeme)
else:
idn_lexeme = self.idnTable.add(self.buf, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(idn_lexeme)
self.buf = ''
self.buf2 = ''
self.buf3 = ''
def evalLAMBDA(tree, env):
# print("In evalLAMBDA")
params = tree.right.right.left
body = tree.right.right.right.right.left
right = Lexeme("JOIN", "JOIN", body, env)
close = Lexeme("CLOSURE", "CLOSURE", params, right)
return close
def statements(self):
if self._debug: print(" in statements")
tree = Lexeme(token_type="STATEMENTS")
tree.left = self.statement()
tree.right = self.optStatements()
return tree
def nextOptToken( self ): while True: token = self._token_src.nextOptToken() if token: if token.isIndentation() and isinstance( token.lexemeValue(), str ) : v = token.lexemeValue() if v == self.previous(): continue else: # print( 'COMPARE', len( self.previous() ), len( v ) ) # print( 'Stack', self._previous ) if v.startswith( self.previous() ): # It is a new, deeper indentation level. # Record the nesting level. self._previous.append( v ) # Return an indent. return Lexeme( LexemeType.Indentation, 1 ) else: # It is a shallower indentation level. # Keep popping the stack and pushing outdents. self._previous.pop() self._token_src.pushToken( token ) return Lexeme( LexemeType.Indentation, -1 ) else: return token elif self._previous: self._previous.pop() return Lexeme( LexemeType.Indentation, -1 ) else: return token
def expr(self):
# print("In expr")
p = self.primary()
if (self.operatorPending()):
o = self.operator()
e = self.expr()
return Lexeme("EXPR", "EXPR", Lexeme("OPERATOR", o, p, e))
return self.cons("EXPR", p, None)
def lexDivide(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (ch == "/"):
return Lexeme("INTEGERDIVIDE", "//")
else:
self.pushbackCharacter()
return Lexeme("DIVIDE", "/")
def commaChain(self):
if self._debug: print(" in commaChain")
self.match("COMMA")
tree = Lexeme(token_type="GLUE")
tree.left = self.expression()
tree.right = self.optCommaChain()
return tree
def evalOR(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) or int(r.lvalue)))
elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
return Lexeme("BOOLEAN", (l.lvalue or r.lvalue))
else:
raise Exception("ERROR: Can't or: " + str(l) + " or " + str(r))
def evalAND(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) and int(r.lvalue)))
elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
return Lexeme("BOOLEAN", (l.lvalue and r.lvalue))
else:
raise Exception("ERROR: Can't and: " + str(l) + " and " + str(r))
def whileStatement(self):
if self._debug: print(" in whileStatement")
self.match("WHILE")
tree = Lexeme(token_type="WHILE_STATEMENT")
tree.left = self.booleanExpression()
self.match("COMMA")
tree.right = self.block()
return tree
def assignment(self):
if self._debug: print(" in assignment")
self.match("SET")
tree = Lexeme(token_type="ASSIGNMENT")
tree.left = self.match("VARIABLE")
self.match("TO")
tree.right = self.expression()
self.match("PERIOD")
return tree
def evalFUNCDEF(tree, env):
# print("In evalFUNCDEF")
variable = str(tree.right.left.lvalue)
# print(variable)
params = tree.right.right.right.left.left
body = tree.right.right.right.right.right.left
right = Lexeme("JOIN", "JOIN", body, env)
close = Lexeme("CLOSURE", "CLOSURE", params, right)
ret = insert(variable, close, env)
return ret
def optCommaChain(self):
if self._debug: print(" in optCommaChain")
tree = None
if self.check("COMMA"):
tree = Lexeme(token_type="GLUE")
self.match("COMMA")
if self.expressionPending():
tree.left = self.expression()
tree.right = self.optCommaChain()
return tree
def evalAPPEND(tree, env):
# print("In evalAPPEND")
value = evaluate(tree.right.right.left.left.left.left, env)
v = eval(value.lvalue)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue.append(new)
def lexMinus(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (not (ch.isdigit())):
self.pushbackCharacter()
return Lexeme("MINUS", "-")
else:
while (ch.isdigit()):
buff += ch
ch = self.getCharacter()
self.pushbackCharacter()
return Lexeme("INTEGER", buff)
def insert(self, variable, value):
if(self.environment.left.left != None):
self.environment.left.ltype = "JOIN"
self.environment.left.lvalue = "JOIN"
self.environment.left = Lexeme("IDS", "IDS", variable, self.environment.left)
self.environment.right.left.ltype = "JOIN"
self.environment.right.left.lvalue = "JOIN"
self.environment.right.left = Lexeme("VALS", "VALS", value, self.environment.right.left)
else:
self.environment.left.left = variable
self.environment.right.left.left = value
return value
def evalLESSEQUAL(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (l.lvalue <= r.lvalue))
elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
else:
raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
def lexOp(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (buff == ">"):
if (ch == "="):
return Lexeme("GREATEREQUAL", ">=")
else:
self.pushbackCharacter()
return Lexeme("GREATER", ">")
elif (buff == "<"):
if (ch == "="):
return Lexeme("LESSEQUAL", "<=")
else:
self.pushbackCharacter()
return Lexeme("LESS", "<")
elif (buff == "="):
if (ch == "="):
return Lexeme("DOUBLEEQUAL", "==")
else:
self.pushbackCharacter()
return Lexeme("EQUAL", "=")
elif (buff == "!"):
if (ch == "="):
return Lexeme("NOTEQUAL", "!=")
else:
self.pushbackCharacter()
return Lexeme("NOT", "!")
def evalGREATER(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
# print("{0} greater {1}?".format(l, r))
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (l.lvalue > r.lvalue))
elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
else:
raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
def evalINSERT(tree, env):
# print("In evalINSERT")
index = evaluate(
tree.right.right.left.right.right.left.right.right.left.left.left.left,
env)
value = evaluate(tree.right.right.left.left.left.left, env)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
i = eval(index.lvalue)
v = eval(value.lvalue)
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue.insert(i, v)
def __init__(self, parser, derinet_db):
form_to_lexemes = derinet_db.lemma_to_lexemes
lexemes = list(derinet_db.id_to_lexeme.values())
with MorfFlexParser(morfflex_file_name, derinet_db) as morfflex:
for lexeme in morfflex:
# If the lexeme is in the database already, we have to ensure it has a different parent (or no parent at all).
# Otherwise don't add duplicates.
duplicate = False
for already_present_node in derinet_db.get_by_lemma(
lexeme.lemma):
if already_present_node.parent and already_present_node.parent.lemma == lexeme.parent_lemma:
duplicate = True
break
if not duplicate:
parents = derinet_db.get_by_lemma(lexeme.parent_lemma)
if parents:
lexeme.set_parent(parents[0])
lexemes.append(lexeme)
form_to_lexemes[lexeme.lemma].append(lexeme)
else:
#raise Exception("Parent of '%s' with lemma '%s' not found in the database." % (lexeme.lemma, lexeme.parent_lemma))
parent = Lexeme(lexeme.parent_lemma)
lexemes.append(parent)
form_to_lexemes[parent.lemma].append(parent)
lexeme.set_parent(parent)
lexemes.append(lexeme)
form_to_lexemes[lexeme.lemma].append(lexeme)
self.lexemes = lexemes
self.form_to_lexemes = form_to_lexemes
def add(self, value, line_number, column_number):
if not self.get_by_value(value):
lexeme = Lexeme(value, self._code, line_number, column_number)
self._lexemes.append(lexeme)
self._code += 1
return lexeme
return self.get_by_value(value)
def evalMULTIPLY(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) * int(r.lvalue)))
else:
raise Exception("ERROR: Can't multiply: " + str(l) + " and " + str(r))
def evalMINUS(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) - int(r.lvalue)))
else:
raise Exception("ERROR: Can't subtract: " + str(l) + " and " + str(r))
def evalINTEGERDIVIDE(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) // int(r.lvalue)))
else:
raise Exception("ERROR: Can't divide: " + str(l) + " and " + str(r))
def next(self):
line = self.filehandle.readline()
if line:
line = line.rstrip('\n')
id, lemma, techlemma, pos, parent = line.split('\t')
return Lexeme(lemma, id=id, parent_id=parent)
else:
raise StopIteration()
def evalPOWER(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue)**int(r.lvalue)))
else:
raise Exception("ERROR: Can't raise(" + str(l) + " to " + str(r) +
" power.")
def evalOPERATOR(tree, env):
# print("In evalOPERATOR")
l = tree.left
r = tree.right
op = tree.lvalue
# op = tree.lvalue.left
new = Lexeme(str(op).strip(), str(op).strip(), l, r)
return evaluate(new, env)
def segment_word(self, word, analysis=None):
"""Takes a string representation of the word form to segment and (optionally) its analysis (Lemma) returned by MorphoDiTa. Returns a list of strings representing the individual morphs of the word."""
node = None
parent_node = None
# First, try to find the word in the database.
nodes = self.db.get_by_lemma(word)
if nodes:
node = nodes[0]
elif analysis is not None:
# If the word is not in the database itself, try to find its lemma there.
lemma = techlemma_to_lemma(analysis.lemma)
parent_nodes = self.db.get_by_lemma(lemma)
if parent_nodes:
# The word is not in the database, but its lemma is.
# Create a new node for the word and propagate the bounds to it.
parent_node = parent_nodes[0]
node = Lexeme(word, parent_lemma=lemma)
node.detect_stems(parent_node)
node.copy_morph_bounds(parent_node)
if node:
return node.morphs()
else:
# If all else fails, consider the word to be a single morph.
logger.debug("Word '%s' not recognized. No segmentation given.",
word)
return [word]
def evalEXPRLIST(tree, env):
# print("In evalEXPRLIST")
r = None
if (tree.right == None):
return evaluate(tree.left, env)
if (tree.right.right.left != None):
r = evaluate(tree.right.right.left, env)
new = Lexeme("JOIN", "JOIN", evaluate(tree.left, env), r)
return new
def lexNumber(self, ch):
buff = "" + ch
ch = self.getCharacter()
while (ch.isdigit()):
buff += ch
ch = self.getCharacter()
self.pushbackCharacter()
return Lexeme("INTEGER", buff)
def lexString(self):
buff = ""
ch = self.getCharacter()
while (ch != '\"'):
if (ch == '\\'): ch = self.getCharacter()
buff += ch
ch = self.getCharacter()
return Lexeme("STRING", "\"" + buff + "\"")
def program(self):
if self._debug: print(" in program")
tree = Lexeme(token_type="PROGRAM")
sections = []
while self.functionDefPending() or self.statementsPending():
if self.functionDefPending():
sections.append(self.functionDef())
if self.statementsPending():
sections.append(self.statements())
self.match("EOF")
tmp = Lexeme(token_type="GLUE")
tracer = tmp
for section in sections:
tracer.left = section
tracer.right = Lexeme(token_type="GLUE")
tracer = tracer.right
tree.right = tmp
return tree
def evalPRIMARY(tree, env):
# print("In evalPRIMARY")
# print(tree.left)
if (tree.right == None):
return evaluate(tree.left, env)
elif (tree.left.ltype == "OPAREN"):
return evaluate(tree.right.left, env)
elif (tree.left.ltype == "OBRACKET"):
# print("MADE AN ARRAY")
elements = evaluate(tree.right.left, env)
# print(elements.ltype)
arr = makeArgList(elements, env)
# print(arr)
return evaluate(Lexeme("ARRAY", arr, None, None), env)
def evalSET(tree, env):
# print("In evalREMOVE")
index = evaluate(
tree.right.right.left.right.right.left.right.right.left.left.left.left,
env)
value = evaluate(tree.right.right.left.left.left.left, env)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
# print(arr.ltype)
# print(arr.lvalue)
if (isinstance(index.lvalue, str)):
i = eval(index.lvalue)
else:
i = index.lvalue
if (isinstance(value.lvalue, str)):
v = eval(value.lvalue)
else:
v = value.lvalue
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue[i] = new
def booleanExpression(self):
tree = Lexeme(token_type="BOOLEAN_EXPRESSION")
tmp = Lexeme(token_type="GLUE")
tmp.left = self.expression()
self.match("IS")
tree.left = self.boolOp()
tmp.right = self.expression()
tree.right = tmp
return tree
def ifStatement(self):
if self._debug: print(" in ifStatement")
self.match("IF")
tree = Lexeme(token_type="IF_STATEMENT")
tmp = Lexeme(token_type="GLUE")
tree.left = self.booleanExpression()
self.match("COMMA")
tmp.left = self.block()
tmp.right = self.optOtherwise()
tree.right = tmp
return tree
def sequence(self):
if self._debug: print(" in sequence")
tree = Lexeme(token_type="LIST")
tree.left = self.expression()
if self.check("AND"):
self.match("AND")
tmp = Lexeme(token_type="GLUE")
tmp.left = self.expression()
tree.right = tmp
elif self.commaChainPending():
tmp = self.commaChain()
self.match("AND")
tracer = tmp
while tracer.right:
tracer = tracer.right
tmp2 = Lexeme(token_type="GLUE")
tmp2.left = self.expression()
tracer.right = tmp2
tree.right = tmp
return tree
def sequence(self):
if self._debug: print(" in sequence")
tree = Lexeme(token_type="GLUE") # this will be attached to the "head" in my environment
tree.left = self.expression()
if self.check("AND"):
self.match("AND")
tmp = Lexeme(token_type="GLUE")
tmp.left = self.expression()
tree.right = tmp
elif self.commaChainPending():
tmp = self.commaChain()
self.match("AND")
tracer = tmp
while tracer.right:
tracer = tracer.right
tracer.left = self.expression()
tree.right = tmp
return tree
