def evalLAMBDA(tree, env):
# print("In evalLAMBDA")
params = tree.right.right.left
body = tree.right.right.right.right.left
right = Lexeme("JOIN", "JOIN", body, env)
close = Lexeme("CLOSURE", "CLOSURE", params, right)
return close
def nextOptToken( self ): while True: token = self._token_src.nextOptToken() if token: if token.isIndentation() and isinstance( token.lexemeValue(), str ) : v = token.lexemeValue() if v == self.previous(): continue else: # print( 'COMPARE', len( self.previous() ), len( v ) ) # print( 'Stack', self._previous ) if v.startswith( self.previous() ): # It is a new, deeper indentation level. # Record the nesting level. self._previous.append( v ) # Return an indent. return Lexeme( LexemeType.Indentation, 1 ) else: # It is a shallower indentation level. # Keep popping the stack and pushing outdents. self._previous.pop() self._token_src.pushToken( token ) return Lexeme( LexemeType.Indentation, -1 ) else: return token elif self._previous: self._previous.pop() return Lexeme( LexemeType.Indentation, -1 ) else: return token
def get_lexeme(self):
if self.buf in self.keywordsTable:
keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(keyword_lexeme)
elif self.buf in self.delTable:
delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
self.line_number,
self.column_number - len(self.buf) + 1)
self.lexemesTable.append(delim_lexeme)
elif self.buf.isdigit() and self.buf2 == '':
u_int_lexeme = self.uIntsTable.add(
int(self.buf), self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(u_int_lexeme)
elif self.buf.isdigit() and self.buf2 == ":" and self.buf3.isdigit():
if int(self.buf) <= 24 and int(self.buf3) <= 60:
time_lexeme = self.timeTablle.add(
self.buf + self.buf2 + self.buf3, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(time_lexeme)
else:
idn_lexeme = self.idnTable.add(self.buf, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(idn_lexeme)
self.buf = ''
self.buf2 = ''
self.buf3 = ''
def get_lexeme(self):
if self.buf in self.keywordsTable:
keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(keyword_lexeme)
elif self.buf in self.delTable:
delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
self.line_number,
self.column_number - len(self.buf) + 1)
self.lexemesTable.append(delim_lexeme)
elif self.buf.isdigit():
u_int_lexeme = self.uIntsTable.add(
int(self.buf), self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(u_int_lexeme)
else:
idn_lexeme = self.idnTable.add(self.buf, self.line_number,
self.column_number - len(self.buf))
self.lexemesTable.append(idn_lexeme)
self.buf = ''
def expr(self):
# print("In expr")
p = self.primary()
if (self.operatorPending()):
o = self.operator()
e = self.expr()
return Lexeme("EXPR", "EXPR", Lexeme("OPERATOR", o, p, e))
return self.cons("EXPR", p, None)
def lexDivide(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (ch == "/"):
return Lexeme("INTEGERDIVIDE", "//")
else:
self.pushbackCharacter()
return Lexeme("DIVIDE", "/")
def evalOR(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) or int(r.lvalue)))
elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
return Lexeme("BOOLEAN", (l.lvalue or r.lvalue))
else:
raise Exception("ERROR: Can't or: " + str(l) + " or " + str(r))
def evalAND(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) and int(r.lvalue)))
elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
return Lexeme("BOOLEAN", (l.lvalue and r.lvalue))
else:
raise Exception("ERROR: Can't and: " + str(l) + " and " + str(r))
def evalFUNCDEF(tree, env):
# print("In evalFUNCDEF")
variable = str(tree.right.left.lvalue)
# print(variable)
params = tree.right.right.right.left.left
body = tree.right.right.right.right.right.left
right = Lexeme("JOIN", "JOIN", body, env)
close = Lexeme("CLOSURE", "CLOSURE", params, right)
ret = insert(variable, close, env)
return ret
def evalAPPEND(tree, env):
# print("In evalAPPEND")
value = evaluate(tree.right.right.left.left.left.left, env)
v = eval(value.lvalue)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue.append(new)
def lexMinus(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (not (ch.isdigit())):
self.pushbackCharacter()
return Lexeme("MINUS", "-")
else:
while (ch.isdigit()):
buff += ch
ch = self.getCharacter()
self.pushbackCharacter()
return Lexeme("INTEGER", buff)
def insert(self, variable, value):
if(self.environment.left.left != None):
self.environment.left.ltype = "JOIN"
self.environment.left.lvalue = "JOIN"
self.environment.left = Lexeme("IDS", "IDS", variable, self.environment.left)
self.environment.right.left.ltype = "JOIN"
self.environment.right.left.lvalue = "JOIN"
self.environment.right.left = Lexeme("VALS", "VALS", value, self.environment.right.left)
else:
self.environment.left.left = variable
self.environment.right.left.left = value
return value
def evalLESSEQUAL(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (l.lvalue <= r.lvalue))
elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
else:
raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
def evalGREATER(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
# print("{0} greater {1}?".format(l, r))
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (l.lvalue > r.lvalue))
elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
else:
raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
def lexOp(self, ch):
buff = "" + ch
ch = self.getCharacter()
if (buff == ">"):
if (ch == "="):
return Lexeme("GREATEREQUAL", ">=")
else:
self.pushbackCharacter()
return Lexeme("GREATER", ">")
elif (buff == "<"):
if (ch == "="):
return Lexeme("LESSEQUAL", "<=")
else:
self.pushbackCharacter()
return Lexeme("LESS", "<")
elif (buff == "="):
if (ch == "="):
return Lexeme("DOUBLEEQUAL", "==")
else:
self.pushbackCharacter()
return Lexeme("EQUAL", "=")
elif (buff == "!"):
if (ch == "="):
return Lexeme("NOTEQUAL", "!=")
else:
self.pushbackCharacter()
return Lexeme("NOT", "!")
def evalINSERT(tree, env):
# print("In evalINSERT")
index = evaluate(
tree.right.right.left.right.right.left.right.right.left.left.left.left,
env)
value = evaluate(tree.right.right.left.left.left.left, env)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
i = eval(index.lvalue)
v = eval(value.lvalue)
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue.insert(i, v)
def evalMINUS(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) - int(r.lvalue)))
else:
raise Exception("ERROR: Can't subtract: " + str(l) + " and " + str(r))
def evalMULTIPLY(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) * int(r.lvalue)))
else:
raise Exception("ERROR: Can't multiply: " + str(l) + " and " + str(r))
def evalINTEGERDIVIDE(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue) // int(r.lvalue)))
else:
raise Exception("ERROR: Can't divide: " + str(l) + " and " + str(r))
def segment_word(self, word, analysis=None):
"""Takes a string representation of the word form to segment and (optionally) its analysis (Lemma) returned by MorphoDiTa. Returns a list of strings representing the individual morphs of the word."""
node = None
parent_node = None
# First, try to find the word in the database.
nodes = self.db.get_by_lemma(word)
if nodes:
node = nodes[0]
elif analysis is not None:
# If the word is not in the database itself, try to find its lemma there.
lemma = techlemma_to_lemma(analysis.lemma)
parent_nodes = self.db.get_by_lemma(lemma)
if parent_nodes:
# The word is not in the database, but its lemma is.
# Create a new node for the word and propagate the bounds to it.
parent_node = parent_nodes[0]
node = Lexeme(word, parent_lemma=lemma)
node.detect_stems(parent_node)
node.copy_morph_bounds(parent_node)
if node:
return node.morphs()
else:
# If all else fails, consider the word to be a single morph.
logger.debug("Word '%s' not recognized. No segmentation given.",
word)
return [word]
def __init__(self, parser, derinet_db):
form_to_lexemes = derinet_db.lemma_to_lexemes
lexemes = list(derinet_db.id_to_lexeme.values())
with MorfFlexParser(morfflex_file_name, derinet_db) as morfflex:
for lexeme in morfflex:
# If the lexeme is in the database already, we have to ensure it has a different parent (or no parent at all).
# Otherwise don't add duplicates.
duplicate = False
for already_present_node in derinet_db.get_by_lemma(
lexeme.lemma):
if already_present_node.parent and already_present_node.parent.lemma == lexeme.parent_lemma:
duplicate = True
break
if not duplicate:
parents = derinet_db.get_by_lemma(lexeme.parent_lemma)
if parents:
lexeme.set_parent(parents[0])
lexemes.append(lexeme)
form_to_lexemes[lexeme.lemma].append(lexeme)
else:
#raise Exception("Parent of '%s' with lemma '%s' not found in the database." % (lexeme.lemma, lexeme.parent_lemma))
parent = Lexeme(lexeme.parent_lemma)
lexemes.append(parent)
form_to_lexemes[parent.lemma].append(parent)
lexeme.set_parent(parent)
lexemes.append(lexeme)
form_to_lexemes[lexeme.lemma].append(lexeme)
self.lexemes = lexemes
self.form_to_lexemes = form_to_lexemes
def add(self, value, line_number, column_number):
if not self.get_by_value(value):
lexeme = Lexeme(value, self._code, line_number, column_number)
self._lexemes.append(lexeme)
self._code += 1
return lexeme
return self.get_by_value(value)
def evalOPERATOR(tree, env):
# print("In evalOPERATOR")
l = tree.left
r = tree.right
op = tree.lvalue
# op = tree.lvalue.left
new = Lexeme(str(op).strip(), str(op).strip(), l, r)
return evaluate(new, env)
def next(self):
line = self.filehandle.readline()
if line:
line = line.rstrip('\n')
id, lemma, techlemma, pos, parent = line.split('\t')
return Lexeme(lemma, id=id, parent_id=parent)
else:
raise StopIteration()
def evalPOWER(tree, env):
l = evaluate(tree.left, env)
r = evaluate(tree.right, env)
if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
return Lexeme("INTEGER", (int(l.lvalue)**int(r.lvalue)))
else:
raise Exception("ERROR: Can't raise(" + str(l) + " to " + str(r) +
" power.")
def evalEXPRLIST(tree, env):
# print("In evalEXPRLIST")
r = None
if (tree.right == None):
return evaluate(tree.left, env)
if (tree.right.right.left != None):
r = evaluate(tree.right.right.left, env)
new = Lexeme("JOIN", "JOIN", evaluate(tree.left, env), r)
return new
def lexNumber(self, ch):
buff = "" + ch
ch = self.getCharacter()
while (ch.isdigit()):
buff += ch
ch = self.getCharacter()
self.pushbackCharacter()
return Lexeme("INTEGER", buff)
def lexString(self):
buff = ""
ch = self.getCharacter()
while (ch != '\"'):
if (ch == '\\'): ch = self.getCharacter()
buff += ch
ch = self.getCharacter()
return Lexeme("STRING", "\"" + buff + "\"")
def evalSET(tree, env):
# print("In evalREMOVE")
index = evaluate(
tree.right.right.left.right.right.left.right.right.left.left.left.left,
env)
value = evaluate(tree.right.right.left.left.left.left, env)
arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
env)
# print(arr.ltype)
# print(arr.lvalue)
if (isinstance(index.lvalue, str)):
i = eval(index.lvalue)
else:
i = index.lvalue
if (isinstance(value.lvalue, str)):
v = eval(value.lvalue)
else:
v = value.lvalue
if (isinstance(v, str)):
new = Lexeme("STRING", v)
elif (isinstance(v, int)):
new = Lexeme("INTEGER", v)
arr.lvalue[i] = new
def evalPRIMARY(tree, env):
# print("In evalPRIMARY")
# print(tree.left)
if (tree.right == None):
return evaluate(tree.left, env)
elif (tree.left.ltype == "OPAREN"):
return evaluate(tree.right.left, env)
elif (tree.left.ltype == "OBRACKET"):
# print("MADE AN ARRAY")
elements = evaluate(tree.right.left, env)
# print(elements.ltype)
arr = makeArgList(elements, env)
# print(arr)
return evaluate(Lexeme("ARRAY", arr, None, None), env)