Ejemplo n.º 1
0
def evalLAMBDA(tree, env):
    # print("In evalLAMBDA")
    params = tree.right.right.left
    body = tree.right.right.right.right.left
    right = Lexeme("JOIN", "JOIN", body, env)
    close = Lexeme("CLOSURE", "CLOSURE", params, right)
    return close
Ejemplo n.º 2
0
	def nextOptToken( self ):
		while True:
			token = self._token_src.nextOptToken()
			if token:
				if token.isIndentation() and isinstance( token.lexemeValue(), str ) :
					v = token.lexemeValue()
					if v == self.previous():
						continue
					else:
						# print( 'COMPARE', len( self.previous() ), len( v ) )
						# print( 'Stack', self._previous )
						if v.startswith( self.previous() ):
							# It is a new, deeper indentation level.
							# Record the nesting level.
							self._previous.append( v )
							# Return an indent.
							return Lexeme( LexemeType.Indentation, 1 )
						else:
							# It is a shallower indentation level.
							# Keep popping the stack and pushing outdents.
							self._previous.pop()
							self._token_src.pushToken( token )
							return Lexeme( LexemeType.Indentation, -1 )
				else:
					return token
			elif self._previous:
				self._previous.pop()
				return Lexeme( LexemeType.Indentation, -1 )
			else:
				return token
Ejemplo n.º 3
0
    def get_lexeme(self):

        if self.buf in self.keywordsTable:
            keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
                                    self.line_number,
                                    self.column_number - len(self.buf))
            self.lexemesTable.append(keyword_lexeme)

        elif self.buf in self.delTable:
            delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
                                  self.line_number,
                                  self.column_number - len(self.buf) + 1)
            self.lexemesTable.append(delim_lexeme)

        elif self.buf.isdigit() and self.buf2 == '':
            u_int_lexeme = self.uIntsTable.add(
                int(self.buf), self.line_number,
                self.column_number - len(self.buf))
            self.lexemesTable.append(u_int_lexeme)

        elif self.buf.isdigit() and self.buf2 == ":" and self.buf3.isdigit():
            if int(self.buf) <= 24 and int(self.buf3) <= 60:
                time_lexeme = self.timeTablle.add(
                    self.buf + self.buf2 + self.buf3, self.line_number,
                    self.column_number - len(self.buf))
                self.lexemesTable.append(time_lexeme)

        else:
            idn_lexeme = self.idnTable.add(self.buf, self.line_number,
                                           self.column_number - len(self.buf))
            self.lexemesTable.append(idn_lexeme)

        self.buf = ''
        self.buf2 = ''
        self.buf3 = ''
Ejemplo n.º 4
0
    def get_lexeme(self):

        if self.buf in self.keywordsTable:
            keyword_lexeme = Lexeme(self.buf, self.keywordsTable[self.buf],
                                    self.line_number,
                                    self.column_number - len(self.buf))
            self.lexemesTable.append(keyword_lexeme)

        elif self.buf in self.delTable:
            delim_lexeme = Lexeme(self.buf, self.delTable[self.buf],
                                  self.line_number,
                                  self.column_number - len(self.buf) + 1)
            self.lexemesTable.append(delim_lexeme)

        elif self.buf.isdigit():
            u_int_lexeme = self.uIntsTable.add(
                int(self.buf), self.line_number,
                self.column_number - len(self.buf))
            self.lexemesTable.append(u_int_lexeme)

        else:
            idn_lexeme = self.idnTable.add(self.buf, self.line_number,
                                           self.column_number - len(self.buf))
            self.lexemesTable.append(idn_lexeme)

        self.buf = ''
Ejemplo n.º 5
0
 def expr(self):
     # print("In expr")
     p = self.primary()
     if (self.operatorPending()):
         o = self.operator()
         e = self.expr()
         return Lexeme("EXPR", "EXPR", Lexeme("OPERATOR", o, p, e))
     return self.cons("EXPR", p, None)
Ejemplo n.º 6
0
 def lexDivide(self, ch):
     buff = "" + ch
     ch = self.getCharacter()
     if (ch == "/"):
         return Lexeme("INTEGERDIVIDE", "//")
     else:
         self.pushbackCharacter()
         return Lexeme("DIVIDE", "/")
Ejemplo n.º 7
0
def evalOR(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue) or int(r.lvalue)))
    elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
        return Lexeme("BOOLEAN", (l.lvalue or r.lvalue))
    else:
        raise Exception("ERROR: Can't or: " + str(l) + " or " + str(r))
Ejemplo n.º 8
0
def evalAND(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue) and int(r.lvalue)))
    elif ((l.ltype == "BOOLEAN") and (r.ltype == "BOOLEAN")):
        return Lexeme("BOOLEAN", (l.lvalue and r.lvalue))
    else:
        raise Exception("ERROR: Can't and: " + str(l) + " and " + str(r))
Ejemplo n.º 9
0
def evalFUNCDEF(tree, env):
    # print("In evalFUNCDEF")
    variable = str(tree.right.left.lvalue)
    # print(variable)
    params = tree.right.right.right.left.left
    body = tree.right.right.right.right.right.left
    right = Lexeme("JOIN", "JOIN", body, env)
    close = Lexeme("CLOSURE", "CLOSURE", params, right)
    ret = insert(variable, close, env)
    return ret
Ejemplo n.º 10
0
def evalAPPEND(tree, env):
    # print("In evalAPPEND")
    value = evaluate(tree.right.right.left.left.left.left, env)
    v = eval(value.lvalue)
    arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
                   env)
    if (isinstance(v, str)):
        new = Lexeme("STRING", v)
    elif (isinstance(v, int)):
        new = Lexeme("INTEGER", v)
    arr.lvalue.append(new)
Ejemplo n.º 11
0
 def lexMinus(self, ch):
     buff = "" + ch
     ch = self.getCharacter()
     if (not (ch.isdigit())):
         self.pushbackCharacter()
         return Lexeme("MINUS", "-")
     else:
         while (ch.isdigit()):
             buff += ch
             ch = self.getCharacter()
         self.pushbackCharacter()
         return Lexeme("INTEGER", buff)
Ejemplo n.º 12
0
 def insert(self, variable, value):
     if(self.environment.left.left != None):
         self.environment.left.ltype = "JOIN"
         self.environment.left.lvalue = "JOIN"
         self.environment.left = Lexeme("IDS", "IDS", variable, self.environment.left)
         self.environment.right.left.ltype = "JOIN"
         self.environment.right.left.lvalue = "JOIN"
         self.environment.right.left = Lexeme("VALS", "VALS", value, self.environment.right.left)
     else:
         self.environment.left.left = variable
         self.environment.right.left.left = value
     return value
Ejemplo n.º 13
0
def evalLESSEQUAL(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
    elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
        return Lexeme("BOOLEAN", (l.lvalue <= r.lvalue))
    elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
        return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
    elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
        return Lexeme("BOOLEAN", (int(l.lvalue) <= int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
Ejemplo n.º 14
0
def evalGREATER(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    # print("{0} greater {1}?".format(l, r))
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
    elif ((l.ltype == "STRING") and (r.ltype == "STRING")):
        return Lexeme("BOOLEAN", (l.lvalue > r.lvalue))
    elif ((l.ltype == "INTEGER") and (r.ltype == "STRING")):
        return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
    elif ((l.ltype == "STRING") and (r.ltype == "INTEGER")):
        return Lexeme("BOOLEAN", (int(l.lvalue) > int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't equate: " + str(l) + " and " + str(r))
Ejemplo n.º 15
0
 def lexOp(self, ch):
     buff = "" + ch
     ch = self.getCharacter()
     if (buff == ">"):
         if (ch == "="):
             return Lexeme("GREATEREQUAL", ">=")
         else:
             self.pushbackCharacter()
             return Lexeme("GREATER", ">")
     elif (buff == "<"):
         if (ch == "="):
             return Lexeme("LESSEQUAL", "<=")
         else:
             self.pushbackCharacter()
             return Lexeme("LESS", "<")
     elif (buff == "="):
         if (ch == "="):
             return Lexeme("DOUBLEEQUAL", "==")
         else:
             self.pushbackCharacter()
             return Lexeme("EQUAL", "=")
     elif (buff == "!"):
         if (ch == "="):
             return Lexeme("NOTEQUAL", "!=")
         else:
             self.pushbackCharacter()
             return Lexeme("NOT", "!")
Ejemplo n.º 16
0
def evalINSERT(tree, env):
    # print("In evalINSERT")
    index = evaluate(
        tree.right.right.left.right.right.left.right.right.left.left.left.left,
        env)
    value = evaluate(tree.right.right.left.left.left.left, env)
    arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
                   env)
    i = eval(index.lvalue)
    v = eval(value.lvalue)
    if (isinstance(v, str)):
        new = Lexeme("STRING", v)
    elif (isinstance(v, int)):
        new = Lexeme("INTEGER", v)
    arr.lvalue.insert(i, v)
Ejemplo n.º 17
0
def evalMINUS(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue) - int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't subtract: " + str(l) + " and " + str(r))
Ejemplo n.º 18
0
def evalMULTIPLY(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue) * int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't multiply: " + str(l) + " and " + str(r))
Ejemplo n.º 19
0
def evalINTEGERDIVIDE(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue) // int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't divide: " + str(l) + " and " + str(r))
Ejemplo n.º 20
0
    def segment_word(self, word, analysis=None):
        """Takes a string representation of the word form to segment and (optionally) its analysis (Lemma) returned by MorphoDiTa. Returns a list of strings representing the individual morphs of the word."""
        node = None
        parent_node = None

        # First, try to find the word in the database.
        nodes = self.db.get_by_lemma(word)
        if nodes:
            node = nodes[0]
        elif analysis is not None:
            # If the word is not in the database itself, try to find its lemma there.
            lemma = techlemma_to_lemma(analysis.lemma)
            parent_nodes = self.db.get_by_lemma(lemma)
            if parent_nodes:
                # The word is not in the database, but its lemma is.
                # Create a new node for the word and propagate the bounds to it.
                parent_node = parent_nodes[0]
                node = Lexeme(word, parent_lemma=lemma)
                node.detect_stems(parent_node)
                node.copy_morph_bounds(parent_node)

        if node:
            return node.morphs()
        else:
            # If all else fails, consider the word to be a single morph.
            logger.debug("Word '%s' not recognized. No segmentation given.",
                         word)
            return [word]
Ejemplo n.º 21
0
    def __init__(self, parser, derinet_db):
        form_to_lexemes = derinet_db.lemma_to_lexemes
        lexemes = list(derinet_db.id_to_lexeme.values())
        with MorfFlexParser(morfflex_file_name, derinet_db) as morfflex:
            for lexeme in morfflex:
                # If the lexeme is in the database already, we have to ensure it has a different parent (or no parent at all).
                # Otherwise don't add duplicates.
                duplicate = False
                for already_present_node in derinet_db.get_by_lemma(
                        lexeme.lemma):
                    if already_present_node.parent and already_present_node.parent.lemma == lexeme.parent_lemma:
                        duplicate = True
                        break

                if not duplicate:
                    parents = derinet_db.get_by_lemma(lexeme.parent_lemma)
                    if parents:
                        lexeme.set_parent(parents[0])
                        lexemes.append(lexeme)
                        form_to_lexemes[lexeme.lemma].append(lexeme)
                    else:
                        #raise Exception("Parent of '%s' with lemma '%s' not found in the database." % (lexeme.lemma, lexeme.parent_lemma))
                        parent = Lexeme(lexeme.parent_lemma)
                        lexemes.append(parent)
                        form_to_lexemes[parent.lemma].append(parent)
                        lexeme.set_parent(parent)
                        lexemes.append(lexeme)
                        form_to_lexemes[lexeme.lemma].append(lexeme)

        self.lexemes = lexemes
        self.form_to_lexemes = form_to_lexemes
Ejemplo n.º 22
0
 def add(self, value, line_number, column_number):
     if not self.get_by_value(value):
         lexeme = Lexeme(value, self._code, line_number, column_number)
         self._lexemes.append(lexeme)
         self._code += 1
         return lexeme
     return self.get_by_value(value)
Ejemplo n.º 23
0
def evalOPERATOR(tree, env):
    # print("In evalOPERATOR")
    l = tree.left
    r = tree.right
    op = tree.lvalue
    # op = tree.lvalue.left
    new = Lexeme(str(op).strip(), str(op).strip(), l, r)
    return evaluate(new, env)
Ejemplo n.º 24
0
 def next(self):
     line = self.filehandle.readline()
     if line:
         line = line.rstrip('\n')
         id, lemma, techlemma, pos, parent = line.split('\t')
         return Lexeme(lemma, id=id, parent_id=parent)
     else:
         raise StopIteration()
Ejemplo n.º 25
0
def evalPOWER(tree, env):
    l = evaluate(tree.left, env)
    r = evaluate(tree.right, env)
    if ((l.ltype == "INTEGER") and (r.ltype == "INTEGER")):
        return Lexeme("INTEGER", (int(l.lvalue)**int(r.lvalue)))
    else:
        raise Exception("ERROR: Can't raise(" + str(l) + " to " + str(r) +
                        " power.")
Ejemplo n.º 26
0
def evalEXPRLIST(tree, env):
    # print("In evalEXPRLIST")
    r = None
    if (tree.right == None):
        return evaluate(tree.left, env)
    if (tree.right.right.left != None):
        r = evaluate(tree.right.right.left, env)
        new = Lexeme("JOIN", "JOIN", evaluate(tree.left, env), r)
    return new
Ejemplo n.º 27
0
    def lexNumber(self, ch):
        buff = "" + ch
        ch = self.getCharacter()
        while (ch.isdigit()):
            buff += ch
            ch = self.getCharacter()
        self.pushbackCharacter()

        return Lexeme("INTEGER", buff)
Ejemplo n.º 28
0
    def lexString(self):
        buff = ""
        ch = self.getCharacter()
        while (ch != '\"'):
            if (ch == '\\'): ch = self.getCharacter()
            buff += ch
            ch = self.getCharacter()

        return Lexeme("STRING", "\"" + buff + "\"")
Ejemplo n.º 29
0
def evalSET(tree, env):
    # print("In evalREMOVE")
    index = evaluate(
        tree.right.right.left.right.right.left.right.right.left.left.left.left,
        env)
    value = evaluate(tree.right.right.left.left.left.left, env)
    arr = evaluate(tree.right.right.left.right.right.left.left.left.left.left,
                   env)
    # print(arr.ltype)
    # print(arr.lvalue)
    if (isinstance(index.lvalue, str)):
        i = eval(index.lvalue)
    else:
        i = index.lvalue
    if (isinstance(value.lvalue, str)):
        v = eval(value.lvalue)
    else:
        v = value.lvalue
    if (isinstance(v, str)):
        new = Lexeme("STRING", v)
    elif (isinstance(v, int)):
        new = Lexeme("INTEGER", v)
    arr.lvalue[i] = new
Ejemplo n.º 30
0
def evalPRIMARY(tree, env):
    # print("In evalPRIMARY")
    # print(tree.left)
    if (tree.right == None):
        return evaluate(tree.left, env)
    elif (tree.left.ltype == "OPAREN"):
        return evaluate(tree.right.left, env)
    elif (tree.left.ltype == "OBRACKET"):
        # print("MADE AN ARRAY")
        elements = evaluate(tree.right.left, env)
        # print(elements.ltype)
        arr = makeArgList(elements, env)
        # print(arr)
        return evaluate(Lexeme("ARRAY", arr, None, None), env)