def advance(self): #if self.suppressAdvance ? self.supressAdvance = False : pass if self.suppressAdvance == True: self.suppressAdvance = False return if self.operand != None: self.operand = None self.currentOp = self.nextOp try: nt = self.bs.nextToken() except: nt = Token(TokenType.END_FILE, 'EOF') #print self.currentOp, self.currentOp.toString(), nt, nt.toString() if nt == "": return if nt.getTokenType() == TokenType.END_FILE: self.nextOp = nt return elif nt.getTokenType() == TokenType.SYMBOL or nt.getTokenType() == TokenType.LITERAL: self.operand = nt print "OPERAND SET", self.operand.toString() try: nt = self.bs.nextToken() except: nt = Token(TokenType.END_FILE, 'EOF') self.nextOp = nt print hex(self.jvm.getPC()), "(", self.advances ,")", self.currentOp.toString(), self.operand, self.nextOp.toString() self.advances += 1
def insert(self, Element, Token): tokenType = self._MyXMLDocument.createElement(Token.type()) token = self._MyXMLDocument.createTextNode(" "+Token.text()+" ") tokenType.appendChild(token) if(Token.text() != ""): self._MyTokenizer.advance() Element.appendChild(tokenType) return tokenType
def filterTokens(self, original): result = [] for vertex in original: copy = Token(vertex.index) for property in vertex.getPropertyTypes(): if property not in self._forbiddenProperties: copy.addProperty(property=property, value=vertex.getProperty(property)) result.append(copy) return result
def __init__(self): Token.__init__(self) params = urllib.urlencode({}) headers = { "X-Auth-Token":self.apitoken, "Content-type":"application/json" } conn = httplib.HTTPConnection(self.apiurlt[1]) conn.request("GET", "%s/flavors/detail?" %self.apiurlt[2], params, headers) response = conn.getresponse() self.flavor_metadata = response.read() self.flavor_metadata = json.loads(self.flavor_metadata) conn.close()
def filter(self, original=NLPInstance): edges = self.filterEdges(original.getEdges()) if not self._collaps: return NLPInstance(tokens=original.tokens, edges=edges, renderType=original.renderType, splitPoints=original.splitPoints) else: tokens = set() for e in edges: if e.renderType == Edge.RenderType.dependency: tokens.add(e.From) tokens.add(e.To) else: if e.renderType == Edge.RenderType.span: for i in range(e.From.index, e.To.index + 1): tokens.add(original.getToken(index=i)) _sorted = sorted(tokens, key=attrgetter("int_index")) updatedTokens = [] old2new = {} new2old = {} for t in _sorted: newToken = Token(len(updatedTokens)) newToken.merge(original.tokens[int(t.index)]) old2new[t] = newToken new2old[newToken] = t updatedTokens.append(newToken) updatedEdges = set() for e in edges: updatedEdges.add(Edge(From=old2new[e.From], To=old2new[e.To], label=e.label, note=e.note, Type=e.type, renderType=e.renderType, description=e.description)) # find new split points splitPoints = [] newTokenIndex = 0 for oldSplitPoint in original.splitPoints: newToken = updatedTokens[newTokenIndex] oldToken = new2old[newToken] while newTokenIndex + 1 < len(tokens) and oldToken.index < oldSplitPoint: newTokenIndex += 1 newToken = updatedTokens[newTokenIndex] oldToken = new2old[newToken] splitPoints.append(newTokenIndex) return NLPInstance(tokens=updatedTokens, edges=updatedEdges, renderType=original.renderType, splitPoints=splitPoints)
def getPOSTaggerOutput(inputFileName): executePOSTagger(inputFileName) tokensList = [] phrasesList = [] # open the file in read mode and iterates its content with open( parameters.paths["taggerOutput"] + "/" + inputFileName + 'Tagged.txt', 'r') as fileText: for line in fileText: splitedLine = line.split() if len(splitedLine) == 0: # when a phrase ends, the PoSTagger outputs a newline # all tokens from current phrase have been obtained # add current list of tokens to phrases list phrasesList.append(tokensList) # empty list of tokens tokensList = [] elif len(splitedLine) == 2: # when PoSTagger does not know the lemma, he only outputs the original set of words and the tags tokenContent = splitedLine[0] tags = splitedLine[1] currentToken = Token.Token(tokenContent, tokenContent, tags) tokensList.append(currentToken) else: # normal case -> current token have original content, lemma and tags tokenContent = splitedLine[0] lemma = splitedLine[1] tags = splitedLine[2] currentToken = Token.Token(tokenContent, lemma, tags) tokensList.append(currentToken) return phrasesList
def make_equals(self): tok_type = C_EQ pos_start = self.pos.copy() self.advance() if self.current_char == '=': self.advance() tok_type = C_EE return Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def check_commentline(tokenlist: List[str], current_line: int) -> Tuple[bool, List[Token.Token]]: """Checks if the given construction is of the type 'comment'. If it is, the first value will return True and the second value will return a list of tokens. If it isn't of the type 'comment', the first value will return False and the second value wil return None or an error token. Args: tokenlist (List[str]): A list of strings consisting of an instruction and their parameters Returns(either): bool, List[Token.Token]: Returns a bool(whether the token is of this type) and a list of tokens, which is the instruction and the parameters. bool, None : Returns a bool(whether the token is of this type) and None """ nextToken, tokenlist = tokenlist.next() if nextToken != "#": return False, [ Token.Token('ERROR', "Token is not of type 'comment'", current_line) ] return True, [Token.Token('IGNORE', 'commentline', current_line)]
def scan_tokens(self): ''' While our self.current, the iterator, has not reached the end, keep scanning and adding tokens At the end of our source input, we will add an EOF token ''' while not self.is_at_end(): self.start = self.current self._scan_token() self.tokens.append(Token.Token(TokenType.EOF, "", self.line)) return self.tokens
def _id(self): result = '' while self.curr_char is not None and self.curr_char.isalnum( ) or self.curr_char == '_': result += self.curr_char self.advance() token = Token.Token(Token.ID, result) return token
def expresionID(c): global flagID, valor, fila, columna if isLetter(c) or isNumber(c): valor += c columna += 1 return elif ord(c) == 95: #_ valor += c columna += 1 return elif ord(c) == 32: #espacio #valor += c columna += 1 if valor == 'restaurante' or valor == 'Restaurante': Simbolos.append( Token.token(valor, fila, (columna - 1 - len(valor)), 'Palabra_Reservada')) else: Simbolos.append( Token.token(valor, fila, (columna - 1 - len(valor)), 'Identificador')) valor = "" flagID = False elif ord(c) == 61: #= if valor == 'restaurante' or valor == 'Restaurante': Simbolos.append( Token.token(valor, fila, (columna - len(valor)), 'Palabra_Reservada')) else: Simbolos.append( Token.token(valor, fila, (columna - len(valor)), 'Identificador')) columna += 1 Simbolos.append(Token.token('=', fila, (columna - 2), 'Simbolo_igual')) valor = "" flagID = False elif ord(c) == 59: #; if valor == 'restaurante' or valor == 'Restaurante': Simbolos.append( Token.token(valor, fila, (columna - len(valor)), 'Palabra_Reservada')) else: Simbolos.append( Token.token(valor, fila, (columna - len(valor)), 'Identificador')) columna += 1 Simbolos.append( Token.token(';', fila, (columna - 2), 'Simbolo_PuntoyComa')) valor = "" flagID = False else: Errores.append(Error.error(c, 'identificador No valido', fila, columna))
def make_minus_or_arrow(self): tok_type = C_MINUS pos_start = self.pos.copy() self.advance() if self.current_char == '>': self.advance() tok_type = C_ARROW return Token(tok_type, pos_start=pos_start, pos_end=self.pos)
def make_number(self): num_str = '' dot_count = 0 pos_start = self.pos.copy() # find out whether str number is int or float while self.current_char != None and self.current_char in DIGITS + '.': if self.current_char == '.': # as soon a we find a single dot(.) , we mark the number as float if dot_count == 1: break dot_count += 1 num_str += self.current_char self.advance() # float case if dot_count == 0: return Token(C_INT, int(num_str), pos_start, self.pos) # int case else: return Token(C_FLOAT, float(num_str), pos_start, self.pos)
def compileDo(self, Element): doStatementText = "doStatement" # insert 'doStatement' Element = self.insert(Element, Token(doStatementText)) # insert 'do' self.insert(Element, self._MyTokenizer.token()) self.compileSubroutineCall(Element) # insert ';' self.insert(Element, self._MyTokenizer.token())
def compileExpressionList(self, Element): expresionListText = "expressionList" # insert 'subroutineCall' Element = self.insert(Element, Token(expresionListText)) while (not (self._MyTokenizer.token().text() == ")")): if self._MyTokenizer.token().text() == ",": # insert ',' self.insert(Element, self._MyTokenizer.token()) self.compileExpression(Element)
def __init__(self, p): self.path = p self.total_words = 0 self.single_appear = 0 self.unique_words = 0 self.top30_words = 0 self.avg_word_doc = 0 self.seconds_elapsed = 0 self.collection = Token.Token(p) self.sorted_dict = {}
def evalBS(text): alexer = Alexer(text) tokens = [] error = False while alexer.current_char != None: if alexer.current_char in DIGITS: tokens.append(alexer.make_number()) elif alexer.current_char in CHARS: tokens.append(alexer.make_string()) elif alexer.current_char == "\"": tokens.append(alexer.make_literal_string()) else: tokens.append(alexer.make_tokens()) if tokens[-1] == "err": error = True tokens[-1] = Token.Token("ERR", alexer.error) break if tokens[-1].type != "ERR": tokens.append(Token.Token("NEWLINE", "\n")) return [tokens, error]
def test_channels_listall(): '''Test channels_listall function''' db.reset_DB() auth_register('*****@*****.**', 'pas123456', 'Bob', 'Smith') token = Token.generateToken('*****@*****.**') channels_create(token, 'Hi', True) channels_create(token, 'Bye', True) assert channels_listall(token) == dumps({'channels' : [{"channel_id": 1, "is_public": True,\ "owner_members": [1], "members": [1], "name": "Hi"}, {"channel_id": 2, "is_public": True, \ "owner_members": [1], "members": [1], "name": "Bye"}]})
def recognize(self): current = 'S' idPointer = [] pointer = None for l, line in enumerate(self.token_in): for token in line: for i in token: if i in self.states[current].keys(): current = self.states[current][i][0] else: if len(token) == 1 and i in self.states[current].keys( ): current = self.states[current][i][0] else: current = 'err' break if current != 'err' and 'ε' in self.states[current].keys(): if token in self.reserved: id = token else: id = 'Id' if token not in idPointer: idPointer.append(token) pointer = idPointer.index(token) novo = Token(token, l + 1, id, current, False, pointer) self.TS.append(novo) pointer = None else: error = False id = "Id" try: float(token) id = 'digit' except: error = True novo = Token(token, l + 1, id, 'err', error, pointer) self.TS.append(novo) current = 'S' final = Token('EOF', l + 1, 'EOF', current, False, None) self.TS.append(final)
def endScope(self): dying_scope = self.scopes.pop() for (name, number) in dying_scope.items(): if number >= 1 and name != "this": ''' This name was never referenced. The error reporter expects a name Token, not just a string, so re-create one. ''' raise Resolver.ResolutionError( Token.Token(IDENTIFIER, name, None, number), f"Variable {name} never referenced in its scope")
def test_search(): '''Test that all strings in messages are the same as the query_str''' db.reset_DB() auth_register('*****@*****.**', 'pas123456', 'Hosh', 'Mehta') token = Token.generateToken('*****@*****.**') channels_create(token, 'Yes', True) message_send(token, 1, 'Hi') query_str = 'Hi' messages = search(token, query_str) for _ in messages: assert query_str == 'Hi'
def compileReturn(self, Element): returnStatementText = "returnStatement" # insert 'returnStatement' Element = self.insert(Element, Token(returnStatementText)) # insert 'return' self.insert(Element, self._MyTokenizer.token()) if self._MyTokenizer.token().text() != ";": self.compileExpression(Element) # insert ';' self.insert(Element, self._MyTokenizer.token())
def number(self): result = '' while self.current_char is not None and self.current_char.isdigit(): result+=self.current_char self.advance() if self.current_char == '.': result+=self.current_char self.advance() while self.current_char is not None and self.current_char.isdigit(): result+=self.current_char self.advance() token = Token('REAL_CONST',float(result)) else: token = Token('INT_CONST',int(result)) return token
def make_not_equal(self): pos_start = self.pos.copy() self.advance() if self.current_char == "=": self.advance() return Token(TT_NE, pos_start=pos_start, pos_end=self.pos), None self.advance() return None, ExpectedCharError(pos_start, self.pos, "'=' (after '!' mark)")
def __read_symbol(self): result = '' while not self.__is_eof() and self.__char().isalpha() or self.__char( ).isdigit() or self.__char() == '_': result = result + self.__char() self.__next() if result.upper() in ['LABEL', 'DAT']: result = Token('DIRECTIVE', result.upper()) elif result.upper() in opcodes: result = Token('INSTRUCTION', result.upper()) else: return Token('LABEL', result) return result
def StateSimpleComment(line, column, text, word): global counter, columna counter += 1 columna += 1 if counter < len(text): if text[counter] != "\n": #IDENTIFICADOR return StateSimpleComment(line, column, text, word + text[counter]) else: Aux = Token(word + text[counter], 'Comentario Unilinea', line, column) counter += 1 columna += 1 return Aux #agregar automata de identificador en el arbol, con el valor else: Aux = Token(word, 'Comentario Unilinea', line, column) counter += 1 columna += 1 return Aux
def insertarToken(self, tipo, lex, fila, columna): #command= messagebox.showinfo(message= "Aceptó " + lex, title="Análisis RMT") self.resultadoToken = self.resultadoToken + "Token->" + str( tipo) + " Lexema->" + self.lexema + "\n" self.tablaToken = self.tablaToken + "<TR><TD>" + tipo + "</TD><TD>" + self.lexema + "</TD><TD>" + str( fila) + "</TD><TD>" + str(columna) + "</TD></TR>" token = Token(tipo, lex, 0, columna) self.listaTokens.append(token) print('Token ' + tipo + ' ' + lex) self.lexema = '' self.estado = 0
async def get_current_user(token: str = Depends(oauth2_scheme)): print(token) credentials_exception = HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Could not validate credentials", headers={"WWW-Authenticate": "Bearer"}, ) return Token.verify_token(token, credentials_exception) # user = db.query(Users).get(token_data['id']) # if user is None: # raise credentials_exception # return user
def make_triple_token(self, first, second, tt_1, tt_2, tt_3): token_type = tt_1 pos_start = self.pos.copy() self.advance() if self.current_char == first: self.advance() token_type = tt_2 elif self.current_char == second: self.advance() token_type = tt_3 return Token(token_type, pos_start=pos_start, pos_end=self.pos)
def StateMultiComment(line, column, text, word): global counter, columna counter += 1 columna += 1 if (counter + 1 < len(text)): if (re.search(r"[\*]", text[counter]) and re.search(r"[\/]", text[counter + 1])): #IDENTIFICADOR Aux = Token(word + text[counter] + text[counter + 1], 'Comentario Multiilinea', line, column) counter += 2 columna += 2 return Aux else: return StateMultiComment(line, column, text, word + text[counter]) #agregar automata de identificador en el arbol, con el valor else: Errores.append(Token(word, 'Error', linea, columna)) counter += 1 columna += 1
def all(self): #it will call function next() repeatedly to implement assert self.__clean token_array = [] while self.__valid(): token = self.next() if token.get_token_name() != 'eof': token_array.append(token) token_array.append( Token('eof', '', self.__line, self.__pos, self.__pos)) return token_array
def make_tokens(self): val = None #while self.current_char != None: if self.current_char != None: if Dicts.type.get(self.current_char, None) != None: val = Token.Token(Dicts.type.get(self.current_char, None), self.current_char) else: self.error = Error.IllegalCharException(self.current_char) val = "err" self.advance() return val
async def send_files_to_file_channel(message, file_channel): al = [] try: for a in message.attachments: al.append(discord.File(Token.get_attachment_file_path(message, a))) m = await file_channel.send(files=al) al = [] for a in m.attachments: al.append(a.url) except Exception as e: print(e) return al
def next(self): # mark it "dirty," can't call "all()" anymore self.__clean = False # skip leading whitespace and comments (if any) while self.__valid(): if self.__char() in blanks: if self.__char() in eol: self.__line += 1 self.__tick() #Find the start of comment by searching (* elif self.__is_start_comment(): self.__comment() else: break # reset token start position self.__start = self.__pos # recognize an actual token now (we hope) if self.__valid(): if self.__char() in letters: token = self.__identifierOrKeyword() elif self.__char() in digits: token = self.__number() elif self.__char() in symbols: token = self.__symbol() else: # nope, something we can't handle... self.__have_error = True self.__error('error: ' + 'Unexpected symbol: ' + self.__char() + ' Line:' + str(self.__line) + ' Position:' + str(self.__pos)) self.__tick() token = Token('eof', '', self.__line, self.__pos, self.__pos) if not self.__valid(): token = Token('eof', '', self.__line, self.__pos, self.__pos) return token
def get_next_token(self): while self.curr_char is not None: if self.curr_char.isspace(): self.skip_whitespace() continue if self.curr_char.isalpha(): return self._id() if self.curr_char.isdigit(): return Token.Token(Token.INTEGER, self.integer()) if self.curr_char == '=': self.advance() return Token.Token(Token.ASSIGN, '=') if self.curr_char == ';': self.advance() return Token.Token(Token.SEMI, ';') if self.curr_char == '+': self.advance() return Token.Token(Token.PLUS, '+') if self.curr_char == '-': self.advance() return Token.Token(Token.MINUS, '-') if self.curr_char == '*': self.advance() return Token.Token(Token.MUL, '*') if self.curr_char == '(': self.advance() return Token.Token(Token.LPAREN, '(') if self.curr_char == ')': self.advance() return Token.Token(Token.RPAREN, ')') self.error() return Token.Token(Token.EOF, None)
def __init__(self, fn, isdebug = 0): self.isdebug = isdebug # 0은 디버깅하지 않겠다는 의미 self.basename = fn[:fn.rfind('.')] self.directive = [] self.token = Token(fn) self.token.nextToken() # Root Symbol Table 등록 self.globalSymbolTable = self.initSymbolTable() self.localSymbolTable = [] # function이나 class앞의 template이나 attribute같은 것들의 정보를 가지고 있는... self.directive = [] # 아무것도 없으면 Root임 self.namespaceStack = [] self.loadedSymbolList = [] self.mustcompile = []
def get(self): # Construct and return the next token in sourceText #--------------------------------------------------------- # PROCESS WHITE SPACE / COMMENTS #--------------------------------------------------------- #ignore any whitespace or comments # print self.character while self.c1 in WHITESPACE_CHARS or self.c2 == "/*": # process whitespace while self.c1 in WHITESPACE_CHARS: token = Token(self.character) token.type = WHITESPACE self.getChar() while self.c1 in WHITESPACE_CHARS: token.cargo += self.c1 self.getChar() # process comments while self.c2 == "/*": # we found comment start token = Token(self.character) token.type = COMMENT token.cargo = self.c2 self.getChar() # read past the first character of a 2-character token self.getChar() # read past the second character of a 2-character token while not (self.c2 == "*/"): if self.c1 == self.ENDMARK: token.abort("Found end of file before end of comment") token.cargo += self.c1 self.getChar() token.cargo += self.c2 # append the */ to the token cargo self.getChar() # read past the first character of a 2-character token self.getChar() # read past the second character of a 2-character token # return token # only if we want the lexer to return comments #--------------------------------------------------------- # END PROCESS WHITE SPACE / COMMENTS #--------------------------------------------------------- # Create a new token. It will remember position and line info from character token = Token(self.character) # End of file if self.c1 == self.ENDMARK: token.type = EOF return token # A variable starts if self.c1 in VAR_START_CHARS: token.type = VAR # get the whole variable self.getChar() while self.c1 in VAR_CHARS: token.cargo += self.c1 self.getChar() # We found a keyword if token.cargo in keywords: token.type = token.cargo return token # A literal starts if self.c1 in LITERAL_START_CHARS: token.type = LITERAL self.getChar() while self.c1 in LITERAL_CHARS: if self.c1 == ',' and self.c2 not in LITERAL_CHARS: break token.cargo += self.c1 self.getChar() return token # A string starts if self.c1 in STRING_DELIMITERS: # remember the quoteChar (single or double quote) # so we can look for the same character to terminate the quote. quoteChar = self.c1 self.getChar() while self.c1 != quoteChar: if self.c1 == self.ENDMARK: token.abort("Found end of file before end of string literal") token.cargo += self.c1 # append quoted character to text self.getChar() token.cargo += self.c1 # append close quote to text self.getChar() token.type = STRING return token # Two char symbols if self.c2 in twoCharSymbols: token.cargo = self.c2 token.type = token.cargo # for symbols, the token type is same as the cargo self.getChar() # read past the first character of a 2-character token self.getChar() # read past the second character of a 2-character token return token if self.c1 in singleCharSymbols: token.type = token.cargo # for symbols, the token type is same as the cargo self.getChar() # read past the symbol return token # Else we found a token we don't recognise, so abort token.abort("Symbol not recognised: " + self.dq(self.c1))
def filter(self, original = NLPInstance): if len(self._allowedStrings) > 0: # first filter out tokens not containing allowed strings old2new = {} new2old = {} tokens = [] for t in original.tokens: stopped = False for property in t.getPropertyTypes(): if stopped: break prop = t.getProperty(property) for allowed in self._allowedStrings: if stopped: break # todo: this can surely be implemented in a nicer way (e.g. no reparsing of interval) if property.name == "Index" and re.match("\d+-\d+", allowed): split = allowed.split("-") From = int(split[0]) to = int(split[1]) for i in range(From, to+1): if(prop == str(i)): newVertex = Token(len(tokens)) newVertex.merge(t) tokens.append(newVertex) old2new[t] = newVertex new2old[newVertex] = t stopped = True break else: if self._wholeWord: b = prop == allowed else: b= allowed in prop if b: newVertex = Token(len(tokens)) newVertex.merge(t) tokens.append(newVertex) old2new[t] = newVertex new2old[newVertex] = t stopped = True break # update edges and remove those that have vertices not in the new vertex set edges = [] for e in original.getEdges(): if e.From not in old2new or e.To not in old2new: continue newFrom = old2new[e.From] newTo = old2new[e.To] edges.append((Edge(From=newFrom, To=newTo, label=e.label, note=e.note, Type=e.type, renderType=e.renderType, description=e.description))) # find new split points (have to be changed becouse instance has new token sequence) splitPoints = [] newTokenIndex = 0 for oldSplitPoint in original.splitPoints: newToken = tokens[newTokenIndex] oldToken = new2old[newToken] while newTokenIndex + 1 < len(tokens)and oldToken.index < oldSplitPoint: newTokenIndex += 1 newToken = tokens[newTokenIndex] oldToken = new2old[newToken] return NLPInstance(tokens=self.filterTokens(tokens), edges=edges, renderType=original.renderType, splitPoints=splitPoints) else: filteredTokens = self.filterTokens(original.tokens) return NLPInstance(tokens=filteredTokens, edges=original.getEdges(), renderType=original.renderType)
res = try_brower(browser, url) fres = res.read() jdata = json.loads(fres) fres = json.dumps(jdata, ensure_ascii=False) FileUtility.write("data/posts/"+post_id+"/likes.json",fres) url = 'https://graph.facebook.com/'+post_id+'/comments'+ ('&access_token=%s' % access_token) res = try_brower(browser, url) fres = res.read() jdata = json.loads(fres) fres = json.dumps(jdata, ensure_ascii=False) FileUtility.write("data/posts/"+post_id+"/comments.json",fres) if __name__ == '__main__': # Initialize the needed modules CHandler = urllib2.HTTPCookieProcessor(cookielib.CookieJar()) browser = urllib2.build_opener(CHandler) browser.addheaders = [('User-agent', 'InFB - [email protected] - http://ruel.me')] urllib2.install_opener(browser) FileUtility.user = '******' FileUtility.make_data_path() user = '******' passw = 'plumggmtutu' token = Token.get(user, passw) fetch_core('100000154563058_407470182627805', token, browser)
def addTokenWithProperties(self, *properties): token = Token(len(self._tokens)) for prop in properties: token.addProperty(prop) self._tokens.append(token) self._map[token.index] = token
def __init__(self, fn): self.token = Token(fn) self.token.nextToken()
class Parser(): def __init__(self, fn): self.token = Token(fn) self.token.nextToken()
class Parser: def __init__(self, fn, isdebug = 0): self.isdebug = isdebug # 0은 디버깅하지 않겠다는 의미 self.basename = fn[:fn.rfind('.')] self.directive = [] self.token = Token(fn) self.token.nextToken() # Root Symbol Table 등록 self.globalSymbolTable = self.initSymbolTable() self.localSymbolTable = [] # function이나 class앞의 template이나 attribute같은 것들의 정보를 가지고 있는... self.directive = [] # 아무것도 없으면 Root임 self.namespaceStack = [] self.loadedSymbolList = [] self.mustcompile = [] def initSymbolTable(self): symtbl = SymbolTable() symtbl.register({'@type':'namespace','@name':"System"}) symtbl.register({'@type':'namespace','@name':"System.lang"}) symtbl.register({'@type':'namespace','@name':"System.out"}) namespaceObject = "System.lang.Object" namespaceByte = "System.lang.Byte" namespaceChar = "System.lang.Char" namespaceShort = "System.lang.Short" namespaceInt = "System.lang.Integer" namespaceLong = "System.lang.Long" namespaceFloat = "System.lang.Float" namespaceDouble = "System.lang.Double" namespaceString = "System.lang.String" namespaceBoolean = "System.lang.Boolean" namespaceArray = "System.lang.Array" # System.lang.Object symtbl.register({'@type':'class','@name':namespaceObject}) symtbl.register({'@type':'class','@name':namespaceByte}) symtbl.register({'@type':'class','@name':namespaceChar}) symtbl.register({'@type':'class','@name':namespaceShort}) symtbl.register({'@type':'class','@name':namespaceInt}) symtbl.register({ '@type':'native def', '@name':namespaceInt + '.=', '@args':[ASTType(namespaceInt)], '@vtype':ASTType(namespaceInt), '@method':lambda se,dst: ASTNativeMove(se, dst)}) symtbl.register({ '@type':'native def', '@name':'+', '@args':[ASTType(namespaceInt), ASTType(namespaceInt)], '@vtype':ASTType(namespaceInt), '@method':lambda src1,src2: ASTNativeAdd(src1, src2)}) symtbl.register({ '@type':'native def', '@name':'+=', '@args':[ASTType(namespaceInt), ASTType(namespaceInt)], '@vtype':ASTType(namespaceInt), '@method':lambda se,dst: ASTNativeMove(se, ASTNativeAdd(src, dst))}) symtbl.register({'@type':'class','@name':namespaceLong}) symtbl.register({'@type':'class','@name':namespaceFloat}) symtbl.register({'@type':'class','@name':namespaceDouble}) symtbl.register({'@type':'class','@name':namespaceString}) symtbl.register({'@type':'class','@name':namespaceBoolean}) symtbl.register({'@type':'class','@name':namespaceArray}) symtbl.register({ '@type':'def', '@name':"System.lang.Array.length", '@args':None, '@vtype':ASTType(name="System.lang.Integer", templ = None, ranks = None)}) symtbl.register({ '@type':'def', '@name':"System.lang.Array.toRange", '@args':None, '@vtype':ASTType(name="System.lang.Array", templ = None, ranks = None)}) symtbl.register({ '@type':'def', '@name':"System.lang.Array.getNext", '@args':None, '@vtype':ASTType(name="System.lang.Integer", templ = None, ranks = None)}) symtbl.register({ '@type':'def', '@name':"System.lang.Array.end", '@args':None, '@vtype':ASTType(name="System.lang.Boolean", templ = None, ranks = None)}) symtbl.register({'@type':'alias', '@name':"object", '@fullname':namespaceObject}) symtbl.register({'@type':'alias', '@name':"byte", '@fullname':namespaceByte}) symtbl.register({'@type':'alias', '@name':"short", '@fullname':namespaceShort}) symtbl.register({'@type':'alias', '@name':"int", '@fullname':namespaceInt}) symtbl.register({'@type':'alias', '@name':"float", '@fullname':namespaceFloat}) symtbl.register({'@type':'alias', '@name':"double", '@fullname':namespaceDouble}) symtbl.register({'@type':'alias', '@name':"string", '@fullname':namespaceString}) symtbl.register({'@type':'alias', '@name':'bool', '@fullname':namespaceBoolean}) symtbl.register({'@type':'alias', '@name':"string", '@fullname':namespaceString}) symtbl.register({ '@type':'def', '@name':'System.out.println', '@args':[ASTType(name=namespaceString, templ = None, ranks = None)], '@vtype':ASTType(name="void", templ = None, ranks = None)}) return symtbl def nextToken(self): self.token.nextToken() def match(self, word): return self.token.match(word) def same(self, word): return self.token.same(word) def matchType(self, word): return self.token.matchType(word) def sameType(self, word): return self.token.sameType(word) def isEnd(self): return self.token.reachEnd() def getTokValue(self): return self.token.tok.value def getTokType(self): return self.token.tok.type def getName(self): if self.match('_'): return '_' return self.token.matchType('id') def getNames(self): names = [] while not self.isEnd(): names.append(self.getName()) if not self.match('.'): break return ".".join(names) def parse(self): if self.isdebug == 1: print "entering parse" parsingList = [] while not self.isEnd(): tree = None if self.same('namespace'): self.parseNamespace() elif self.same('class'): self.parseClass() raise Exception("parse", "class") elif self.same('template'): result = self.parseTemplate() self.directive.append(result) elif self.same('@'): result = self.parseAttribute() self.directive.append(result) elif self.same('def'): self.parseDef() elif self.same('native'): # 예는 push로.. self.directive.append('native') pass else: break if self.isdebug == 1: print "ending parse" def parseNamespace(self): if not self.match('namespace'): return path = self.getNames() self.namespaceStack.append(path) self.loadedSymbolList.append(set([])) self.parseNamespaceBody() self.loadedSymbolList.pop() self.namespaceStack.pop() # symbol search할때도 사용할예정 def getWorkingPath(self): return ".".join(self.namespaceStack) def parseNamespaceBody(self): if not self.match('{'): return self.parse() self.match('}') def parseClass(self): if not self.match('class'): return names = self.getNames() classname = ".".join(self.namespaceStack + [names]) # 검색 symbol list에 등록만 해놓는다. # 정의가 안되어 있다가 나중에 사용되면 실제 symbol table에 body가 없으므로, # body가 없다고 에러를 내면 된다. # self.loadedSymbolList와 self.namespaceStack은 단지 symbol을 만들때와 symbol참조를 위해서만 쓰인다. self.loadedSymbolList[-1] |= set([classname]) if self.match(';'): return self.namespaceStack.push(name) body = self.parseClassBody() self.namespaceStack.pop() symtbl = self.getRecentSymbolTable() # class의 body는 variable만 있어야 한다. # class의 method들은 symbol로 등록될 것 이다. symtbl.registerSymbol({"@type": "class", "@name": classname, "@attribute": None, "@body": body}) def parseClassBody(self): if not self.match('{'): return body = {} while not self.match('}'): if self.match('val'): # 상수선언 name = self.getName() if body.has_key(name): print "Error) duplicated name :", name raise NameError content = {"@type": "val", "@vtype": ASTType("System.lang.Integer")} if self.match(':'): content['@vtype'] = self.parseType() if self.match('='): content['@init'] = self.parseInitExpr() body[name] = content elif self.match('var'): # 변수선언 name = self.getName() if body.has_key(name): print "Error) duplicated name :", name raise NameError content = {"@type": "var", "@vtype": ASTType("System.lang.Integer")} if self.match(':'): content['@vtype'] = self.parseType() if self.match('='): content['@init'] = self.parseInitExpr() body[name] = content elif self.match('def'): # 함수 name = self.getName() # 인자까지 봐야지 중복인지를 체크할 수 있음 content = {"@type": "def"} if self.match('('): args = self.parseDefArgsList() if not self.match(')'): print "Error) Needed ')'" raise SyntaxError content['@args'] = args if self.match(':'): # return type type = self.parseType() content['@vtype'] = type else: content['@vtype'] = None # return이 없음을 의미 (c의 void) if self.match('='): defbody = self.parseExpr() content['@body'] = defbody elif self.match('{'): defbody = self.parseExprs() if not self.match('}'): print "Error) Needed '}'" raise SyntaxError content['@body'] = defbody else: print "Error) Needed Body" raise SyntaxError # 함수이름을 native symbol로 변경 realn = convertToNativeSymbol(name, content['@args'], content['@vtype']) # TODO : Auto Casting은 일단 지원하지 않는다. if body.has_key(realn): print "Error) Multiple declaration :", fname raise SyntaxError body[realn] = content return body def parseInitExpr(self): # 여긴 상수나 간단한 계산하는 루틴정도? # 아님 배열 raise NotImplementedError def parseAttribute(self): if not self.match('@'): return pass def parseTemplate(self): if not self.match('template'): return None params = self.parseTemplateArguments() if params == None: print "Error) Needs some template parameters" return for param in params: sym.registerTemplateVariable(param.name, param.type) if self.same('class'): target = self.parseClass() elif self.same('def'): target = self.parseDef() self.globalSymbolTable.register({ "@type": 'template def', "@name": target['name'], "@vtype": target['rettype'], "@body": target['body'], "@template args": params}) # TODO : 그리고 먼가 파일로 만드는 코드 추가 else: print "Error) Dont use template in this type" raise Exception("parseTemplate", "wrong type") self.pop() return ASTTemplate(params, target) def parseTemplateArguments(self): if not self.match('<'): return None # 선언할때는 function argument처럼 args = [self.parseTemplateArgument()] while self.match(','): args.append(self.parseTemplateArgument()) if not self.match('>'): return None return args def parseTemplateArgument(self): name = self.getName() type = "typename" # 일단 임시로 if self.match(':'): type = self.getNames() return ASTTemplateArg(name, type) # 함수 선언 형태 # def func(argname1: argtype1, argname2: argtype2, ...) = expr # def func(argname1: argtype1, argname2: argtype2, ...) { exprs } # def func(argname1: argtype1, argname2: argtype2, ...):rettype = expr # def func(argname1: argtype1, argname2: argtype2, ...):rettype { exprs } # 변수 선언 : (Scala처럼 구분하지 않는게 좋을듯) # def variableName; // 이럴 경우 정수형으로 가정 # def variableName:variableType; # def variableName = <initial expr>; // 이거랑 # def variableName:variableType = <initial expr>; // 이거는 함수취급 def makeFullPath(self, fn): return ".".join(self.namespaceStack + [fn]) def parseDefBody(self): if self.isdebug == 1: print "entering parseDefBody" body = None if self.isdebug == 1: print "getTokValue : %s" % (self.getTokValue()) if self.match('='): body = self.parseExpr() elif self.match('{'): body = self.parseExprs() self.match('}') if self.isdebug == 1: print "ending parseDefBody" return body def parseDef(self): if not self.match('def'): return None if self.isdebug == 1: print "entering parseDef" # 이름을 얻습니다. only = self.getNames() fn = ".".join(self.namespaceStack + [only]) #print "Function name : %s" % (fn) # 함수용 local symbol table을 만듭니다. self.localSymbolTable = [{}] # argument가 나오는지 검사합니다. args = self.parseDefArgsList() # check localSymTbl = self.localSymbolTable[-1] for arg in args: if localSymTbl.has_key(arg.name): print "Error) Duplicated Name" raise SyntaxError if not self.globalSymbolTable.findType(arg.type.name): print "Error) Unknown Type" raise SyntaxError localSymTbl[arg.name] = arg.type nativeSymbol = mangling(fn, args) if self.globalSymbolTable.find({'@type': 'def', '@name': fn, '@args': args}): print "Error) Duplicated Name" raise Exception("Error", "Error") #localSymTbl.printDoc() # To parse return type rettype = self.parseReturnType() # To parse body of function body = self.parseDefBody() if body == None: print "Error) Body Empty : in %s" % (nativeSymbol) raise Exception("Error", "Error") if rettype != 'void': if isinstance(body, ASTExprs) or isinstance(body, ASTSimpleExprs): # return을 명시적으로 적지 않았다면, 마지막 expression의 결과를 return값으로 한다. lastExpr = body.exprs[-1] if not isinstance(lastExpr, ASTReturn): body.exprs[-1] = ASTReturn(lastExpr) else: # isinstance(body, ASTExpr): body = ASTExpr(ASTReturn(body)) print "&&=", type(rettype) print "**=", body # 바로전에 template이 선언되었다면 여기도 영향을 받아야만 한다. # 일단 지금은 영향을 받지 않는다고 가정한다. self.globalSymbolTable.register({ "@type": "def", "@name": fn, "@args": args, "@vtype": rettype, "@body": body, "@symbols": self.localSymbolTable}) self.localSymbolTable = [{}] #print "1", nativeSymbol, self.globalSymbolTable[nativeSymbol] self.mustcompile.append((self.globalSymbolTable[nativeSymbol], nativeSymbol)) if self.isdebug == 1: print "ending parseDef" def parseDefArgsList(self): if not self.match('('): return None args = [] while not self.isEnd(): arg = self.parseDefArg() if arg == None: break args.append(arg) if not self.match(','): break if not self.match(')'): print "Error) Needed ')'" return None return args def parseReturnType(self): # if return type is none, if not self.match(':'): return ASTType(name = "System.lang.Integer", templ = None, ranks = None) return self.parseType() def parseDefArg(self): name = self.getName() if name == None: return None typeStr = ASTType(name = "System.lang.Integer", templ = None, ranks = None) if self.match(':'): typeStr = self.parseType() defval = None if self.match('='): defval = self.parseBasicSimpleExpr() # if typeStr == None: makeError return ASTDefArg(name = name, type = typeStr, defval = defval) def parseTemplatePart(self): #raise Exception('parseTemplatePart', 'Not Implemented') return None def matchTemplateInfo(self, typeInfo, templateInfo): #raise Exception('matchTemplateInfo', 'Not Implemented') return True def parseType(self): #if self.isdebug == 1: print "starting parseType" idStr = self.getNames() if self.isdebug == 1: print ".".join(idStr) # 해당 type이 존재하는지 검사합니다. tp = self.globalSymbolTable.findType(idStr) if tp == None: print "Unknown Type : %s" % (idStr) sys.exit(-1) if tp == 'alias': idStr = self.globalSymbolTable.find(idStr) #print "(", idStr #tmpl = self.parseTemplatePart() #if not self.matchTemplateInfo(result, tmpl): # # 일단 현재는 pass # print "Error) Not matched template information" # pass #print "type's full name = %s" % (idStr) #tmpl = self.parseTemplatePart() #ename, body = symbolTable.search(names.array) #if ename == None: # print "doesn't exist symbol : %s" % (".".join(names.array)) # sys.exit(-1) # 일단 죽이고... 나중에 에러처리 생각 #else: # names.array = ename rank = self.parseRankList() if self.isdebug == 1: print "ending parseType" return ASTType(name = idStr, templ = None, ranks = rank) def parseRankList(self): lst = [] while self.match('['): rank = ASTRank(self.parseSimpleExpr()) lst.append(rank) if not self.match(']'): print "Error) Need ']'" return ASTRankList(lst) def parseExprs(self): lst = [] while not self.isEnd(): ret = self.parseExpr() if ret == None: break if isinstance(ret, ASTExprs): lst += ret.exprs elif isinstance(ret, list): lst += ret else: # ?? lst.append(ret) if len(lst) == 0: return None return ASTExprs(lst) def parseExpr(self): ret = None if self.same('if'): ret = self.parseIfStmt() elif self.same('for'): ret = self.parseForStmt() elif self.same('var'): ret = self.parseVar() elif self.same('val'): ret = self.parseVal() elif self.same('{'): ret = self.parseBlockExprs() else: ret = self.parseSimpleExpr1() #print "***",ret self.match(';') #s = raw_input() return ret def parseIfStmt(self): if not self.match('if'): return None cond = self.parseExpr() self.match(':') body = self.parseExpr() return ASTIf(cond, body) def parseForStmt(self): if not self.match('for'): return None cond = self.parseBasicSimpleExpr() if cond == None: print "Error) Needed to identifier" raise SyntaxError if not self.match('<='): print "Error) Needed to <=" raise SyntaxError generator = self.parseSimpleExpr() if generator == None: print "Error) Needed generator" raise SyntaxError body = None if self.match(':'): body = self.parseExpr() elif self.match('{'): body = self.parseExprs() self.match('}') else: print "Error) Needed '{' '}' or '='" raise NotImplementedError return ASTFor(cond, generator, body) def convertToASTType(self, obj): if isinstance(obj, ASTType): return obj elif isinstance(obj, ASTListGenerateType1): return self.convertToASTType(obj.start) elif isinstance(obj, ASTWord) and obj.vtype != None: if isinstance(obj.vtype, ASTType): return obj.vtype # 이건 비정상적인 경우, 이렇게 찾아들어오면 안된다. elif isinstance(obj.vtype, dict): return obj.vtype['@vtype'] else: print "))", obj.vtype raise NotImplementedError elif isinstance(obj, ASTWord) and isinstance(obj.type, ASTType): return obj.type elif isinstance(obj, ASTListGenerateType1): return ASTType('System.lang.Array') elif isinstance(obj, ASTCalleeArgType1): return self.convertToASTType(obj.type) else: print "**", obj raise NotImplementedError def parseVar(self): if not self.match('var'): return None sym = self.localSymbolTable[-1] hist = [] while True: name = self.getName() if sym.has_key(name): print "has duplicated name" raise Exception('Error', 'Duplicated Name') return None type = None if self.match(':'): type = self.parseType() else: type = ASTType(name = 'System.lang.Integer', templ = None, ranks = None) #print "name =", name # 변수 초기화 tree = None if self.match('='): right = self.parseSimpleExpr() query = {"@name": '=', '@type': 'def'} query['@args'] = [type, self.convertToASTType(right)] symbol = self.globalSymbolTable.find(query) #print "2", symbol tree = ASTOperator(ASTWord('id', '='), ASTWord('id', name, type), right) hist.append(tree) sym[name] = {"@type": "var", "@vtype": type} if not self.match(','): break self.match(';') return hist def parseVal(self): if not self.match('val'): return None sym = self.localSymbolTable[-1] hist = [] while True: name = self.getName() if sym.has_key(name): print "has duplicated name" raise Exception('Error', 'Duplicated Name') return None type = None if self.match(':'): type = self.parseType() else: type = ASTType(name = 'System.lang.Integer', templ = None, ranks = None) #print "name =", name # 변수 초기화 tree = None if self.match('='): right = self.parseSimpleExpr() query = {"@name": '=', '@type': 'def'} query['@args'] = [type, self.convertToASTType(right)] symbol = self.globalSymbolTable.find(query) print "3", symbol tree = ASTOperator(ASTWord('id', '='), ASTWord('id', name, type), right) hist.append(tree) sym[name] = {"@type": "var", "@vtype": type} if not self.match(','): break self.match(';') return hist def parseBlockExprs(self): return None def parseSimpleExpr1(self): ret = self.parseSimpleExprs() # not yet! #if self.match('?'): # body = self.parseMatchingCases() # ret = ASTPatternMatch(cond = ret, body = body) return ret def parseSimpleExprs(self): history = [] while not self.isEnd(): tree = self.parseSimpleExpr() if tree == None: break if self.match(','): hist = [tree] while self.match(','): tree = self.parseSimpleExpr() hist.append(tree) tree = ASTSet(hist) history.append(tree) nhist = len(history) print "nhist = ", nhist if nhist == 0: return None elif nhist == 1: return history[0] #self.match(';') # caution!! return ASTSimpleExprs(history) def parseSimpleExpr(self): if self.isdebug == 1: print "entering parseSimpleExpr()" tree = self.parseBasicSimpleExpr() if tree == None: return None while not self.isEnd(): if self.isdebug == 1: print self.getTokValue() if self.match('.'): right = self.parseBasicSimpleExpr() if isinstance(tree, ASTWord): if isinstance(right, ASTWord): tree = ASTNames([tree.value, right.value]) elif isinstance(right, ASTFuncCall): tree = ASTFuncCall(ASTNames([tree.value, right.name.value]), right.body) elif isinstance(right, ASTIndexing): tree = ASTIndexing(ASTNames([tree.value, right.name.value]), right.history) elif isinstance(tree, ASTNames): if isinstance(right, ASTWord): tree = ASTNames(tree.array + [right.value]) elif isinstance(right, ASTFuncCall): tree = ASTFuncCall(ASTNames(tree.array + [right.name.value]), right.args) elif isinstance(right, ASTIndexing): tree = ASTIndexing(ASTNames(tree.array + [right.name.value]), right.history) else: tok = self.token.tok # Global Operator 함수로 첫번째 찾는다. (C++의 operator + (left, right)라는 식..) content = {'@type': 'def', '@name': tok.value} content['@args'] = [self.convertToASTType(tree), self.convertToASTType(right)] symbol = self.globalSymbolTable.find(content) if symbol == None: # 없다면, left.type의 operator로 찾는다. (C++의 someclass::operator + (right)...) content = {'@type': 'def', '@name': self.convertToASTType(tree).name + "." + tok.value} content['@args'] = [self.convertToASTType(right)] symbol = self.globalSymbolTable.find(content) print "4", symbol, content if symbol != None: if symbol['@type'] == 'native def': raise NotImplementedError else: tree = ASTFuncCall(content['@name'], tree, right) elif symbol == None: tree = ASTOperator(ASTWord(tok.type, tok.value), tree, right) # array elif self.sameType('id'): #if isinstance(tree, ASTSet): # #if len(tree.lst) != 1: # # print "error!!" # make error!! # if self.checktype(tree.lst[0]): # tree = ASTCasting(tree.lst[0], ASTWord(tok.type, tok.value)) tokVal = self.getTokValue() tokType = self.getTokType() mid = ASTWord(tokType, tokVal) self.token.nextToken() right = self.parseBasicSimpleExpr() #print "here : ", mid, tree, right if right != None: content = {'@type': 'def', '@name': tokVal} content['@args'] = [self.convertToASTType(tree), self.convertToASTType(right)] symbol = self.globalSymbolTable.find(content) if symbol == None: # 없다면, left.type의 operator로 찾는다. (C++의 someclass::operator + (right)...) content = {'@type': 'def', '@name': self.convertToASTType(tree).name + "." + tokVal} content['@args'] = [self.convertToASTType(right)] symbol = self.globalSymbolTable.find(content) if symbol != None: if symbol['@type'] == 'native def': raise NotImplementedError else: tree = ASTFuncCall(content['@name'], [tree, right]) else: tree = ASTOperator(mid, tree, right) else: # for example, 'a++' or 'a+' tree = ASTUnary(tree, mid) else: break if isinstance(tree, ASTFuncCall): candidates = set([]) path = None if isinstance(tree, ASTNames): path = ".".join(tree.array) elif isinstance(tree.name, ASTNames): path = ".".join(tree.name.array) else: path = tree.name ret = self.globalSymbolTable.find({'@type':'def', '@name':path, '@args':map(lambda x: self.convertToASTType(x), tree.args)}) if ret == None: print "Error) Not Symbol :", path, map(lambda x: self.convertToASTType(x), tree.args) raise SyntaxError if self.isdebug == 1: print "ending parseSimpleExpr()" return tree def parseBasicSimpleExpr(self): tok = self.token.tok if tok == None: return None #print "calling parseBasicSimpleExpr" #print "value =", tok.value, tok.type if self.matchType('stringLiteral'): return ASTWord(ASTType('System.lang.String'), tok.value) elif self.matchType('integerLiteral'): return ASTWord(ASTType('System.lang.Integer'), tok.value) elif self.matchType('floatLiteral'): return ASTWord(ASTType('System.lang.Float'), tok.value) elif self.match('true'): return ASTWord(ASTType('System.lang.Boolean'), '1') elif self.match('false'): return ASTWord(ASTType('System.lang.Boolean'), '0') elif self.match('return'): #print "entering return" expr = self.parseSimpleExpr() #print "@@", expr return ASTReturn(expr) #elif self.match('def'): # ret = self.parseDefInnerFunc() #if len(ret.name) != 1: # print "don't use namespace!" # sys.exit(-1) #realname = ret.name[0] #if realname == '_': # realname = self.genTemporaryName() #if self.findAt(tbl = self.local_symtbl, target = ret.name): # print "already defined!" # sys.exit(-1) #typename = convertType(ret.ret) #if not self.validateType(typename): # print "not declare type" # sys.exit(-1) #self.local_symtbl[realname] = { # "attribute": ["lambda"], # "args": ret.args, # "type": typename, # "body": ret.body} #return ret elif self.matchType('id'): if self.same('['): history = [] while self.match('['): history.append(self.parseSimpleExpr()) self.match(']') return ASTIndexing(ASTWord(tok.type, tok.value), history) elif self.match('('): # TODO 함수의 그것인지 아닌지에 대한 구분이 필요하다. args = self.parseDefArgListForFuncCall() if not self.match(')'): print "Error) Need ')'" # TODO: 호출하려는 function에 대한 정보를 얻어서 입력된 argument들의 type과 비교하여, # 현재 symbol table에 호출할 수 있는 function이 있는지를 찾는 코드가 있어야 한다. # 몇번의 try가 필요할지도...(auto casting때문에...) # 예를 들어, 호출하려는 함수의 인자중에 char*를 갖는데, 해당 함수의 인자에는 char*를 사용하지 않고 System.lang.String만 사용하는 경우는 # 자동으로 char*를 String으로 auto casting해주어야 한다. # 그 반대의 경우는 String을 char*로 casting해주어야 한다. # 단, 해당 class에 해당 변환을 지원해준다는 가정이 필요하다.(즉, 해당 casting을 지원해주는지 여부를 체크해야만 한다.) # 알고리즘 # 1. 현재의 argument들의 type들로 구성된 function을 찾는다. # 2. 만일 없다면, argument들의 갯수가 동일한 함수.... (이건 내일 생각) return ASTFuncCall(ASTWord(tok.type, tok.value), args) elif self.match('...'): right = self.parseSimpleExpr() return ASTListGenerateType1(ASTWord(tok.type, tok.value), right) # 여기서 빠진 것은 Arrya<T이어야 한다는 사실(Type이 빠졌다는 소리) else: vtype = None for symbolTable in reversed(self.localSymbolTable): if symbolTable.has_key(tok.value): vtype = symbolTable[tok.value] break if vtype == None: vtype = self.globalSymbolTable.findType(tok.value) return ASTWord(tok.type, tok.value, vtype) elif self.match('_'): return ASTWord('v', tok.value) elif self.match('['): history = [] tree = self.parseSimpleExpr() if self.match('...'): right = self.parseSimpleExpr() self.match(']') return ASTListGenerateType1(tree, right) elif self.match(','): history.append(tree) while self.match(','): item = self.parseSimpleExpr() history.append(item) self.match(']') return ASTListValue(history) self.match(']') return ASTListValue([tree]) elif self.match('('): tree = self.parseSimpleExpr1() self.match(')') return ASTWrap(tree) #else: # print tok # raise Exception("parseBasicSimpleExpr", "Not implemented") return None def parseDefArgListForFuncCall(self): args = [] while True: arg = self.parseSimpleExpr() if isinstance(arg, ASTWord): if arg.type == 'id': symtbl = self.localSymbolTable[-1] if not symtbl.has_key(arg.value): print "Error) Not found :", arg.value raise SyntaxError args.append(ASTCalleeArgType1(value = arg, type = symtbl[arg.value])) else: args.append(ASTCalleeArgType1(value = arg, type = arg.type)) else: print arg raise NotImplementedError if not self.match(','): break return args
eojeol_size = len(parsed_eojeol) for index, (word, type, pos) in enumerate(parsed_eojeol): util.Logger.debug(id, word.encode('utf-8'), pos) eoe = False # end_of_eojeol eos = False # end_of_sentence if index+1 == eojeol_size: eoe = True eos = _eos else: eos = False util.Logger.debug(word.encode('utf-8')+'/'+type+'-'+pos, eoe, eos) tokens.append(token.Token(id, word.encode('utf-8'), type, pos, eoe, eos)) id += 1 index += 1 ej_index += 1 file.close() return tokens if __name__ == "__main__": import sys if len(sys.argv) != 2: print "python Tokenizer.py filename" print "python Tokenizer.py /home/psyoblade/workspace/corpus/sejong/tagged/raws/Spoken/Quasi/1/AH000475.TMP.raws" sys.exit() tokenizer = Tokenizer() file = open(sys.argv[1]) for token in tokenizer.tokenize(file): token.debug() file.close()
elif message.content.startswith('$mtgtext'): card_id = MagicCard.card_check(message.content[9:]) if card_id: text = MagicCard.card_text(card_id) await client.send_message(message.channel, text) elif message.content.startswith('$mtgprice'): card_id = MagicCard.card_check(message.content[10:]) if card_id: price = MagicCard.card_price(message.content[10:]) await client.send_message(message.channel, price) elif message.content.startswith('$mtg'): card_id = MagicCard.card_check(message.content[5:]) if card_id: reply = MagicCard.card_text(card_id) reply += MagicCard.card_price(message.content[5:]) imgname = MagicCard.card_image(card_id) await client.send_file(message.channel, imgname, content=reply) @client.event async def on_ready(): print('Logged in as') print(client.user.name) print(client.user.id) print('------') client.run(Token.get_token())
def nextToken(self): tk = "" if self._saveToken != None: tk = self._saveToken self._saveToken = None return tk ch = self._readCharacter() try: while re.match(r"\s", ch): ch = self._readCharacter() except: pass if ch == "": print "Empty string" pass if re.match("[a-zA-Z]", ch): tk = self._readSymbol(ch) elif re.match("[0-9]", ch): tk = self._readLiteral(ch) elif re.match("#", ch): tk = ch while re.match("[^;]", ch): ch = self._readCharacter() tk += ch if tk == "#debugon;": print "debugging" self.debugOn() elif tk == "#debugoff;": self.debugOff() elif tk == "#dump;": self._symbols.dump() elif tk == "#EOF;": return Token(TokenType.END_FILE, 'EOF') else: self.setWarning("Invalid Pragma, " + tk + " on line number " + str(self.getLineNumber())) return self.nextToken(); elif re.match(r"[<>!=+\-\*\(\)\{\}\[\],\.;]", ch): tk = ch ch = self._readCharacter() if re.match("[<>!=]=", tk + ch): tk += ch else: self._unreadCharacter() op = self._operators[tk] tk = Token(op.getTokenType(), op.toString()) print tk.toString() elif re.match("/", ch): tk = ch ch = self._readCharacter() if re.match("//", tk + ch): next = self._readCharacter() while next != '\n': next = self._readCharacter() tk = self.nextToken() elif re.match(r"/\*", tk +ch): this = self._readCharacter() next = self._readCharacter() while this != '*': this = next next = self._readCharacter() tk = self.nextToken() else: self._unreadCharacter() op = self._operators[tk] tk = Token(op.getTokenType(), op.toString()) elif re.match("'", ch): ch = self._readCharacter() throwaway = self._readCharacter() tk = Token(TokenType.LITERAL, str(ord(ch))) else: print "EOF-EOF-EOF" tk = self._readEOF() self._saveToken = None if tk != "" and tk != None: return tk else: raise CompilerException("Error: Invalid Token" + ch + self.fln())