def __init__(self): for keyword in keywords: token_id = Tokens.get_id(keyword) if token_id != -1: self.install_keyword(keyword, token_id) for standard_id in standard_identifiers: self.install_id(standard_id, 0, 0)
def main(argv): try: nameOfFile = argv[-1] except IndexError: nameOfFile = "input.txt" contentOfFile = Utils.loadStringFromFile(nameOfFile) lexerAnalyser = LexerAnalyser(contentOfFile) listOfTokens = lexerAnalyser.returnListOfTokens() tokens = Tokens(listOfTokens) tree = E(tokens) Utils.writeInFile(tree.dotStr()) print("=D")
def add_terms_weight(self): con = DB.connect() table = self._table cur = con.cursor() email_count = Emails.get_email_count() dictionary = Dictionary.fetch_all() tokens = Tokens.fetch_token_all() for token in tokens: tfidf = self.calc_tfidf(email_count, token[Tokens.tf], dictionary[token[Tokens.term]]) cur.execute("INSERT INTO %s VALUES (%ld, %s, %d, %s)" % (table, token[Tokens.email_id], token[Tokens.term], tfidf, token[Tokens.classs])) DB.close(con)
def __init__(self): self.oracionID = uuid4() self.oracion = None self.tokens = Tokens() self.arbol = []
class Parser(object): # Constructeur def __init__(self): self.liste = Tokens() self.ast = Ast() self.indentation = 0 self.nb_errors = 0 # Destructeur def terminer(self): self.ast.terminer() # Printeur def __str__(self): return "it's me, Mario" # Tokenizer def read_tokens(self, file_path): fichier = open(file_path) lecteur = fichier.readline g = tokenize.generate_tokens(lecteur) for _, token, coord, _, _ in g: if token == "\n": continue self.liste.add(token, coord[0], coord[1]) print self.liste # Commencer l'analyse def analyse(self): self.ast.add_brain(self.parse_brain()) print "Nombre d'erreurs de parsing :\t", self.nb_errors self.ast.visiter() # Verifier si le token courant est bien de ce type et passer au suivant def expect(self, kind): token = self.liste.get() if re.search(kind, token): sys.stdout.write(self.liste.pick()) sys.stdout.write(" ") return token else: self.endl() self.nb_errors += 1 x, y = self.liste.coord() print "[Error] [" + str(x) + ";" + str(y) + "] Expected token : " + kind + ", get : " + token # Tester si le token actuel est de ce type def test(self, kind): return re.search(kind, self.liste.get()) # Tester si le token suivant est de ce type def test_plus_1(self, kind): return re.search(kind, self.liste.get_plus_1()) # Ajouter une indentation def indent(self): self.indentation += 1 # Supprimer une indentation def desindent(self): self.indentation -= 1 # Nouvelle ligne def endl(self): sys.stdout.write("\n") for i in range(self.indentation): sys.stdout.write("\t") ######### Grammaire ... # Bloc principal de cerveau def parse_brain(self): self.expect(d.brain) token = self.expect(d.identifier) brain = T_brain(self.ast, token) self.expect(d.laccol) self.indent() self.endl() continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.var): brain.add_b_var(self.parse_block_variables()) elif self.test(d.layers): brain.add_b_layers(self.parse_block_layers()) elif self.test(d.inp): brain.add_b_input(self.parse_block_input()) elif self.test(d.out): brain.add_b_output(self.parse_block_output()) elif self.test(d.obj): brain.add_b_goal(self.parse_block_goal()) elif self.test(d.struct): brain.add_b_structure(self.parse_block_structure()) elif self.test(d.interf): brain.add_b_interface(self.parse_block_interface()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return brain # Bloc declaration de variables def parse_block_variables(self): self.expect(d.var) self.expect(d.laccol) self.indent() self.endl() b_var = T_block_vars(self.ast, "variables") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.integer): b_var.add_int(self.parse_declaration_int()) elif self.test(d.scale): b_var.add_scale(self.parse_declaration_scale()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_var # Bloc declaration d'inputs def parse_block_input(self): self.expect(d.inp) self.expect(d.laccol) self.indent() self.endl() b_input = T_block_inputs(self.ast, "inputs") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.identifier): b_input.add_input(self.parse_declaration_ioc()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_input # Bloc declaration d'outputs def parse_block_output(self): self.expect(d.out) self.expect(d.laccol) self.indent() self.endl() b_output = T_block_outputs(self.ast, "outputs") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.identifier): b_output.add_output(self.parse_declaration_ioc()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_output # Bloc declaration d'objectifs def parse_block_goal(self): self.expect(d.obj) self.expect(d.laccol) self.indent() self.endl() b_goal = T_block_goal(self.ast, "objectifs") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.identifier): b_goal.add_goal(self.parse_declaration_ioc()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_goal # Bloc declaration de couches def parse_block_layers(self): self.expect(d.layers) self.expect(d.laccol) self.indent() self.endl() b_couche = T_block_couches(self.ast, "couches") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.identifier): b_couche.add_couche(self.parse_declaration_ioc()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_couche # Bloc declaration de structure def parse_block_structure(self): self.expect(d.struct) self.expect(d.laccol) self.indent() self.endl() b_struct = T_block_struct(self.ast, "structures") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.identifier): b_struct.add_struct(self.parse_declaration_structure()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_struct # Bloc declaration d'interface def parse_block_interface(self): self.expect(d.interf) self.expect(d.laccol) self.indent() self.endl() b_interface = T_block_interface(self.ast, "interfaces") continuer = True while continuer: if self.test(d.raccol): continuer = False elif self.test(d.clock): b_interface.add_clock(self.parse_declaration_clock()) elif self.test(d.reset): b_interface.add_reset(self.parse_declaration_reset()) elif self.test(d.learn): b_interface.add_learn(self.parse_declaration_learn()) else: continuer = False self.desindent() self.expect(d.raccol) self.endl() return b_interface # Declaration d'une constante def parse_declaration_int(self): self.expect(d.integer) token = self.expect(d.identifier) self.expect(d.equal) number = self.expect(d.number) self.expect(d.semicolon) self.endl() return T_int(self.ast, token, number) # Appel d'un nombre ou d'une constante def parse_const(self): if self.test(d.number): return self.expect(d.number) else: return T_const(self.ast, self.expect(d.identifier)) # Declaration d'une echelle def parse_declaration_scale(self): self.expect(d.scale) token = self.expect(d.identifier) self.expect(d.equal) n1 = self.parse_const() self.expect(d.comma) n2 = self.parse_const() self.expect(d.semicolon) self.endl() return T_scale(self.ast, token, n1, n2) # Corps de declaration d'une input, output ou couche def parse_declaration_ioc(self): name = self.expect(d.identifier) size_tab = 1 scale = T_scale(self.ast, "1_scale_unitaire", 0, 1) if self.test(d.lbracket): self.expect(d.lbracket) size_tab = self.parse_const() self.expect(d.rbracket) if self.test(d.semicolon): self.expect(d.semicolon) self.endl() else: self.expect(d.scale) if self.test_plus_1(d.comma): n1 = self.parse_const() self.expect(d.comma) n2 = self.parse_const() scale = T_scale(self.ast, "0_scale_perso", n1, n2) self.expect(d.semicolon) self.endl() else: scale = T_scale(self.ast, self.expect(d.identifier), 0, 0) self.expect(d.semicolon) self.endl() return T_ioc(self.ast, name, size_tab, scale) # Declaration d'une structure def parse_declaration_structure(self): liste_structures = [] id1 = self.expect(d.identifier) id1_from = -1 id1_to = -1 if self.test(d.lbracket): self.expect(d.lbracket) id1_from = self.parse_const() self.expect(d.to) id1_to = self.parse_const() self.expect(d.rbracket) continuer = True while continuer: if self.test(d.semicolon): self.expect(d.semicolon) continuer = False elif self.test(d.substract): self.expect(d.substract) self.expect(d.sup) id2 = self.expect(d.identifier) id2_from = -1 id2_to = -1 if self.test(d.lbracket): self.expect(d.lbracket) id2_from = self.parse_const() self.expect(d.to) id2_to = self.parse_const() self.expect(d.rbracket) liste_structures.append(T_structure(self.ast, id1, id1_from, id1_to, id2, id2_from, id2_to)) id1 = id2 id1_from = id2_from id1_to = id2_to else: continuer = False self.endl() return liste_structures # Declaration de l'horloge def parse_declaration_clock(self): self.expect(d.clock) clock = T_clock(self.ast, self.expect(d.identifier)) self.expect(d.semicolon) self.endl() return clock # Declaration du reset def parse_declaration_reset(self): self.expect(d.reset) reset = T_reset(self.ast, self.expect(d.identifier)) self.expect(d.semicolon) self.endl() return reset # Declaration de l'activateur d'apprentissage def parse_declaration_learn(self): self.expect(d.learn) learn = T_learn(self.ast, self.expect(d.identifier)) self.expect(d.semicolon) self.endl() return learn
def __init__(self): self.liste = Tokens() self.ast = Ast() self.indentation = 0 self.nb_errors = 0