def __init__(self, req, jug): self.pars = ParserCACM() self.pars.initFile(req) self.jug = pd.read_csv( jug, delim_whitespace=True, header=None, index_col=False, names=['queryId', 'docId', 'sub-theme', 'score'], dtype={ 'queryId': str, 'docId': str }, converters={'queryId': lambda x: str(int(x))})
def indexation(self): ## Creat docs and stems docFrom self.docs=CreatDocs() self.stems=CreatTerms() self.docFrom=CreatdocFrom() self.parser=ParserCACM() self.texRepresenter=PorterStemmer() self.linkin,self.linkout=createlink() return self
class QueryParser(object): """Class for query reading from file""" def __init__(self, query_file, relevance_file): self.query = open(query_file, 'r') self.textRepresenter = PorterStemmer() #init boolean to be able to close source files self.already_closed = False #Create parser to read query_file #WARNING WILL ONLY WORK ON CACM DATASET TODO FIND SOLUTION self.parser = ParserCACM() self.parser.initFile(query_file) #Build a dictionary (query_id, list of relevant documents) self.relevant_docs = {} with open(relevance_file, 'r') as f: for line in f: data = line.split(" ") query_id = int(data[0]) if (not self.relevant_docs.has_key(query_id)): self.relevant_docs[query_id] = [] #A list is added per relevant doc for later use of couple (themes, score) self.relevant_docs.get(query_id).append( [int(data[1]), None, None]) def nextQuery(self): """Return next Query object""" query_data = self.parser.nextDocument() if (query_data == None): if (not self.already_closed): self.query.close() self.already_closed = True return -1 query_id = query_data.getId() query_text = query_data.getText() query_tf = self.textRepresenter.getTextRepresentation(query_text) relevant_docs_to_query = np.array( self.relevant_docs.get(int(query_id), [[None, None, None]])) return Query(query_id, query_text, query_tf, relevant_docs_to_query)
class QueryParserCACM(QueryParser): def __init__(self, req, jug): self.pars = ParserCACM() self.pars.initFile(req) self.jug = pd.read_csv( jug, delim_whitespace=True, header=None, index_col=False, names=['queryId', 'docId', 'sub-theme', 'score'], dtype={ 'queryId': str, 'docId': str }, converters={'queryId': lambda x: str(int(x))}) def nextQuery(self): ''' :return: return a dictionary {'id' : id of the query, 'text' : text of the querry, 'revelent' : {docId : (sub-theme, score)}} ''' return self.__next__() def __next__(self): q = self.pars.nextDocument() if q: return { 'id': q.identifier, 'text': q.text, 'revelent': self.jug[self.jug.queryId == q.getId()].drop( 'queryId', axis=1).set_index('docId').to_dict('index') } raise StopIteration() def __iter__(self): return self
def __init__(self, query_file, relevance_file): self.query = open(query_file, 'r') self.textRepresenter = PorterStemmer() #init boolean to be able to close source files self.already_closed = False #Create parser to read query_file #WARNING WILL ONLY WORK ON CACM DATASET TODO FIND SOLUTION self.parser = ParserCACM() self.parser.initFile(query_file) #Build a dictionary (query_id, list of relevant documents) self.relevant_docs = {} with open(relevance_file, 'r') as f: for line in f: data = line.split(" ") query_id = int(data[0]) if (not self.relevant_docs.has_key(query_id)): self.relevant_docs[query_id] = [] #A list is added per relevant doc for later use of couple (themes, score) self.relevant_docs.get(query_id).append( [int(data[1]), None, None])
def test_weighter(): parser = ParserCACM() textRepresenter = PorterStemmer() fname = "data/cacm/cacm.txt" I = Index(parser, textRepresenter) I.indexation(fname) weighters = [Binary(I), TF(I), TF_IDF(I), Log(I), Log_plus(I)] for i, w in enumerate(weighters): print "Test of weighter" + str(i) print "getDocWeightsForDoc" print w.getDocWeightsForDoc("20") print "getDocWeightsForStem" print w.getDocWeightsForStem("accelerat") print "getDocWeightsForQuery" print w.getWeightsForQuery(I.getTfsForDoc("20"))
def initIndex(database_file): """Init Index or load it if previously computed""" sys.stdout.write("Indexing database...") sys.stdout.flush() if os.path.isfile('Index.p'): I = pickle.load( open( "Index.p", "rb" ) ) else: parser = ParserCACM() textRepresenter = PorterStemmer() I = Index(parser,textRepresenter) I.indexation(database_file) I.parser = None pickle.dump( I, open( "Index.p", "wb" ) ) sys.stdout.write("Done!\n") sys.stdout.flush() return I
def createlink(): linkout={} linkin={} ps=ParserCACM() ps.initFile("cacm/cacm.txt") doc=ps.nextDocument() while doc: linkout[doc.getId()]=set() links=doc.get('links').split(';') for link in links: if link!='': linkout[doc.getId()].add(link) if link not in linkin: linkin[link]=set() # value unique linkin[link].add(doc.getId()) doc=ps.nextDocument() return linkin,linkout
print "get top 3 documents = ", '[%s]' % ', '.join( map(str, query_results[i][0:3])) return query_results if __name__ == "__main__": fname = "data/cacm/cacm.txt" sys.stdout.write("Indexing database...") sys.stdout.flush() if os.path.isfile('Index.p'): I = pickle.load(open("Index.p", "rb")) else: parser = ParserCACM() textRepresenter = PorterStemmer() I = Index(parser, textRepresenter) I.indexation(fname) I.parser = None pickle.dump(I, open("Index.p", "wb")) sys.stdout.write("Done!\n") sys.stdout.flush() sys.stdout.write("Creating weighters...") sys.stdout.flush() if os.path.isfile('Vectoriel.p'): models = pickle.load(open("Models.p", "rb")) else:
def __init__(self): ParserCACM.__init__(self)
with open(self.stems_filename, 'w') as f: json.dump(stems, f) with open(self.docs_filename, 'w') as f: json.dump(docs, f) return def string2json(self, string): '''convert "{'4': 1}{'7': 10}" to {'4':1, '7':10} ''' string_list = [s + '}' for s in string.split('}')][:-1] json_list = [ast.literal_eval(s) for s in string_list] return {d.keys()[0]: d.values()[0] for d in json_list} if __name__ == '__main__': parser = ParserCACM() parser.initFile("cacm/cacm.txt") index = Index(name='test', parser=ParserCACM, textRepresenter=PorterStemmer, create_index=False) for i in range(20): doc = parser.nextDocument() doc_id = doc.getId() doc_rep = PorterStemmer().getTextRepresentation(doc.getText()) print('doc_rep', doc_rep) print('length', index.getDocsLength(doc_id))
index_file_inverse.seek(self.index_inverse[stem][0]) doc_tf = index_file_inverse.read(self.index_inverse[stem][1]) index_file_inverse.flush() index_file_inverse.close() return doc_tf def getStrDoc(self, id_doc): f = open(self.docFrom[id_doc][0], 'r') f.seek(self.docFrom[id_doc][1]) doc = f.read(self.docFrom[id_doc][2]) f.flush() f.close() return doc def _dict_to_file(self, dict): return ''.join([self._line_to_file(i, dict[i]) for i in dict.keys()]) def _line_to_file(self, i, v): return str(i) + '|' + str(v) + ' ' tr = PorterStemmer() parser = ParserCACM() parser.initFile("C:\Users\king\Desktop\DAC2\RI\TP1-6\RI\cacm\cacm.txt") doc = parser.nextDocument() print type(doc.others["links"])
from ParserCACM import ParserCACM, QueryParser from TextRepresenter import PorterStemmer if __name__=='__main__': rel_filename = 'cacm/cacm.rel' query_filename = 'cacm/cacm.qry' index = Index(name='test', docFrom=None, parser=ParserCACM, textRepresenter=PorterStemmer, create_index=False) weighter = Weighter(index) parser = ParserCACM() parser.initFile('cacm/cacm.txt') doc = parser.nextDocument() print(doc.others['links']) # for d in range(20,22): # docId = str(d) # print(ParserCACM().getDocument(docId)) # print(weighter.getDocWeightsForDoc(docId), index.getDocsLength(docId)) # q = QueryParser(query_filename, rel_filename) # train_queries, test_queries = q.split_query_dataset() # print(len(train_queries), len(test_queries))