def commit_indexing_set_old(self):
''' after filling an indexing set the actual indexing needs to be done
'''
import sqlite3
training_id = str(self.training_id)
conn = sqlite3.connect(self.sqlserver)
c = conn.cursor()
# fetch the content
sql = "SELECT * FROM gensimIndexingSet"+str(training_id)
print ( sql )
c.execute( sql )
# just fetch all items
indexing_data = c.fetchmany(500);
#service = similarities.SessionServer(self.rootlocation, autosession=True)
service = SessionServer(self.rootlocation + 'gensimTraining'+str(self.training_id), autosession=True) # create a local server
import Pyro4
service = Pyro4.Proxy(Pyro4.locateNS().lookup('gensim.testserver'))
print self.rootlocation + 'gensimTraining'+str(self.training_id)
while len(indexing_data) > 0 :
corpus = [{'id': str(id), 'tokens': utils.simple_preprocess(text)}
for (text, id) in indexing_data]
service.index(corpus) ## TODO we don't have a corpus yet, but we definatly need one big
indexing_data = c.fetchmany(500)
service.autosession = True
time.sleep(0.5)
# sql = "DROP TABLE IF EXISTS gensimIndexingSet"+str(training_id)
# c.execute( sql )
self.delete_set()
self.init_indexing_set()
return 'indexing done'
