def handle_index(self):
"""
Handle a reindexation
"""
objid = random.choice(self.allids)
obj = SeSQLResultSet.load((self.classname, objid))
index(obj)
return "(%s, %s)" % (self.classname, objid)
def longquery(self, limit=None):
"""
Perform a long query and return a lazy Django result set
"""
query = self._do_longquery(limit)
if limit:
query = query.fetchmany(limit)
return SeSQLResultSet(list(query), self.fields)
def handle(self, *apps, **options):
"""
Handle the command
"""
if len(apps) != 2:
print "Syntax : sesqlindex <classname> <objid>"
sys.exit(1)
obj = SeSQLResultSet.load(apps)
try:
index(obj)
transaction.commit()
except:
transaction.rollback()
raise
def handle(self, *apps, **options):
"""
Handle the command
"""
if len(apps) != 2:
print "Syntax : sesqlindex <classname> <objid>"
sys.exit(1)
obj = SeSQLResultSet.load(apps)
try:
index(obj)
transaction.commit()
except:
transaction.rollback()
raise
def shortquery(self, limit=50):
"""
Perform a long query and return a lazy Django result set
"""
table = self.get_table_name()
if table == config.MASTER_TABLE_NAME:
# Multitable or unprecise query ? Falling back to longquery
log.warning("Query on master table will not be optimized on %s" %
self.query)
return self.longquery(limit)
if "sesql_relevance" in self.order or "-sesql_relevance" in self.order:
# Order on relevance ? Falling back to longquery
log.info("Query sorting on relevance will not be optimized on %s" %
self.query)
return self.longquery(limit)
log.debug("Trying short query for %s" % self.query)
cursor = self._do_smart_query(limit)
return SeSQLResultSet(list(cursor), self.fields)
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute("DELETE FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
cursor.execute("SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (classname, len(allids),
len(already),
len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last, STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute(
"DELETE FROM %s WHERE classname=%%s" %
config.MASTER_TABLE_NAME, (classname, ))
cursor.execute(
"SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname, ))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (
classname, len(allids), len(already), len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last,
STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (
i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()