def process_chunk(self):
"""
Process a chunk
"""
cursor = connection.cursor()
cursor.execute("""SELECT classname, objid
FROM sesql_reindex_schedule
ORDER BY scheduled_at ASC LIMIT %d""" % self.chunk)
rows = cursor.fetchall()
if not rows:
transaction.rollback()
return
self.log.info("Found %d row(s) to reindex" % len(rows))
done = set()
for row in rows:
try:
row = tuple(row)
if not row in done:
self.log.info("Reindexing %s:%d" % row)
done.add(row)
try:
obj = results.SeSQLResultSet.load(row)
index.index(obj)
except config.orm.not_found:
self.log.info("%s:%d doesn't exist anymore, undexing" % row)
index.unindex(row)
cursor.execute("""DELETE FROM sesql_reindex_schedule
WHERE classname=%s AND objid=%s""", row)
except Exception, e:
self.log.error('Error in row %s:%s : %s' % (row[0], row[1], e))
if cmd["debug"]:
import pdb
pdb.post_mortem()
def handle_index(self):
"""
Handle a reindexation
"""
objid = random.choice(self.allids)
obj = SeSQLResultSet.load((self.classname, objid))
index(obj)
return "(%s, %s)" % (self.classname, objid)
def handle(self, *apps, **options):
"""
Handle the command
"""
if len(apps) != 2:
print "Syntax : sesqlindex <classname> <objid>"
sys.exit(1)
obj = SeSQLResultSet.load(apps)
try:
index(obj)
transaction.commit()
except:
transaction.rollback()
raise
def handle(self, *apps, **options):
"""
Handle the command
"""
if len(apps) != 2:
print "Syntax : sesqlindex <classname> <objid>"
sys.exit(1)
obj = SeSQLResultSet.load(apps)
try:
index(obj)
transaction.commit()
except:
transaction.rollback()
raise
def handle_index(instance, isunindex = False):
# Trick to defer import
from sesql import config
from sesql.index import unindex, index, schedule_reindex
if getattr(config, 'ASYNCHRONOUS_INDEXING', False):
return schedule_reindex(instance)
else:
if isunindex:
return unindex(instance)
else:
return index(instance)
def handle_index(instance, isunindex=False):
# Trick to defer import
from sesql import config
from sesql.index import unindex, index, schedule_reindex
if getattr(config, 'ASYNCHRONOUS_INDEXING', False):
return schedule_reindex(instance)
else:
if isunindex:
return unindex(instance)
else:
return index(instance)
def iteration(self):
"""
Perform one iteration : reindex everything
"""
self.switch_to_old()
cursor = config.orm.cursor()
opts = (self.options['datefield'], config.MASTER_TABLE_NAME)
query = 'SELECT classname, id, %s FROM %s WHERE true' % opts
vals = ()
if self.state['since']:
query += ' AND %s >= %%s' % (self.options['datefield'],)
vals += (self.state['since'],)
if self.state['last']:
query += ' AND (classname != %s OR id != %s)'
vals += self.state['last']
query += ' ORDER BY %s ASC LIMIT %d' % (self.options['datefield'],
self.options['step'])
if self.options['verbose']:
print "Performing %s (%s)" % (query, vals)
cursor.execute(query, vals)
self.switch_to_new()
nb = 0
for item in cursor:
if self.options['verbose']:
print "Indexing %s" % (item,)
obj = results.SeSQLResultSet.load((item[0], item[1]))
index.index(obj, index_related = False)
last = item[2]
nb += 1
if nb:
self.state['last'] = (item[0], item[1])
self.state['since'] = last
self.state['nb'] = nb
self.state['done'] = self.state.get('done', 0) + nb
return nb
def iteration(self):
"""
Perform one iteration : reindex everything
"""
self.switch_to_old()
cursor = config.orm.cursor()
opts = (self.options['datefield'], config.MASTER_TABLE_NAME)
query = 'SELECT classname, id, %s FROM %s WHERE true' % opts
vals = ()
if self.state['since']:
query += ' AND %s >= %%s' % (self.options['datefield'], )
vals += (self.state['since'], )
if self.state['last']:
query += ' AND (classname != %s OR id != %s)'
vals += self.state['last']
query += ' ORDER BY %s ASC LIMIT %d' % (self.options['datefield'],
self.options['step'])
if self.options['verbose']:
print "Performing %s (%s)" % (query, vals)
cursor.execute(query, vals)
self.switch_to_new()
nb = 0
for item in cursor:
if self.options['verbose']:
print "Indexing %s" % (item, )
obj = results.SeSQLResultSet.load((item[0], item[1]))
index.index(obj, index_related=False)
last = item[2]
nb += 1
if nb:
self.state['last'] = (item[0], item[1])
self.state['since'] = last
self.state['nb'] = nb
self.state['done'] = self.state.get('done', 0) + nb
return nb
def process_chunk(self):
"""
Process a chunk
"""
cursor = connection.cursor()
cursor.execute("""SELECT classname, objid
FROM sesql_reindex_schedule
ORDER BY scheduled_at ASC LIMIT %d""" % self.chunk)
rows = cursor.fetchall()
if not rows:
transaction.rollback()
return
self.log.info("Found %d row(s) to reindex" % len(rows))
done = set()
for row in rows:
try:
row = tuple(row)
if not row in done:
self.log.info("Reindexing %s:%d" % row)
done.add(row)
try:
obj = results.SeSQLResultSet.load(row)
index.index(obj)
except config.orm.not_found:
self.log.info("%s:%d doesn't exist anymore, undexing" %
row)
index.unindex(row)
cursor.execute(
"""DELETE FROM sesql_reindex_schedule
WHERE classname=%s AND objid=%s""", row)
except Exception, e:
self.log.error('Error in row %s:%s : %s' % (row[0], row[1], e))
if cmd["debug"]:
import pdb
pdb.post_mortem()
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute("DELETE FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
cursor.execute("SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (classname, len(allids),
len(already),
len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last, STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute(
"DELETE FROM %s WHERE classname=%%s" %
config.MASTER_TABLE_NAME, (classname, ))
cursor.execute(
"SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname, ))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (
classname, len(allids), len(already), len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last,
STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (
i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()
def update_cb(self, mapper, connection, target):
"""
Object was created or deleted
"""
from sesql import index
return index.index(target)
def update_cb(self, mapper, connection, target):
"""
Object was created or deleted
"""
from sesql import index
return index.index(target)
