def handle(self, *apps, **options):
"""
Handle the command
"""
self.options = options
# If we have at least a query thread, build list of queries
if options["long-threads"] or options["short-threads"]:
if not options["queryfile"]:
print "--queryfile is mandatory if a query thread is enabled"
sys.exit(1)
print "Loading queries list..."
self.queries = [ (q.strip(), eval(q)) for q in open(options["queryfile"]) if q.strip() ]
# If we have at least a reindex thread, load list of objects
if options["index-threads"]:
if not options["index-type"]:
print "--index-type is mandatory if a reindex thread is enabled"
sys.exit(1)
self.classname = options["index-type"]
self.klass = typemap.get_class_by_name(self.classname)
if not hasattr(self.klass, "objects"):
print "No such type : ", self.classname
sys.exit(1)
print "Loading object ids list..."
klass = typemap.get_class_by_name(options["index-type"])
self.allids = [ int(a['id']) for a in klass.objects.values('id') ]
# Load queries
self.short = []
self.long = []
self.index = []
self.threads = []
self.lock = threading.RLock()
self.lock.acquire()
# Starting threads
print "Starting threads..."
self.start_threads(options['short-threads'], self.handle_short,
self.short, options["short-delay"])
self.start_threads(options['long-threads'], self.handle_long,
self.long, options["long-delay"])
self.start_threads(options['index-threads'], self.handle_index,
self.index, options["index-delay"])
# Waiting
print "Running benchmark..."
self.running = True
self.lock.release()
time.sleep(options["duration"])
# Killing threads
print "Killing threads..."
self.kill_threads()
# Display results
self.display_results()
def load(obj):
"""
Get a given object
"""
objclass, objid = obj[:2]
objclass = typemap.get_class_by_name(objclass)
if not objclass:
return config.orm.not_found
entry = "%s:%s" % (objclass.__name__, objid)
log.debug("Fetching %s" % entry)
return config.orm.load_object(objclass, objid)
def load(obj):
"""
Get a given object
"""
objclass, objid = obj[:2]
objclass = typemap.get_class_by_name(objclass)
if not objclass:
return config.orm.not_found
entry = "%s:%s" % (objclass.__name__, objid)
log.debug("Fetching %s" % entry)
return config.orm.load_object(objclass, objid)
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute("DELETE FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
cursor.execute("SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname,))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (classname, len(allids),
len(already),
len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last, STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()
def reindex(self, classname, reindex=False, flush=False, dry_run=False):
"""
Reindex a single class
"""
klass = typemap.get_class_by_name(classname)
if not hasattr(klass, "objects"):
return
print "=> Starting reindexing for %s" % classname
if dry_run:
print "Dry-run mode."
sys.stdout.flush()
objs = klass.objects.values('id')
if self.options["order"]:
objs = objs.order_by(self.options["order"])
allids = [int(a['id']) for a in objs]
cursor = connection.cursor()
if flush:
print "Flushing all already indexed objects for class %s" % classname
cursor.execute(
"DELETE FROM %s WHERE classname=%%s" %
config.MASTER_TABLE_NAME, (classname, ))
cursor.execute(
"SELECT id FROM %s WHERE classname=%%s" % config.MASTER_TABLE_NAME,
(classname, ))
already = set([int(c[0]) for c in cursor])
if not reindex:
missing = [oid for oid in allids if not oid in already]
else:
missing = allids
print "%s : %d object(s), %d already indexed, reindexing %d" % (
classname, len(allids), len(already), len(missing))
if dry_run:
print "Dry-run: don't proceed. Rollback."
transaction.rollback()
return
sys.stdout.flush()
nb = len(missing)
full_tmr = Timer()
def disp_stats():
if not nb:
return
full_tmr.stop()
elapsed = full_tmr.get_global()
elapsed_last = full_tmr.peek()
done = float(i + 1) / float(nb)
eta = elapsed / done * (1 - done)
print "**SeSQL reindex step stats**"
print " - %d objects in %.2f s, rate %.2f" % (STEP, elapsed_last,
STEP / elapsed_last)
print "**SeSQL global reindex on %s stats**" % classname
print " - %d / %d ( %04.1f %% ) in %.2f s, rate %.2f, ETA %.2f s" % (
i + 1, nb, 100 * done, elapsed, i / elapsed, eta)
sys.stdout.flush()
full_tmr.start()
for i, oid in enumerate(missing):
obj = None
try:
obj = SeSQLResultSet.load((classname, oid))
index(obj)
transaction.commit()
except Exception, e:
print "Error indexing (%s,%s) : %s" % (classname, oid, e)
transaction.rollback()
del obj
if i % STEP == STEP - 1:
disp_stats()
