def doTest():
c = createAndChdir([ 'lgTask.test.addTask' ], threaded = True)
c._database.drop_collection('test')
col = c._database['test']
col.insert({ 'id': 'a', 'value': 0 })
a = time.time()
for i in range(10000):
c.createTask("AddTask", value = i)
initTime = time.time() - a
a = time.time()
runProcessor(5.0)
if col.find_one({ 'id': 'a' })['value'] != 10000:
print("FAIL")
b = time.time()
print("Time: {0}".format(b - a))
print("Init time: {0}".format(initTime))
def _doTest(testType):
# Generate objs beforehand, to not include that as any part of the tests
objs = DumpTaskBase.getObjects()
objSize = _rsize(objs[0])
x = 6
y = 6
print("""Benchmarking # of messages through pipe with {} pushing
tasks and {} pulling tasks with object sizes of {} KB spread
over 3 keys and batch size of {} objects""".format(x, y,
objSize / 1024, OBJECT_COUNT))
c = createAndChdir([ 'talk' ], threaded = True)
cc = c._database['test']
#cc.save({ '_id': 'count', 'value': 0, 'count': 0 })
keys = [ '1', '2', '3' ]
def getKey(index):
return keys[index % len(keys)]
dumpTask = 'DumpTask' + testType.title()
readTask = 'ReadTask' + testType.title()
for i in range(x):
c.createTask(dumpTask, key = getKey(i))
for i in range(y):
c.createTask(readTask, key = getKey(i))
if testType == 'talk':
# talk - 16000/s
##OLD, PUSH BEHAVIOR: 2570 /s avg (before opt, needs run again)
# no setup needed
pass
elif testType == 'redis':
# 3500 /s avg
# Ensure all instances are running
import redis
for p in REDIS_PORTS:
try:
r = redis.StrictRedis(port=p, db=0)
try:
r.dbsize()
finally:
r.connection_pool.disconnect()
except:
print(traceback.format_exc())
raise Exception(("Ensure redis is running on {} ({} redises "
+ "needed)").format(p, len(REDIS_PORTS)))
r = redis.StrictRedis(db=0)
for k in keys:
r.delete(k)
r.connection_pool.disconnect()
elif testType == 'rabbitmq':
# 3300 /s avg
import puka
p = puka.Client()
p.wait(p.connect())
for k in keys:
try:
p.wait(p.queue_delete(k))
except puka.NotFound:
pass
# Define queues before tasks too... give it an advantage that way
p.wait(p.queue_declare(queue = k, durable = False))
elif testType == 'mongo':
# 2000 / s avg...
for i in range(x):
c.createTask("DumpTaskMongo", key=getKey(i))
for i in range(y):
c.createTask("ReadTaskMongo", key=getKey(i))
elif testType == 'speed':
# 400173 /s avg
pass
a = time.time()
runProcessor(TEST_TIME)
b = time.time()
print('=' * 80)
totalCount = 0
totalVal = 0.0
totalLag = 0.0
for doc in cc.find():
totalCount += doc['count']
totalVal += doc['value']
totalLag += doc['latency'] * doc['count']
if totalCount > 0:
totalLag /= totalCount
print("{0} / {1:.2f} s avg lag".format(totalCount, totalLag))
avg = totalCount / (b - a)
print("{} avg msgs/sec".format(avg))
return avg
