def doTest(): c = createAndChdir([ 'lgTask.test.addTask' ], threaded = True) c._database.drop_collection('test') col = c._database['test'] col.insert({ 'id': 'a', 'value': 0 }) a = time.time() for i in range(10000): c.createTask("AddTask", value = i) initTime = time.time() - a a = time.time() runProcessor(5.0) if col.find_one({ 'id': 'a' })['value'] != 10000: print("FAIL") b = time.time() print("Time: {0}".format(b - a)) print("Init time: {0}".format(initTime))
def _doTest(testType): # Generate objs beforehand, to not include that as any part of the tests objs = DumpTaskBase.getObjects() objSize = _rsize(objs[0]) x = 6 y = 6 print("""Benchmarking # of messages through pipe with {} pushing tasks and {} pulling tasks with object sizes of {} KB spread over 3 keys and batch size of {} objects""".format(x, y, objSize / 1024, OBJECT_COUNT)) c = createAndChdir([ 'talk' ], threaded = True) cc = c._database['test'] #cc.save({ '_id': 'count', 'value': 0, 'count': 0 }) keys = [ '1', '2', '3' ] def getKey(index): return keys[index % len(keys)] dumpTask = 'DumpTask' + testType.title() readTask = 'ReadTask' + testType.title() for i in range(x): c.createTask(dumpTask, key = getKey(i)) for i in range(y): c.createTask(readTask, key = getKey(i)) if testType == 'talk': # talk - 16000/s ##OLD, PUSH BEHAVIOR: 2570 /s avg (before opt, needs run again) # no setup needed pass elif testType == 'redis': # 3500 /s avg # Ensure all instances are running import redis for p in REDIS_PORTS: try: r = redis.StrictRedis(port=p, db=0) try: r.dbsize() finally: r.connection_pool.disconnect() except: print(traceback.format_exc()) raise Exception(("Ensure redis is running on {} ({} redises " + "needed)").format(p, len(REDIS_PORTS))) r = redis.StrictRedis(db=0) for k in keys: r.delete(k) r.connection_pool.disconnect() elif testType == 'rabbitmq': # 3300 /s avg import puka p = puka.Client() p.wait(p.connect()) for k in keys: try: p.wait(p.queue_delete(k)) except puka.NotFound: pass # Define queues before tasks too... give it an advantage that way p.wait(p.queue_declare(queue = k, durable = False)) elif testType == 'mongo': # 2000 / s avg... for i in range(x): c.createTask("DumpTaskMongo", key=getKey(i)) for i in range(y): c.createTask("ReadTaskMongo", key=getKey(i)) elif testType == 'speed': # 400173 /s avg pass a = time.time() runProcessor(TEST_TIME) b = time.time() print('=' * 80) totalCount = 0 totalVal = 0.0 totalLag = 0.0 for doc in cc.find(): totalCount += doc['count'] totalVal += doc['value'] totalLag += doc['latency'] * doc['count'] if totalCount > 0: totalLag /= totalCount print("{0} / {1:.2f} s avg lag".format(totalCount, totalLag)) avg = totalCount / (b - a) print("{} avg msgs/sec".format(avg)) return avg