def test_notify_queue_performance_single_thread(self):
round = 10000
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
client = ServiceClient(service)
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
assert False
def get_client(redis_conf, rmq_conf):
conf = rmq_conf
if mode == "redis":
conf = redis_conf
meta = ServiceMeta.from_json(conf)
client = ServiceClient(meta) if mode == "redis" else ServiceClient(meta, amqp_url=amqp_url, client_id="")
client.start()
return wrap_client(client)
def test_rpc_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_queue"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 3)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_rpc_queue"]["results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
def test_notify_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_sharding1"]["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
assert len(results1) + len(results2) + len(results3) == 20
old_size1 = len(results1)
old_size2 = len(results2)
old_size3 = len(results3)
results1[:] = []
results2[:] = []
results3[:] = []
for i in range(100):
ev = TestService.EVENT.copy()
ev.key = str(i % 20)
self.client.send(ev, ev.key, True, 1)
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
new_size1 = len(results1)
new_size2 = len(results2)
new_size3 = len(results3)
print old_size1
print old_size2
print old_size3
print new_size1
print new_size2
print new_size3
assert old_size1 * 5 == new_size1
assert old_size2 * 5 == new_size2
assert old_size3 * 5 == new_size3
def test_notify_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_queue"]["service"])
self.client.start()
self.client.send(TestService.EVENT, "", True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(2)
results = TestService.server_repo["service_notify_queue"][
"results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
def test_rpc_shuffle(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_shuffle1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_rpc_shuffle1"][
"results"]
results2 = TestService.server_repo["service_rpc_shuffle2"][
"results"]
assert len(results1) + len(results2) == 1
def test_rpc_topic(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_polling_topic1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, [TestService.EVENT, TestService.EVENT])
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_polling_topic1"][
"results"]
results2 = TestService.server_repo["service_polling_topic2"][
"results"]
assert len(results1) == 1
assert len(results2) == 1
def test_notify_topic_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_tosharding1"]
["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_tosharding1"][
"results"]
results2 = TestService.server_repo["service_notify_tosharding2"][
"results"]
results3 = TestService.server_repo["service_notify_tosharding3"][
"results"]
assert len(results1) + len(results2) == 20
assert len(results3) == 20
def test_rpc_queue_performance_multiple_thread(self):
round = 5000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
self.client = ServiceClient(service)
self.client.start()
start = millis_now()
threads = []
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
self.client, round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
self.client.close()
assert False
return
def get_misclog_rpc_client(amqp_url):
misc_service = """
{
"name": "misclog_notify",
"callmode": "notify",
"delivermode": "topic",
"serverimpl": "rabbitmq",
"coder": "mail",
"options": {
}
}
"""
client = ServiceClient(ServiceMeta.from_json(misc_service),
amqp_url=amqp_url)
return client
def get_httplog_rpc_client(amqp_url):
http_service = """
{
"name": "httplog_notify",
"callmode": "notify",
"delivermode": "sharding",
"serverimpl": "rabbitmq",
"coder": "mail",
"options": {
"servercardinality": 2
}
}
"""
client = ServiceClient(ServiceMeta.from_json(http_service),
amqp_url=amqp_url)
return client
def test_rpc_queue_performance_single_thread(self):
round = 5000
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
client = ServiceClient(service)
client.start()
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
assert False
def test():
try:
from bigsec_common.metrics.metricsagent import MetricsAgent, set_async_mode
set_async_mode()
except:
pass
metrics_url = {
"server": "redis",
"app": "alaskan",
"redis": {
"host": "127.0.0.1",
"port": 6379,
# 'password': '******',
"type": "redis"
}
}
MetricsAgent.get_instance().initialize_by_dict(metrics_url)
post_stalker_recorder = MyMetricsRecorder(metrics_name='stalker_post_data',
db='default')
post_api_recorder = MyMetricsRecorder(metrics_name='api_request',
db='default')
RedisCtx.get_instance().host = '127.0.0.1'
RedisCtx.get_instance().port = 6379
# RedisCtx.get_instance().password = '******'
try:
meta = ServiceMeta.from_json(Babel_Conf)
babel_client = ServiceClient(meta)
babel_client.start()
now = millis_now()
event = Event("warden", "stalker_data", "key", now, exp_notice)
babel_client.notify(event, event.key, timeout=5)
print('notice notify publish [ok]')
time.sleep(1)
check_result = requests.get(
'http://localhost:8088/alaskan/get_blacklist?ip={}'.format(
exp_notice.get('ip'))).content
if 'true' in check_result:
print('check recv babel [ok]')
else:
print('check recv babel [error]')
except:
import traceback
print('notice notify publish [error][{}]'.format(
traceback.format_exc()))
def test_notify_queue_performance_multiple_thread(self):
round = 10000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
threads = []
clients = [ServiceClient(service) for i in range(numOfThreads)]
for client in clients:
client.start()
start = millis_now()
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
clients[i], round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on receiving {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
for client in clients:
client.close()
assert False
def get_httplog_rpc_client():
http_service = get_babel_file_content('Httplog_redis.service')
client = ServiceClient(ServiceMeta.from_json(http_service))
return client
def get_misclog_rpc_client():
misc_service = get_babel_file_content('Misclog_redis.service')
client = ServiceClient(ServiceMeta.from_json(misc_service))
return client
class TestService:
EVENT = Event("testapp", "testname", "testkey", 0, {})
MODE = "clientserver"
CLIENT = "client" in MODE
SERVER = "server" in MODE
server_repo = dict()
def setup_class(self):
RedisCtx.get_instance().host = "127.0.0.1"
RedisCtx.get_instance().port = 6379
print "start to init service"
TestService.init_services()
print "successfully started the services"
def setup_method(self, method):
if not TestService.SERVER:
return
for v in TestService.server_repo.values():
server = v["server"]
results = v["results"]
results[:] = []
@staticmethod
def build_service(id,
sname,
callmode,
delivermode,
subname="",
sequence="",
cardinality=0):
s = ServiceMeta.from_dict({
"name": sname,
"callmode": callmode,
"delivermode": delivermode,
"serverimpl": "redis",
"coder": "mail",
"options": {
"cdc": "sh",
"sdc": "sh",
"serversubname": subname,
"serverseq": sequence,
"servercardinality": cardinality
}
})
# start the relative server in server mode
if "server" in TestService.MODE:
echo_service, results = get_echo_service()
server = ServiceServer(s, echo_service)
server.start()
else:
server = None
results = None
TestService.server_repo[id] = {
"service": s,
"server": server,
"results": results
}
@staticmethod
def init_services():
# queue notify
TestService.build_service("service_notify_queue",
"service_notify_queue", "notify", "queue")
# 3 services use topic to get data
TestService.build_service("service_notify_topic1",
"service_notify_topic", "notify", "topic",
"consumer1")
TestService.build_service("service_notify_topic2",
"service_notify_topic", "notify", "topic",
"consumer2")
TestService.build_service("service_notify_topic3",
"service_notify_topic", "notify", "topic",
"consumer3")
# 3 services use sharding
TestService.build_service("service_notify_sharding1",
"service_notify_sharding", "notify",
"sharding", "", 1, 3)
TestService.build_service("service_notify_sharding2",
"service_notify_sharding", "notify",
"sharding", "", 2, 3)
TestService.build_service("service_notify_sharding3",
"service_notify_sharding", "notify",
"sharding", "", 3, 3)
# 3 services use shuffle
TestService.build_service("service_notify_shuffle1",
"service_notify_shuffle", "notify",
"shuffle", "", 1, 3)
TestService.build_service("service_notify_shuffle2",
"service_notify_shuffle", "notify",
"shuffle", "", 2, 3)
TestService.build_service("service_notify_shuffle3",
"service_notify_shuffle", "notify",
"shuffle", "", 3, 3)
# rpc queue
TestService.build_service("service_rpc_queue", "service_rpc_queue",
"rpc", "queue", "", 1, 1)
# 2 sharding services for rpc
TestService.build_service("service_rpc_sharding1",
"service_rpc_sharding", "rpc", "sharding",
"", 1, 2)
TestService.build_service("service_rpc_sharding2",
"service_rpc_sharding", "rpc", "sharding",
"", 2, 2)
#
# 2 shuffle services for rpc
TestService.build_service("service_rpc_shuffle1",
"service_rpc_shuffle", "rpc", "shuffle", "",
1, 2)
TestService.build_service("service_rpc_shuffle2",
"service_rpc_shuffle", "rpc", "shuffle", "",
2, 2)
#
# 2 topic services for polling
TestService.build_service("service_polling_topic1",
"service_polling_topic", "polling", "topic",
"consumer1", 1, 2)
TestService.build_service("service_polling_topic2",
"service_polling_topic", "polling", "topic",
"consumer2", 2, 2)
#
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_queue"]["service"])
self.client.start()
self.client.send(TestService.EVENT, "", True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(2)
results = TestService.server_repo["service_notify_queue"][
"results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_topic(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_topic1"]["service"])
self.client.start()
self.client.send(TestService.EVENT, "", True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results = TestService.server_repo["service_notify_topic1"][
"results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
results = TestService.server_repo["service_notify_topic2"][
"results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_sharding1"]["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
assert len(results1) + len(results2) + len(results3) == 20
old_size1 = len(results1)
old_size2 = len(results2)
old_size3 = len(results3)
results1[:] = []
results2[:] = []
results3[:] = []
for i in range(100):
ev = TestService.EVENT.copy()
ev.key = str(i % 20)
self.client.send(ev, ev.key, True, 1)
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
new_size1 = len(results1)
new_size2 = len(results2)
new_size3 = len(results3)
print old_size1
print old_size2
print old_size3
print new_size1
print new_size2
print new_size3
assert old_size1 * 5 == new_size1
assert old_size2 * 5 == new_size2
assert old_size3 * 5 == new_size3
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_shuffle(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_shuffle1"]["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_shuffle1"][
"results"]
results2 = TestService.server_repo["service_notify_shuffle2"][
"results"]
results3 = TestService.server_repo["service_notify_shuffle3"][
"results"]
assert len(results1) + len(results2) + len(results3) == 20
assert len(results1) > 0
assert len(results2) > 0
assert len(results3) > 0
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_queue"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 3)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_rpc_queue"]["results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_topic(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_polling_topic1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, [TestService.EVENT, TestService.EVENT])
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_polling_topic1"][
"results"]
results2 = TestService.server_repo["service_polling_topic2"][
"results"]
assert len(results1) == 1
assert len(results2) == 1
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_sharding1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_rpc_sharding1"][
"results"]
results2 = TestService.server_repo["service_rpc_sharding2"][
"results"]
assert len(results1) + len(results2) == 1
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_shuffle(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_shuffle1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_rpc_shuffle1"][
"results"]
results2 = TestService.server_repo["service_rpc_shuffle2"][
"results"]
assert len(results1) + len(results2) == 1
def _performance_test_one_thread(self, client, round, service):
if TestService.CLIENT:
for i in xrange(round):
result = client.send(TestService.EVENT, "", True, 5)
if not result[0]:
assert result[0] == True
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_queue_performance_single_thread(self):
round = 10000
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
client = ServiceClient(service)
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
assert False
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_notify_queue_performance_multiple_thread(self):
round = 10000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
threads = []
clients = [ServiceClient(service) for i in range(numOfThreads)]
for client in clients:
client.start()
start = millis_now()
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
clients[i], round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on receiving {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
for client in clients:
client.close()
assert False
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_queue_performance_single_thread(self):
round = 5000
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
client = ServiceClient(service)
client.start()
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
assert False
@pytest.mark.skipif("'client' not in TestService.MODE")
def test_rpc_queue_performance_multiple_thread(self):
round = 5000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
self.client = ServiceClient(service)
self.client.start()
start = millis_now()
threads = []
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
self.client, round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
self.client.close()
assert False
return
# @pytest.mark.skipif("'client' not in TestService.MODE")
# def test_rpc_queue_performance_multiple_thread_multiple_client(self):
# round = 4000
# numOfThreads = 10
# if TestService.CLIENT:
# service = TestService.server_repo["service_rpc_queue"]["service"]
# clients = [ServiceClient(service) for i in range(numOfThreads)]
# for client in clients:
# client.start()
# start = millis_now()
# threads = []
# for i in range(numOfThreads):
# threads.append(run_in_thread(self._performance_test_one_thread, clients[i], round/numOfThreads, service))
# for i in range(numOfThreads):
# threads[i].join()
# end = millis_now()
# for client in clients:
# client.close()
# print "spent {} milliseconds on sending {} records, {} per second on average".format((end-start), round, round*1000.0/(end-start))
# assert False
# return
@pytest.mark.skipif("'server' != TestService.MODE")
def test_server(self):
sleep(200)
# if __name__ == "__main__":
# import cProfile
# ts = TestService()
# ts.setup_class()
# ts.test_rpc_queue_performance_multiple_thread()
# pass