def get_client(redis_conf, rmq_conf):
conf = rmq_conf
if mode == "redis":
conf = redis_conf
meta = ServiceMeta.from_json(conf)
client = ServiceClient(meta) if mode == "redis" else ServiceClient(meta, amqp_url=amqp_url, client_id="")
client.start()
return wrap_client(client)
def test_notify_queue_performance_single_thread(self):
round = 10000
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
client = ServiceClient(service)
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
assert False
def test_rpc_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_queue"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 3)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_rpc_queue"]["results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
def test_notify_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_sharding1"]["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
assert len(results1) + len(results2) + len(results3) == 20
old_size1 = len(results1)
old_size2 = len(results2)
old_size3 = len(results3)
results1[:] = []
results2[:] = []
results3[:] = []
for i in range(100):
ev = TestService.EVENT.copy()
ev.key = str(i % 20)
self.client.send(ev, ev.key, True, 1)
sleep(1)
results1 = TestService.server_repo["service_notify_sharding1"][
"results"]
results2 = TestService.server_repo["service_notify_sharding2"][
"results"]
results3 = TestService.server_repo["service_notify_sharding3"][
"results"]
new_size1 = len(results1)
new_size2 = len(results2)
new_size3 = len(results3)
print old_size1
print old_size2
print old_size3
print new_size1
print new_size2
print new_size3
assert old_size1 * 5 == new_size1
assert old_size2 * 5 == new_size2
assert old_size3 * 5 == new_size3
def test_rpc_queue_performance_single_thread(self):
round = 5000
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
client = ServiceClient(service)
client.start()
start = millis_now()
self._performance_test_one_thread(client, round, service)
end = millis_now()
client.close()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
assert False
def test_notify_queue(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_queue"]["service"])
self.client.start()
self.client.send(TestService.EVENT, "", True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(2)
results = TestService.server_repo["service_notify_queue"][
"results"]
assert len(results) == 1
assert results[0] == TestService.EVENT
def test_rpc_shuffle(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_rpc_shuffle1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, TestService.EVENT)
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_rpc_shuffle1"][
"results"]
results2 = TestService.server_repo["service_rpc_shuffle2"][
"results"]
assert len(results1) + len(results2) == 1
def test():
try:
from bigsec_common.metrics.metricsagent import MetricsAgent, set_async_mode
set_async_mode()
except:
pass
metrics_url = {
"server": "redis",
"app": "alaskan",
"redis": {
"host": "127.0.0.1",
"port": 6379,
# 'password': '******',
"type": "redis"
}
}
MetricsAgent.get_instance().initialize_by_dict(metrics_url)
post_stalker_recorder = MyMetricsRecorder(metrics_name='stalker_post_data',
db='default')
post_api_recorder = MyMetricsRecorder(metrics_name='api_request',
db='default')
RedisCtx.get_instance().host = '127.0.0.1'
RedisCtx.get_instance().port = 6379
# RedisCtx.get_instance().password = '******'
try:
meta = ServiceMeta.from_json(Babel_Conf)
babel_client = ServiceClient(meta)
babel_client.start()
now = millis_now()
event = Event("warden", "stalker_data", "key", now, exp_notice)
babel_client.notify(event, event.key, timeout=5)
print('notice notify publish [ok]')
time.sleep(1)
check_result = requests.get(
'http://localhost:8088/alaskan/get_blacklist?ip={}'.format(
exp_notice.get('ip'))).content
if 'true' in check_result:
print('check recv babel [ok]')
else:
print('check recv babel [error]')
except:
import traceback
print('notice notify publish [error][{}]'.format(
traceback.format_exc()))
def test_rpc_topic(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_polling_topic1"]["service"])
self.client.start()
result = self.client.send(TestService.EVENT, "", True, 1)
assert result == (True, [TestService.EVENT, TestService.EVENT])
if TestService.CLIENT and TestService.SERVER:
results1 = TestService.server_repo["service_polling_topic1"][
"results"]
results2 = TestService.server_repo["service_polling_topic2"][
"results"]
assert len(results1) == 1
assert len(results2) == 1
def get_misclog_rpc_client(amqp_url):
misc_service = """
{
"name": "misclog_notify",
"callmode": "notify",
"delivermode": "topic",
"serverimpl": "rabbitmq",
"coder": "mail",
"options": {
}
}
"""
client = ServiceClient(ServiceMeta.from_json(misc_service),
amqp_url=amqp_url)
return client
def get_httplog_rpc_client(amqp_url):
http_service = """
{
"name": "httplog_notify",
"callmode": "notify",
"delivermode": "sharding",
"serverimpl": "rabbitmq",
"coder": "mail",
"options": {
"servercardinality": 2
}
}
"""
client = ServiceClient(ServiceMeta.from_json(http_service),
amqp_url=amqp_url)
return client
def test_notify_topic_sharding(self):
if TestService.CLIENT:
self.client = ServiceClient(
TestService.server_repo["service_notify_tosharding1"]
["service"])
self.client.start()
for i in range(20):
ev = TestService.EVENT.copy()
ev.key = str(i)
self.client.send(ev, ev.key, True, 1)
if TestService.CLIENT and TestService.SERVER:
sleep(1)
results1 = TestService.server_repo["service_notify_tosharding1"][
"results"]
results2 = TestService.server_repo["service_notify_tosharding2"][
"results"]
results3 = TestService.server_repo["service_notify_tosharding3"][
"results"]
assert len(results1) + len(results2) == 20
assert len(results3) == 20
def test_rpc_queue_performance_multiple_thread(self):
round = 5000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_rpc_queue"]["service"]
self.client = ServiceClient(service)
self.client.start()
start = millis_now()
threads = []
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
self.client, round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
self.client.close()
assert False
return
def test_notify_queue_performance_multiple_thread(self):
round = 10000
numOfThreads = 10
if TestService.CLIENT:
service = TestService.server_repo["service_notify_queue"][
"service"]
threads = []
clients = [ServiceClient(service) for i in range(numOfThreads)]
for client in clients:
client.start()
start = millis_now()
for i in range(numOfThreads):
threads.append(
run_in_thread(self._performance_test_one_thread,
clients[i], round / numOfThreads, service))
for i in range(numOfThreads):
threads[i].join()
end = millis_now()
print "spent {} milliseconds on sending {} records, {} per second on average".format(
(end - start), round, round * 1000.0 / (end - start))
if TestService.CLIENT and TestService.SERVER:
results = TestService.server_repo["service_notify_queue"][
"results"]
current = millis_now()
while len(results) != round and (current - start) <= 40000:
sleep(0.1)
current = millis_now()
print "spent {} milliseconds on receiving {} records, {} per second on average".format(
(current - start), len(results),
len(results) * 1000.0 / (current - start))
for client in clients:
client.close()
assert False
def get_misclog_rpc_client():
misc_service = get_babel_file_content('Misclog_redis.service')
client = ServiceClient(ServiceMeta.from_json(misc_service))
return client
def get_httplog_rpc_client():
http_service = get_babel_file_content('Httplog_redis.service')
client = ServiceClient(ServiceMeta.from_json(http_service))
return client
