def test_consul_service_empty_list(self, mock_consul_service):
"""
测试未设置环境变量
"""
from django.conf import settings
os.environ["PERMISSION_REGISTER_NAME_HOST"] = ""
settings.CONSUL_CLIENT_ADDR = {
"HOST": "127.0.0.1",
"PORT": 8500
}
settings.SERVICE_CONF = SERVICE_CONF
try:
consul_service(SERVICE_CONF)
except ImproperlyConfigured as ex:
self.assertEqual(type(ex), ImproperlyConfigured)
def send_task(exchange, routing_key, message_code, *args, **kwargs):
"""
发送实时任务
参数：
exchange/routing_key/message_code, 创建消息服务时返回的配置信息
*args
**kwargs
settings配置：
MESSAGE_SENDER_CONF = {
"SERVICE_CONF": {
"ENV_NAME": "DLJFLS_LSDK_LDKEND",
"VALUE": "xxxxx-svc",
},
"API_PATH": "/api/sparrow_task/producer/send/",
}
"""
message_conf = get_settings_value("MESSAGE_SENDER_CONF")
service_addr = consul_service(message_conf['SERVICE_CONF'])
message_backend = "http://{}{}".format(service_addr,
message_conf['API_PATH'])
task_sender = TaskSender(message_backend)
try:
task_resutl = task_sender.send_task(exchange=exchange,
routing_key=routing_key,
message_code=message_code,
*args,
**kwargs)
return task_resutl
except Exception as ex:
raise HTTPException(ex)
def _build_url(service_conf, api_path):
"""
:param service_conf:
:param api_path:
:return:
"""
servicer_addr = consul_service(service_conf)
return "http://{}{}".format(servicer_addr, api_path)
def test_consul_parameter_no_variable(self, mock_consul_service):
"""
测试设置环境变量:
os.environ["SPARROW_SERVICE_REGISTER_NAME_HOST"] = "127.0.0.1:8001"
"""
from django.conf import settings
os.environ["PERMISSION_REGISTER_NAME_HOST"] = "127.0.0.1:8001"
settings.CONSUL_CLIENT_ADDR = {"HOST": "127.0.0.1", "PORT": 8500}
settings.SERVICE_CONF = SERVICE_CONF
addr = consul_service(SERVICE_CONF)
self.assertEqual(addr, '127.0.0.1:8001')
def test_consul_parameter_variable(self, mock_consul_service):
"""
测试未设置环境变量
"""
from django.conf import settings
os.environ["PERMISSION_REGISTER_NAME_HOST"] = ""
settings.CONSUL_CLIENT_ADDR = {"HOST": "127.0.0.1", "PORT": 8500}
settings.SERVICE_CONF = SERVICE_CONF
addr = consul_service(SERVICE_CONF)
expect_result_list = [
'{}:{}'.format(ii["ServiceAddress"], ii['ServicePort'])
for ii in CONSUL_RETURN_DATA[1]
]
self.assertEqual(addr in expect_result_list, True)
def send_task(exchange,
routing_key,
message_code,
retry_times=3,
*args,
**kwargs):
"""
发送实时任务
参数：
exchange/routing_key/message_code, 创建消息服务时返回的配置信息
*args
**kwargs
settings配置：
MESSAGE_SENDER_CONF = {
"SERVICE_CONF": {
"ENV_NAME": "DLJFLS_LSDK_LDKEND",
"VALUE": "xxxxx-svc",
},
"API_PATH": "/api/sparrow_task/producer/send/",
}
"""
message_conf = get_settings_value("MESSAGE_SENDER_CONF")
service_addr = consul_service(message_conf['SERVICE_CONF'])
message_backend = "http://{}{}".format(service_addr,
message_conf['API_PATH'])
task_sender = TaskSender(message_backend)
# 发送任务出现异常时的初始重试时间间隔
interval_time = 1
error_message = None
for _ in range(retry_times):
try:
task_result = task_sender.send_task(exchange=exchange,
routing_key=routing_key,
message_code=message_code,
*args,
**kwargs)
return task_result
except Exception as ex:
time.sleep(interval_time)
error_message = ex.__str__()
raise Exception("消息发送失败，失败原因{},重试次数{}，消息内容message_code={},消息参数{}{}".format(
error_message, retry_times, message_code, args, kwargs))
def rabbitmq_consumer(queue):
"""
SparrowRabbitmpConsumer
参数：
queue # settings里面queue_conf 配置
settings配置
SPARROW_RABBITMQ_CONSUMER_CONF = {
"MESSAGE_BROKER_CONF": {
"USER_NAME": "hg_test",
"PASSWORD": "******",
"BROKER_SERVICE_CONF": {
"ENV_NAME": "SPARROW_BROKER_HOST",
"VALUE": "sparrow-demo",
},
},
"MESSAGE_BACKEND_CONF": {
"BACKEND_SERVICE_CONF": {
"ENV_NAME": "SPARROW_BACKEND_HOST",
"VALUE": "sparrow-demo",
},
"API_PATH": "/api/sparrow_task/task/update/"
}
}
QUEUE_CONF_1 = {
"QUEUE": "ORDER_PAY_SUC_ALL",
"TARGET_FUNC_MAP": {
"ORDER_PAY_SUC_ONLINE": "message_service.task.task1"
}
}
ps:
SPARROW_RABBITMQ_CONSUMER_CONF # consumer的配置
MESSAGE_BROKER_CONF # rabbitmq配置
USER_NAME # 用户名
PASSWORD # 密码
BROKER_SERVICE_CONF # 依赖consul服务的配置
MESSAGE_BACKEND_CONF
BACKEND_SERVICE_CONF # 依赖consul服务的配置
API_PATH # api 路径
QUEUE_CONF_1 # 队列的配置
QUEUE # 队列名称
TARGET_FUNC_MAP # 队列消费的任务（字典中的键为message code，
对应的值为执行该消息的任务函数路径字符串）
"""
consumer_conf = get_settings_value('SPARROW_RABBITMQ_CONSUMER_CONF')
queue_conf = get_settings_value(queue)
message_backend_path = consumer_conf['MESSAGE_BACKEND_CONF'].get(
'API_PATH', None)
backend_service_conf = consumer_conf['MESSAGE_BACKEND_CONF'].get(
'BACKEND_SERVICE_CONF', None)
broker_service_conf = consumer_conf['MESSAGE_BROKER_CONF'].get(
'BROKER_SERVICE_CONF', None)
broker_service_username = consumer_conf['MESSAGE_BROKER_CONF'].get(
'USER_NAME', None)
broker_service_password = consumer_conf['MESSAGE_BROKER_CONF'].get(
'PASSWORD', None)
virtual_host = consumer_conf['MESSAGE_BROKER_CONF'].get(
'VIRTUAL_HOST', None)
broker_service_addr = consul_service(broker_service_conf)
broker_conf = 'amqp' + "://" + broker_service_username + ":" + broker_service_password + '@' + broker_service_addr + "/" + virtual_host
if message_backend_path:
backend_service_addr = consul_service(backend_service_conf)
consumer = RabbitMQConsumer(queue=queue_conf.get('QUEUE', None),
message_broker=broker_conf,
message_backend="http://{}{}".format(
backend_service_addr,
message_backend_path))
else:
consumer = RabbitMQConsumer(queue=queue_conf.get('QUEUE', None),
message_broker=broker_conf,
message_backend=message_backend_path)
consumer.target_func_map = queue_conf.get('TARGET_FUNC_MAP', None)
consumer.consume()
def _build_url(service_conf, api_path):
servicer_addr = consul_service(service_conf)
return "http://{}{}".format(servicer_addr, api_path)
def rabbitmq_consumer(queue):
"""
SparrowRabbitmpConsumer
参数：
queue # settings里面queue_conf 配置
settings配置
SPARROW_RABBITMQ_CONSUMER_CONF = {
"MESSAGE_BROKER_CONF": {
"USER_NAME": "",
"PASSWORD": "",
"VIRTUAL_HOST": "",
"BROKER_SERVICE_CONF": {
"ENV_NAME": "",
"VALUE": "",
},
},
"ALIYUN_RABBITMQ_BROKER": {
"HOST": "",
"PORT": "",
"VIRTUAL_HOST": '',
"ACCESS_KEY": "",
"ACCESS_SECRET": "",
"RESOURCEOWNERID": ,
"SECURITY_TOKEN": "",
},
"RABBITMQ_SELECTION": "MESSAGE_BROKER_CONF",
"MESSAGE_BACKEND_CONF": {
"BACKEND_SERVICE_CONF": {
"ENV_NAME": "",
"VALUE": "",
},
"API_PATH": ""
},
"RETRY_TIMES": 3,
"INTERVAL_TIME": 3,
"HEARTBEAT": 600,
}
QUEUE_CONF_1 = {
"QUEUE": "ORDER_PAY_SUC_ALL",
"TARGET_FUNC_MAP": {
"ORDER_PAY_SUC_ONLINE": ""
}
}
ps:
SPARROW_RABBITMQ_CONSUMER_CONF # consumer的配置
MESSAGE_BROKER_CONF # rabbitmq配置
USER_NAME # 用户名
PASSWORD # 密码
BROKER_SERVICE_CONF # 依赖consul服务的配置
MESSAGE_BACKEND_CONF
BACKEND_SERVICE_CONF # 依赖consul服务的配置
API_PATH # api 路径
RETRY_TIMES # 错误重试次数，默认3次
INTERVAL_TIME # 错误重试间隔，默认3秒
HEARTBEAT # 消费者与rabbitmq心跳检测间隔，默认600秒
QUEUE_CONF_1 # 队列的配置
QUEUE # 队列名称
TARGET_FUNC_MAP # 队列消费的任务（字典中的键为message code，
对应的值为执行该消息的任务函数路径字符串）
"""
consumer_conf = get_settings_value('SPARROW_RABBITMQ_CONSUMER_CONF')
queue_conf = get_settings_value(queue)
rabbitmq_selection = consumer_conf.get('RABBITMQ_SELECTION',
"MESSAGE_BROKER_CONF")
retry_times = consumer_conf.get('RETRY_TIMES', 3)
interval_time = consumer_conf.get('INTERVAL_TIME', 3)
consumer_heartbeat = consumer_conf.get('HEARTBEAT', 60)
backend_service_conf = consumer_conf['MESSAGE_BACKEND_CONF']
# broker_service_conf = consumer_conf['MESSAGE_BROKER_CONF'].get('BROKER_SERVICE_CONF', None)
# broker_service_username = consumer_conf['MESSAGE_BROKER_CONF'].get('USER_NAME', None)
# broker_service_password = consumer_conf['MESSAGE_BROKER_CONF'].get('PASSWORD', None)
# virtual_host = consumer_conf['MESSAGE_BROKER_CONF'].get('VIRTUAL_HOST', None)
while True:
try:
rabbitmq_conf = consumer_conf[rabbitmq_selection]
virtual_host = rabbitmq_conf.get('VIRTUAL_HOST', None)
if rabbitmq_selection == "ALIYUN_RABBITMQ_BROKER":
# 连接aliyun amqp
host = rabbitmq_conf["HOST"]
port = rabbitmq_conf["PORT"]
accessKey = rabbitmq_conf["ACCESS_KEY"]
accessSecret = rabbitmq_conf["ACCESS_SECRET"]
resourceOwnerId = rabbitmq_conf["RESOURCEOWNERID"]
# security_token = rabbitmq_conf["SECURITY_TOKEN"]
provider = aliyun_provider.AliyunCredentialsProvider(
accessKey, accessSecret, resourceOwnerId)
username = provider.get_username()
password = provider.get_password()
else:
broker_service_conf = rabbitmq_conf.get(
'BROKER_SERVICE_CONF', None)
username = rabbitmq_conf.get('USER_NAME', None)
password = rabbitmq_conf.get('PASSWORD', None)
broker_service_addr = consul_service(broker_service_conf)
host = broker_service_addr.split(':')[0]
port = broker_service_addr.split(':')[1]
message_broker_conf = {
"host": host,
"port": port,
"username": username,
"password": password,
"virtual_host": virtual_host,
}
if backend_service_conf:
consumer = RabbitMQConsumer(
queue=queue_conf.get('QUEUE', None),
message_broker_conf=message_broker_conf,
message_backend_conf=backend_service_conf,
retry_times=retry_times,
interval_time=interval_time,
heartbeat=consumer_heartbeat)
else:
consumer = RabbitMQConsumer(
queue=queue_conf.get('QUEUE', None),
message_broker_conf=message_broker_conf,
retry_times=retry_times,
interval_time=interval_time,
heartbeat=consumer_heartbeat)
consumer.target_func_map = queue_conf.get('TARGET_FUNC_MAP', None)
consumer.consume()
except KeyboardInterrupt:
break
except:
# 如果遇到exception不退出，过几秒重试一下
time.sleep(interval_time)
def rabbitmq_consumer(queue):
"""
SparrowRabbitmpConsumer
参数：
queue # settings里面queue_conf 配置
settings配置
SPARROW_RABBITMQ_CONSUMER_CONF = {
"MESSAGE_BROKER_CONF": {
"USER_NAME": "hg_test",
"PASSWORD": "******",
"BROKER_SERVICE_CONF": {
"ENV_NAME": "SPARROW_BROKER_HOST",
"VALUE": "sparrow-demo",
},
},
"MESSAGE_BACKEND_CONF": {
"BACKEND_SERVICE_CONF": {
"ENV_NAME": "SPARROW_BACKEND_HOST",
"VALUE": "sparrow-demo",
},
"API_PATH": "/api/sparrow_task/task/update/"
},
"RETRY_TIMES": 3,
"INTERVAL_TIME": 3,
"HEARTBEAT": 600,
}
QUEUE_CONF_1 = {
"QUEUE": "ORDER_PAY_SUC_ALL",
"TARGET_FUNC_MAP": {
"ORDER_PAY_SUC_ONLINE": "message_service.task.task1"
}
}
ps:
SPARROW_RABBITMQ_CONSUMER_CONF # consumer的配置
MESSAGE_BROKER_CONF # rabbitmq配置
USER_NAME # 用户名
PASSWORD # 密码
BROKER_SERVICE_CONF # 依赖consul服务的配置
MESSAGE_BACKEND_CONF
BACKEND_SERVICE_CONF # 依赖consul服务的配置
API_PATH # api 路径
RETRY_TIMES # 错误重试次数，默认3次
INTERVAL_TIME # 错误重试间隔，默认3秒
HEARTBEAT # 消费者与rabbitmq心跳检测间隔，默认600秒
QUEUE_CONF_1 # 队列的配置
QUEUE # 队列名称
TARGET_FUNC_MAP # 队列消费的任务（字典中的键为message code，
对应的值为执行该消息的任务函数路径字符串）
"""
consumer_conf = get_settings_value('SPARROW_RABBITMQ_CONSUMER_CONF')
queue_conf = get_settings_value(queue)
retry_times = consumer_conf.get('RETRY_TIMES', 3)
interval_time = consumer_conf.get('INTERVAL_TIME', 3)
consumer_heartbeat = consumer_conf.get('HEARTBEAT', 600)
message_backend_path = consumer_conf['MESSAGE_BACKEND_CONF'].get(
'API_PATH', None)
backend_service_conf = consumer_conf['MESSAGE_BACKEND_CONF'].get(
'BACKEND_SERVICE_CONF', None)
broker_service_conf = consumer_conf['MESSAGE_BROKER_CONF'].get(
'BROKER_SERVICE_CONF', None)
broker_service_username = consumer_conf['MESSAGE_BROKER_CONF'].get(
'USER_NAME', None)
broker_service_password = consumer_conf['MESSAGE_BROKER_CONF'].get(
'PASSWORD', None)
virtual_host = consumer_conf['MESSAGE_BROKER_CONF'].get(
'VIRTUAL_HOST', None)
while True:
try:
broker_service_addr = consul_service(broker_service_conf)
broker_conf = "amqp://{}:{}@{}/{}".format(broker_service_username,
broker_service_password,
broker_service_addr,
virtual_host)
if message_backend_path:
backend_service_addr = consul_service(backend_service_conf)
consumer = RabbitMQConsumer(
queue=queue_conf.get('QUEUE', None),
message_broker=broker_conf,
message_backend="http://{}{}".format(
backend_service_addr, message_backend_path),
retry_times=retry_times,
interval_time=interval_time,
heartbeat=consumer_heartbeat)
else:
consumer = RabbitMQConsumer(
queue=queue_conf.get('QUEUE', None),
message_broker=broker_conf,
message_backend=message_backend_path,
retry_times=retry_times,
interval_time=interval_time,
heartbeat=consumer_heartbeat)
consumer.target_func_map = queue_conf.get('TARGET_FUNC_MAP', None)
consumer.consume()
except KeyboardInterrupt:
break
except:
# 如果遇到exception不退出，过几秒重试一下
time.sleep(interval_time)
