def create_message():
msg = Message(topic)
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_property('property', 'test')
msg.set_body('message body')
return msg
def test_producer_send_orderly_with_sharding_key(orderly_producer):
msg = Message('test')
msg.set_keys('sharding_message')
msg.set_tags('sharding')
msg.set_body('sharding message')
msg.set_property('property', 'test')
ret = orderly_producer.send_orderly_with_sharding_key(msg, 'order1')
assert ret.status == SendStatus.OK
def _send_test_msg(producer):
msg = Message('test')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
msg.set_property('property', 'test')
ret = producer.send_sync(msg)
assert ret.status == SendStatus.OK
class RocketProducer():
produce = None
message = None
# 初始化配置文件
def __init__(self, conf):
pid = uuid.uuid1().hex
self.produce = Producer("pid")
self.produce.set_namesrv_addr(conf)
# 开始链接
def start(self):
return self.produce.start()
# 发送同步消息
def sendSync(self, topic, msg, group='', keys='', tags=''):
if not topic or not msg:
print('消息topic或msg不能为空')
return
msg = bytes(json.dumps(msg), encoding="utf8")
if not self.message:
self.getMessageInstance(topic)
if keys:
self.setkeys(keys)
if tags:
self.setTags(tags)
self.message.set_body(msg)
print('sending')
response = self.produce.send_sync(self.message)
return response
# 发送延时消息 level 为延时等级，总共12级 最大2h
def sendDelay(self, topic, msg, group='', keys='', tags='', level=None):
pass
# 获取消息队列实例
def getMessageInstance(self, topic):
self.message = Message(topic)
# 设置消息体
def setkeys(self, keys):
if keys:
return self.message.set_keys(keys)
# 设置标签
def setTags(self, tags):
if tags:
return self.message.set_tags(tags)
# 关闭链接
def close(self):
self.produce.shutdown()
def test_producer_send_orderly(producer):
msg = Message('test')
msg.set_keys('send_orderly')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_orderly(msg, 1)
assert ret.status == SendStatus.OK
def test_producer_send_async(producer):
stop_event = threading.Event()
errors = []
def on_success(result):
stop_event.set()
if not result.msg_id:
errors.append(AssertionError('Producer send_async failed'))
def on_exception(exc):
stop_event.set()
errors.append(exc)
msg = Message('test')
msg.set_keys('send_async')
msg.set_tags('XXX')
msg.set_body('XXXX')
producer.send_async(msg, on_success, on_exception)
max_wait = 10
wait_count = 0
while not stop_event.is_set():
if wait_count >= max_wait:
stop_event.set()
raise Exception('test timed-out')
time.sleep(1)
wait_count += 1
if errors:
raise errors[0]
def order_timeout(msg):
logger.info("超市消息接收时间 " + f"{datetime.now()}")
msg_body_str = msg.body.decode("utf-8")
msg_body = json.loads(msg_body_str)
order_sn = msg_body["orderSn"]
order = OrderInfo.get(OrderInfo.order_sn == order_sn)
with settings.DB.atomic() as txn:
try:
if order.status != "TRADE_SUCCESS":
order.status == "TRADE_CLOSED"
order.save()
msg = Message("order_reback")
msg.set_keys("mxshop")
msg.set_tags("reback")
msg.set_body(json.dumps({"orderSn": order_sn}))
sync_producer = Producer("order_sender")
sync_producer.set_name_server_address(
f"{settings.RocketMQ_HOST}:{settings.RocketMQ_PORT}")
sync_producer.start()
ret = sync_producer.send_sync(msg)
if ret.status != SendStatus.OK:
raise Exception("发送失败")
sync_producer.shutdown()
except Exception as e:
logger.info(e)
txn.rollback()
return ConsumeStatus.RECONSUME_LATER
return ConsumeStatus.CONSUME_SUCCESS
def test_producer_send_sync(producer):
msg = Message('test')
msg.set_keys('send_sync')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_sync(msg)
assert ret.status == SendStatus.OK
def test_producer_send_async(producer):
def on_success(msg):
print(msg)
msg = Message('test')
msg.set_keys('send_async')
msg.set_tags('XXX')
msg.set_body('XXXX')
producer.send_async(msg, on_success, None)
def send_msg(key, tag, content):
producer = Producer('LOG_MONITOR')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('LOG_MONITOR_TOPIC')
msg.set_keys('key1')
msg.set_tags('tag1')
msg.set_body(content)
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
def test_produce(self):
producer = Producer('PID-XXX')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_sync(msg)
logging.getLogger(__name__).info("msg send: %s, %s, %s", ret.status,
ret.msg_id, ret.offset)
producer.shutdown()
def concrete_realization_of_publish(self, msg):
rocket_msg = Message(self._queue_name)
rocket_msg.set_keys(msg) # 利于检索
# rocket_msg.set_tags('XXX')
rocket_msg.set_body(msg)
# print(msg)
self._producer.send_sync(rocket_msg)
def send_msg(self, *, msg_body):
msg = Message(self._topic)
msg.set_keys(self._keys)
msg.set_tags(self._tags)
msg.set_body(msg_body)
try:
ret = self._producer.send_sync(msg)
logging.getLogger(__name__).info(
"msg send status: %s, id: %s, offset: %s", ret.status,
ret.msg_id, ret.offset)
return True
except RocketMQException as e:
logging.getLogger(__name__).exception(
"failed to send message to mq", exc_info=e)
raise SendMsgException(e)
def CreateOrder(self, request, context):
parent_span = context.get_active_span()
local_execute_dict[parent_span.context.span_id] = parent_span
producer = TransactionMQProducer(group_id="mxshop",
checker_callback=self.check_callback)
producer.set_name_server_address(
f"{settings.RocketMQ_HOST}:{settings.RocketMQ_PORT}")
producer.start()
msg = Message("order_reback")
msg.set_keys("mxshop")
msg.set_tags("order")
order_sn = generate_order_sn(request.userId)
msg_body = {
'orderSn': order_sn,
"userId": request.userId,
"address": request.address,
"name": request.name,
"mobile": request.mobile,
"post": request.post,
"parent_span_id": parent_span.context.span_id
}
msg.set_body(json.dumps(msg_body))
ret = producer.send_message_in_transaction(msg,
self.local_execute,
user_args=None)
logger.info(f"Send status: {ret.status}, id: {ret.msg_id}")
if ret.status != SendStatus.OK:
context.set_code(grpc.StatusCode.INTERNAL)
context.set_details("Create order failed")
return order_pb2.OrderInfoResponse()
while True:
if order_sn in local_execute_dict:
context.set_code(local_execute_dict[order_sn]["code"])
context.set_details(local_execute_dict[order_sn]["detail"])
producer.shutdown()
if local_execute_dict[order_sn]["code"] == grpc.StatusCode.OK:
return order_pb2.OrderInfoResponse(
id=local_execute_dict[order_sn]["order"]["id"],
orderSn=order_sn,
total=local_execute_dict[order_sn]["order"]["total"])
else:
return order_pb2.OrderInfoResponse()
time.sleep(0.1)
def concrete_realization_of_publish(self, msg):
try:
from rocketmq.client import Message
except Exception as e:
# print(traceback.format_exc())
raise ImportError(f'rocketmq包 只支持linux和mac {str(e)}')
rocket_msg = Message(self._queue_name)
rocket_msg.set_keys(msg) # 利于检索
# rocket_msg.set_tags('XXX')
rocket_msg.set_body(msg)
# print(msg)
self._producer.send_sync(rocket_msg)
def oneway_publish(topic,
body,
subtopic=None,
orderly=True,
group_id='crawler'):
body = json.dumps(body)
message = Message(topic)
message.set_body(body)
if subtopic:
message.set_tags(subtopic)
with init_producer(group_id, orderly) as producer:
producer.send_oneway(message)
def test_transaction_producer():
stop_event = threading.Event()
msg_body = 'XXXX'
def on_local_execute(msg, user_args):
return TransactionStatus.UNKNOWN
def on_check(msg):
stop_event.set()
assert msg.body.decode('utf-8') == msg_body
return TransactionStatus.COMMIT
producer = TransactionMQProducer('transactionTestGroup' + str(PY_VERSION),
on_check)
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('test')
msg.set_keys('transaction')
msg.set_tags('XXX')
msg.set_body(msg_body)
producer.send_message_in_transaction(msg, on_local_execute)
while not stop_event.is_set():
time.sleep(2)
producer.shutdown()
def test_producer_send_oneway_orderly(producer):
msg = Message('test')
msg.set_keys('send_oneway_orderly')
msg.set_tags('XXX')
msg.set_body('XXXX')
producer.send_oneway_orderly(msg, 1)
def test_producer_send_async(producer):
msg = Message('test')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
producer.send_async(msg, None, None)
from rocketmq.client import Producer, Message
import sys
import json
import os
producer = Producer('PID-XXX')
producer.set_name_server_address('39.105.21.114:9876')
producer.start()
dat = {
"runId": sys.argv[1],
"type": 2,
"detail": {
"ip": os.environ['IP'],
"port": "6767"
}
}
msg = Message('c2s39d105d21d114')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body(json.dumps(dat))
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
#生产消息producer
from rocketmq.client import Producer, Message
import json
producer = Producer('PID-test')
producer.set_namesrv_addr('120.131.3.191:4324') #rocketmq队列接口地址（服务器ip:port）
producer.start()
msg_body = {"id": "001", "name": "test_mq", "message": "abcdefg"}
ss = json.dumps(msg_body).encode('utf-8')
msg = Message('topic_name') #topic名称
msg.set_keys('xxxxxx')
msg.set_tags('xxxxxx')
msg.set_body(ss) #message body
retmq = producer.send_sync(msg)
print(retmq.status, retmq.msg_id, retmq.offset)
producer.shutdown()
# 消费消息consumer：
# 使用PullConsumer
from rocketmq.client import PullConsumer
consumer = PullConsumer('CID_test')
consumer.set_namesrv_addr('xxx.xxx.xxx.xxx:xxxxx')
consumer.start()
for msg in consumer.pull('topic_name'):
print(msg.id, msg.body)
consumer.shutdown()
def get_msg_instance(topic,
body,
keys='',
tags='',
property=None,
delay_time_level=0):
msg = Message(topic)
if keys:
msg.set_keys(keys)
if tags:
msg.set_tags(tags)
if property:
msg.set_property(property[0], property[1])
if delay_time_level:
msg.set_delay_time_level(delay_time_level)
body = str(body)
# body = bytes(json.dumps(body),encoding="utf8")
msg.set_body(body)
return msg
from rocketmq.client import Producer, Message
producer = Producer('PID-XXX')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
@Author : cjh
@Version : 1.0
@Desciption : None
"""
import os
from pyspark import SparkConf
from pyspark.sql import SparkSession
from rocketmq.client import Producer, Message
import getpass
env_dist = os.environ
print("HOME:", env_dist.get('HOME'))
os.environ['HOME'] = "/var/log"
conf = SparkConf().setAppName('cjh')
print("HOME:", env_dist.get('HOME'))
spark = SparkSession.builder.config(conf=conf).getOrCreate()
spark.range(10).show()
producer = Producer('PID-123')
producer.set_name_server_address('192.168.9.214:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('test')
msg.set_body('msg coming')
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
spark.stop()
def test_producer_send_batch(producer):
batch_msg = []
msg = Message('test')
msg.set_keys('send_batch_1')
msg.set_tags('XXX1')
msg.set_body('XXXX1')
batch_msg.append(msg)
msg = Message('test')
msg.set_keys('send_batch_2')
msg.set_tags('XXX2')
msg.set_body('XXXX2')
batch_msg.append(msg)
ret = producer.send_batch(batch_msg)
assert ret.status == SendStatus.OK
"""
@File : rocketmq_test.py
@Modify Time @Author @Version @Desciption
------------ ------- -------- -----------
2021/3/11 14:15 SeafyLiang 1.0 None
"""
# rocketmq==0.4.4
from rocketmq.client import Producer, Message
import time
from rocketmq.client import PushConsumer
from rocketmq.client import PullConsumer
import json
con_str = '192.168.10.35:9876'
msg = Message('test') # topic
def produce_msg():
"""
推送消息
"""
producer = Producer('PID-001') # 随便
producer.set_namesrv_addr(con_str) # ip和端口
# 推送消息的时候，如果消息所占字节太长，需要手动设置size，代码中设置的是1M。
# producer = Producer('PID-001', max_message_size=1024 * 1024)
producer.start()
msg.set_keys('2020-12-15')
msg.set_tags('explain')
msg.set_body('{"key":"value"}')
"""
def local_execute(self, msg, user_args):
msg_body = json.loads(msg.body.decode("utf-8"))
order_sn = msg_body["orderSn"]
local_execute_dict[order_sn] = {}
parent_span = local_execute_dict[msg_body["parent_span_id"]]
with settings.DB.atomic() as txn:
goods_ids = []
goods_nums = {}
order_amount = 0
order_goods_list = []
tracer = opentracing.global_tracer()
with tracer.start_span("select_shopcart") as select_shopchart_span:
for cart_item in ShoppingCart.select().where(
ShoppingCart.user == msg_body["userId"],
ShoppingCart.checked == True):
goods_ids.append(cart_item.goods)
goods_nums[cart_item.goods] = cart_item.nums
if not goods_ids:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.NOT_FOUND
local_execute_dict[order_sn][
"detail"] = "No item in shopping cart"
return TransactionStatus.ROLLBACK
# query goods info from goods srv
with tracer.start_span("query_goods",
child_of=parent_span) as query_goods_span:
register = consul.ConsulRegister(settings.CONSUL_HOST,
settings.CONSUL_POST)
goods_srv_host, goods_srv_port = register.get_host_port(
f'Service == "{settings.Goods_srv_name}"')
if not goods_srv_host or not goods_srv_port:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.NOT_FOUND
local_execute_dict[order_sn][
"detail"] = "Goods service not available"
return TransactionStatus.ROLLBACK
goods_channel = grpc.insecure_channel(
f"{goods_srv_host}:{goods_srv_port}")
goods_stub = goods_pb2_grpc.GoodsStub(goods_channel)
goods_sell_info = []
try:
goods_rsp = goods_stub.BatchGetGoods(
goods_pb2.BatchGoodsIdInfo(id=goods_ids))
for good in goods_rsp.data:
order_amount += good.shopPrice * goods_nums[good.id]
order_goods = OrderGoods(
goods=good.id,
goods_name=good.name,
goods_image=good.goodsFrontImage,
goods_price=good.shopPrice,
nums=goods_nums[good.id])
order_goods_list.append(order_goods)
goods_sell_info.append(
inventory_pb2.GoodsInvInfo(
goodsId=good.id, num=goods_nums[good.id]))
except grpc.RpcError as e:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNAL
local_execute_dict[order_sn]["detail"] = str(e)
return TransactionStatus.ROLLBACK
# prepare half message
with tracer.start_span("query_inv",
child_of=parent_span) as query_inv_span:
inventory_host, inventory_port = register.get_host_port(
f'Service == "{settings.Inventory_srv_name}"')
if not inventory_host or not inventory_port:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNA
local_execute_dict[order_sn][
"detail"] = "Inventory service not available"
return TransactionStatus.ROLLBACK
inventory_channel = grpc.insecure_channel(
f"{inventory_host}:{inventory_port}")
inventory_channel = grpc.intercept_channel(
inventory_channel, RetryInterceptor)
inv_stub = inventory_pb2_grpc.InventoryStub(inventory_channel)
try:
inv_stub.Sell(
inventory_pb2.SellInfo(goodsInfo=goods_sell_info,
orderSn=order_sn))
except grpc.RpcError as e:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNAL
local_execute_dict[order_sn]["detail"] = str(e)
err_code = e.code()
if err_code == grpc.StatusCode.UNKNOWN or grpc.StatusCode.DEADLINE_EXCEEDED:
return TransactionStatus.COMMIT
else:
return TransactionStatus.ROLLBACK
with tracer.start_span("insert_order",
child_of=parent_span) as insert_order_span:
try:
order = OrderInfo()
order.user = msg_body["userId"]
order.order_sn = order_sn
order.order_amount = order_amount
order.address = msg_body["address"]
order.signer_name = msg_body["name"]
order.singer_mobile = msg_body["mobile"]
order.post = msg_body["post"]
order.save()
for order_goods in order_goods_list:
order_goods.order = order.id
OrderGoods.bulk_create(order_goods_list)
ShoppingCart.delete().where(
ShoppingCart.user == msg_body["userId"],
ShoppingCart.checked == True).execute()
local_execute_dict[order_sn] = {
"code": grpc.StatusCode.OK,
"detail": "Create order succeeded",
"order": {
"id": order.id,
"orderSn": order_sn,
"total": order.order_amount
}
}
#发送延时消息
msg = Message("order_timeout")
msg.set_delay_time_level(16)
msg.set_keys("imooc")
msg.set_tags("cancel")
msg.set_body(json.dumps({"orderSn": order_sn}))
sync_producer = Producer("cancel")
sync_producer.set_name_server_address(
f"{settings.RocketMQ_HOST}:{settings.RocketMQ_PORT}")
sync_producer.start()
ret = sync_producer.send_sync(msg)
if ret.status != SendStatus.OK:
raise Exception("延时消息发送失败")
logger.info("发送延时消息时间" + datetime.now())
sync_producer.shutdown()
except Exception as e:
txn.rollback()
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNA
local_execute_dict[order_sn]["detail"] = str(e)
return TransactionStatus.COMMIT
return TransactionStatus.ROLLBACK
from rocketmq.client import Producer, Message
import json
namesrv = 'localhost:9876'
producer = Producer('PID-test')
producer.set_namesrv_addr(namesrv)
producer.start()
msg_body = {"id": "001", "name": "test_mq", "message": "abcdefg"}
ss = json.dumps(msg_body).encode('utf-8')
msg = Message('topic_name')
# msg.set_keys('xxxxxx')
# msg.set_tags('xxxxxx')
msg.set_body(ss) # message body
retmq = producer.send_sync(msg)
print(retmq.status, retmq.msg_id, retmq.offset)
producer.shutdown()
def getMessageInstance(self, topic):
self.message = Message(topic)
