def __init__(self):
group_id = config.get("rocketmq.group_id")
name_server_ip = config.get("rocketmq.name_server_ip")
port = config.get("rocketmq.name_server_port")
self._producer = Producer(group_id)
self._producer.set_name_server_address("{}:{}".format(
name_server_ip, port))
self._topic = config.get("rocketmq.msg_topic")
self._keys = config.get("rocketmq.msg_keys")
self._tags = config.get("rocketmq.msg_tags")
def send_message_multi_threaded(retry_time):
producer = Producer(gid)
producer.set_name_server_address(name_srv)
msg = create_message()
global MUTEX
MUTEX.acquire()
try:
producer.start()
except Exception as e:
print('ProducerStartFailed:', e)
MUTEX.release()
return
try:
for i in range(retry_time):
ret = producer.send_sync(msg)
if ret.status == 0:
print('send message status: ' + str(ret.status) + ' msgId: ' +
ret.msg_id)
break
else:
print('send message to MQ failed.')
if i == (retry_time - 1):
print('send message to MQ failed after retries.')
except Exception as e:
print('ProducerSendSyncFailed:', e)
finally:
producer.shutdown()
MUTEX.release()
return
def order_timeout(msg):
logger.info("超市消息接收时间 " + f"{datetime.now()}")
msg_body_str = msg.body.decode("utf-8")
msg_body = json.loads(msg_body_str)
order_sn = msg_body["orderSn"]
order = OrderInfo.get(OrderInfo.order_sn == order_sn)
with settings.DB.atomic() as txn:
try:
if order.status != "TRADE_SUCCESS":
order.status == "TRADE_CLOSED"
order.save()
msg = Message("order_reback")
msg.set_keys("mxshop")
msg.set_tags("reback")
msg.set_body(json.dumps({"orderSn": order_sn}))
sync_producer = Producer("order_sender")
sync_producer.set_name_server_address(
f"{settings.RocketMQ_HOST}:{settings.RocketMQ_PORT}")
sync_producer.start()
ret = sync_producer.send_sync(msg)
if ret.status != SendStatus.OK:
raise Exception("发送失败")
sync_producer.shutdown()
except Exception as e:
logger.info(e)
txn.rollback()
return ConsumeStatus.RECONSUME_LATER
return ConsumeStatus.CONSUME_SUCCESS
def send_message_sync(count):
producer = Producer(gid)
producer.set_name_server_address(name_srv)
producer.start()
for n in range(count):
msg = create_message()
ret = producer.send_sync(msg)
print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id)
print ('send sync message done')
producer.shutdown()
def send_orderly_with_sharding_key(count):
producer = Producer(gid, True)
producer.set_name_server_address(name_srv)
producer.start()
for n in range(count):
msg = create_message()
ret = producer.send_orderly_with_sharding_key(msg, 'orderId')
print ('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id)
print ('send sync order message done')
producer.shutdown()
class RmqProductor():
# 生产者实例
producer = None
'''
传入配置文件，链接namesrv
conf:namesrv配置文件
group_id:消费组id
orderly:是否是顺序消息
timeout:超时
compress_level:消息压缩水平 number
max_message_size:消息大小限制
'''
def __init__(self,conf,group_id,orderly=False, timeout=None, compress_level=None, max_message_size=None):
self.producer = Producer(group_id,orderly, timeout, compress_level)
self.producer.set_namesrv_addr(conf)
self.producer.start()
# 发送普通消息
def send_message_sync(self,topic,body,keys='',tags='',property=None,delay_time_level=0):
msg = RmqMessage.get_msg_instance(topic,body,keys,tags)
ret = self.producer.send_sync(msg)
print('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id)
return ret
def send_orderly_with_sharding_key(self,topic,body,order_id,keys='',tags='',property=None,delay_time_level=0):
msg = RmqMessage.get_msg_instance(topic, body, keys, tags, property, delay_time_level)
ret = self.producer.send_orderly_with_sharding_key(msg, order_id)
print('send message status: ' + str(ret.status) + ' msgId: ' + ret.msg_id)
print('send sync order message done')
return ret
def shutdown(self):
self.producer.shutdown()
def send_msg(key, tag, content):
producer = Producer('LOG_MONITOR')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('LOG_MONITOR_TOPIC')
msg.set_keys('key1')
msg.set_tags('tag1')
msg.set_body(content)
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
def test_produce(self):
producer = Producer('PID-XXX')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_sync(msg)
logging.getLogger(__name__).info("msg send: %s, %s, %s", ret.status,
ret.msg_id, ret.offset)
producer.shutdown()
def init_producer(group_id, orderly=True):
ns = get_nameservers()
producer = Producer(group_id, orderly)
producer.set_name_server_address(ns)
with PROD_MUTEXES_MU:
mutex = PROD_MUTEXES.get(group_id)
if not mutex:
mutex = Lock()
PROD_MUTEXES[group_id] = mutex
with mutex:
producer.start()
yield producer
producer.shutdown()
def custom_init(self):
group_id = f'g-{self._queue_name}'
if group_id not in self.__class__.group_id__rocketmq_producer: # 同一个进程中创建多个同组消费者会报错。
producer = Producer(group_id)
producer.set_namesrv_addr(frame_config.ROCKETMQ_NAMESRV_ADDR)
producer.start()
self.__class__.group_id__rocketmq_producer[group_id] = producer
else:
producer = self.__class__.group_id__rocketmq_producer[group_id]
self._producer = producer
def produce_msg():
"""
推送消息
"""
producer = Producer('PID-001') # 随便
producer.set_namesrv_addr(con_str) # ip和端口
# 推送消息的时候，如果消息所占字节太长，需要手动设置size，代码中设置的是1M。
# producer = Producer('PID-001', max_message_size=1024 * 1024)
producer.start()
msg.set_keys('2020-12-15')
msg.set_tags('explain')
msg.set_body('{"key":"value"}')
"""
1、同步发送
Producer 向 broker 发送消息，阻塞当前线程等待 broker 响应 发送结果。
2、异步发送
Producer 首先构建一个向 broker 发送消息的任务，把该任务提交给线程池，等执行完该任务时，回调用户自定义的回调函数，执行处理结果。
3、Oneway 发送
Oneway 方式只负责发送请求，不等待应答，Producer 只负责把请求发出去，而不处理响应结果。
"""
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
class RocketProducer():
produce = None
message = None
# 初始化配置文件
def __init__(self, conf):
pid = uuid.uuid1().hex
self.produce = Producer("pid")
self.produce.set_namesrv_addr(conf)
# 开始链接
def start(self):
return self.produce.start()
# 发送同步消息
def sendSync(self, topic, msg, group='', keys='', tags=''):
if not topic or not msg:
print('消息topic或msg不能为空')
return
msg = bytes(json.dumps(msg), encoding="utf8")
if not self.message:
self.getMessageInstance(topic)
if keys:
self.setkeys(keys)
if tags:
self.setTags(tags)
self.message.set_body(msg)
print('sending')
response = self.produce.send_sync(self.message)
return response
# 发送延时消息 level 为延时等级，总共12级 最大2h
def sendDelay(self, topic, msg, group='', keys='', tags='', level=None):
pass
# 获取消息队列实例
def getMessageInstance(self, topic):
self.message = Message(topic)
# 设置消息体
def setkeys(self, keys):
if keys:
return self.message.set_keys(keys)
# 设置标签
def setTags(self, tags):
if tags:
return self.message.set_tags(tags)
# 关闭链接
def close(self):
self.produce.shutdown()
def custom_init(self):
try:
from rocketmq.client import Producer
except Exception as e:
# print(traceback.format_exc())
raise ImportError(f'rocketmq包 只支持linux和mac {str(e)}')
group_id = f'g-{self._queue_name}'
if group_id not in self.__class__.group_id__rocketmq_producer: # 同一个进程中创建多个同组消费者会报错。
producer = Producer(group_id)
producer.set_namesrv_addr(frame_config.ROCKETMQ_NAMESRV_ADDR)
producer.start()
self.__class__.group_id__rocketmq_producer[group_id] = producer
else:
producer = self.__class__.group_id__rocketmq_producer[group_id]
self._producer = producer
class ScProducer(metaclass=Singleton):
"""RocketMQ message producer"""
_producer: Producer = None
_topic: str = None
_keys: str = None
_tags: str = None
def __init__(self):
group_id = config.get("rocketmq.group_id")
name_server_ip = config.get("rocketmq.name_server_ip")
port = config.get("rocketmq.name_server_port")
self._producer = Producer(group_id)
self._producer.set_name_server_address("{}:{}".format(
name_server_ip, port))
self._topic = config.get("rocketmq.msg_topic")
self._keys = config.get("rocketmq.msg_keys")
self._tags = config.get("rocketmq.msg_tags")
def start(self):
self._producer.start()
def shutdown(self):
self._producer.shutdown()
def send_msg(self, *, msg_body):
msg = Message(self._topic)
msg.set_keys(self._keys)
msg.set_tags(self._tags)
msg.set_body(msg_body)
try:
ret = self._producer.send_sync(msg)
logging.getLogger(__name__).info(
"msg send status: %s, id: %s, offset: %s", ret.status,
ret.msg_id, ret.offset)
return True
except RocketMQException as e:
logging.getLogger(__name__).exception(
"failed to send message to mq", exc_info=e)
raise SendMsgException(e)
def __init__(self,conf,group_id,orderly=False, timeout=None, compress_level=None, max_message_size=None):
self.producer = Producer(group_id,orderly, timeout, compress_level)
self.producer.set_namesrv_addr(conf)
self.producer.start()
def producer():
prod = Producer('producer_group')
prod.set_name_server_address('127.0.0.1:9876')
prod.start()
yield prod
prod.shutdown()
def orderly_producer():
prod = Producer('orderly_producer_group', True)
prod.set_name_server_address('127.0.0.1:9876')
prod.start()
yield prod
prod.shutdown()
@Author : cjh
@Version : 1.0
@Desciption : None
"""
import os
from pyspark import SparkConf
from pyspark.sql import SparkSession
from rocketmq.client import Producer, Message
import getpass
env_dist = os.environ
print("HOME:", env_dist.get('HOME'))
os.environ['HOME'] = "/var/log"
conf = SparkConf().setAppName('cjh')
print("HOME:", env_dist.get('HOME'))
spark = SparkSession.builder.config(conf=conf).getOrCreate()
spark.range(10).show()
producer = Producer('PID-123')
producer.set_name_server_address('192.168.9.214:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('test')
msg.set_body('msg coming')
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
spark.stop()
def local_execute(self, msg, user_args):
msg_body = json.loads(msg.body.decode("utf-8"))
order_sn = msg_body["orderSn"]
local_execute_dict[order_sn] = {}
parent_span = local_execute_dict[msg_body["parent_span_id"]]
with settings.DB.atomic() as txn:
goods_ids = []
goods_nums = {}
order_amount = 0
order_goods_list = []
tracer = opentracing.global_tracer()
with tracer.start_span("select_shopcart") as select_shopchart_span:
for cart_item in ShoppingCart.select().where(
ShoppingCart.user == msg_body["userId"],
ShoppingCart.checked == True):
goods_ids.append(cart_item.goods)
goods_nums[cart_item.goods] = cart_item.nums
if not goods_ids:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.NOT_FOUND
local_execute_dict[order_sn][
"detail"] = "No item in shopping cart"
return TransactionStatus.ROLLBACK
# query goods info from goods srv
with tracer.start_span("query_goods",
child_of=parent_span) as query_goods_span:
register = consul.ConsulRegister(settings.CONSUL_HOST,
settings.CONSUL_POST)
goods_srv_host, goods_srv_port = register.get_host_port(
f'Service == "{settings.Goods_srv_name}"')
if not goods_srv_host or not goods_srv_port:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.NOT_FOUND
local_execute_dict[order_sn][
"detail"] = "Goods service not available"
return TransactionStatus.ROLLBACK
goods_channel = grpc.insecure_channel(
f"{goods_srv_host}:{goods_srv_port}")
goods_stub = goods_pb2_grpc.GoodsStub(goods_channel)
goods_sell_info = []
try:
goods_rsp = goods_stub.BatchGetGoods(
goods_pb2.BatchGoodsIdInfo(id=goods_ids))
for good in goods_rsp.data:
order_amount += good.shopPrice * goods_nums[good.id]
order_goods = OrderGoods(
goods=good.id,
goods_name=good.name,
goods_image=good.goodsFrontImage,
goods_price=good.shopPrice,
nums=goods_nums[good.id])
order_goods_list.append(order_goods)
goods_sell_info.append(
inventory_pb2.GoodsInvInfo(
goodsId=good.id, num=goods_nums[good.id]))
except grpc.RpcError as e:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNAL
local_execute_dict[order_sn]["detail"] = str(e)
return TransactionStatus.ROLLBACK
# prepare half message
with tracer.start_span("query_inv",
child_of=parent_span) as query_inv_span:
inventory_host, inventory_port = register.get_host_port(
f'Service == "{settings.Inventory_srv_name}"')
if not inventory_host or not inventory_port:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNA
local_execute_dict[order_sn][
"detail"] = "Inventory service not available"
return TransactionStatus.ROLLBACK
inventory_channel = grpc.insecure_channel(
f"{inventory_host}:{inventory_port}")
inventory_channel = grpc.intercept_channel(
inventory_channel, RetryInterceptor)
inv_stub = inventory_pb2_grpc.InventoryStub(inventory_channel)
try:
inv_stub.Sell(
inventory_pb2.SellInfo(goodsInfo=goods_sell_info,
orderSn=order_sn))
except grpc.RpcError as e:
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNAL
local_execute_dict[order_sn]["detail"] = str(e)
err_code = e.code()
if err_code == grpc.StatusCode.UNKNOWN or grpc.StatusCode.DEADLINE_EXCEEDED:
return TransactionStatus.COMMIT
else:
return TransactionStatus.ROLLBACK
with tracer.start_span("insert_order",
child_of=parent_span) as insert_order_span:
try:
order = OrderInfo()
order.user = msg_body["userId"]
order.order_sn = order_sn
order.order_amount = order_amount
order.address = msg_body["address"]
order.signer_name = msg_body["name"]
order.singer_mobile = msg_body["mobile"]
order.post = msg_body["post"]
order.save()
for order_goods in order_goods_list:
order_goods.order = order.id
OrderGoods.bulk_create(order_goods_list)
ShoppingCart.delete().where(
ShoppingCart.user == msg_body["userId"],
ShoppingCart.checked == True).execute()
local_execute_dict[order_sn] = {
"code": grpc.StatusCode.OK,
"detail": "Create order succeeded",
"order": {
"id": order.id,
"orderSn": order_sn,
"total": order.order_amount
}
}
#发送延时消息
msg = Message("order_timeout")
msg.set_delay_time_level(16)
msg.set_keys("imooc")
msg.set_tags("cancel")
msg.set_body(json.dumps({"orderSn": order_sn}))
sync_producer = Producer("cancel")
sync_producer.set_name_server_address(
f"{settings.RocketMQ_HOST}:{settings.RocketMQ_PORT}")
sync_producer.start()
ret = sync_producer.send_sync(msg)
if ret.status != SendStatus.OK:
raise Exception("延时消息发送失败")
logger.info("发送延时消息时间" + datetime.now())
sync_producer.shutdown()
except Exception as e:
txn.rollback()
local_execute_dict[order_sn][
"code"] = grpc.StatusCode.INTERNA
local_execute_dict[order_sn]["detail"] = str(e)
return TransactionStatus.COMMIT
return TransactionStatus.ROLLBACK
def producer():
prod = Producer('testGroup')
prod.set_namesrv_addr('127.0.0.1:9876')
prod.start()
yield prod
prod.shutdown()
#生产消息producer
from rocketmq.client import Producer, Message
import json
producer = Producer('PID-test')
producer.set_namesrv_addr('120.131.3.191:4324') #rocketmq队列接口地址（服务器ip:port）
producer.start()
msg_body = {"id": "001", "name": "test_mq", "message": "abcdefg"}
ss = json.dumps(msg_body).encode('utf-8')
msg = Message('topic_name') #topic名称
msg.set_keys('xxxxxx')
msg.set_tags('xxxxxx')
msg.set_body(ss) #message body
retmq = producer.send_sync(msg)
print(retmq.status, retmq.msg_id, retmq.offset)
producer.shutdown()
# 消费消息consumer：
# 使用PullConsumer
from rocketmq.client import PullConsumer
consumer = PullConsumer('CID_test')
consumer.set_namesrv_addr('xxx.xxx.xxx.xxx:xxxxx')
consumer.start()
for msg in consumer.pull('topic_name'):
print(msg.id, msg.body)
consumer.shutdown()
from rocketmq.client import Producer, Message
import sys
import json
import os
producer = Producer('PID-XXX')
producer.set_name_server_address('39.105.21.114:9876')
producer.start()
dat = {
"runId": sys.argv[1],
"type": 2,
"detail": {
"ip": os.environ['IP'],
"port": "6767"
}
}
msg = Message('c2s39d105d21d114')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body(json.dumps(dat))
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
from rocketmq.client import Producer, Message
producer = Producer('PID-XXX')
producer.set_name_server_address('127.0.0.1:9876')
producer.start()
msg = Message('YOUR-TOPIC')
msg.set_keys('XXX')
msg.set_tags('XXX')
msg.set_body('XXXX')
ret = producer.send_sync(msg)
print(ret.status, ret.msg_id, ret.offset)
producer.shutdown()
from rocketmq.client import Producer, Message
import json
namesrv = 'localhost:9876'
producer = Producer('PID-test')
producer.set_namesrv_addr(namesrv)
producer.start()
msg_body = {"id": "001", "name": "test_mq", "message": "abcdefg"}
ss = json.dumps(msg_body).encode('utf-8')
msg = Message('topic_name')
# msg.set_keys('xxxxxx')
# msg.set_tags('xxxxxx')
msg.set_body(ss) # message body
retmq = producer.send_sync(msg)
print(retmq.status, retmq.msg_id, retmq.offset)
producer.shutdown()
def __init__(self, conf):
pid = uuid.uuid1().hex
self.produce = Producer("pid")
self.produce.set_namesrv_addr(conf)