def DataHandleFlow():
"""数据分发处理模块
扮演Flow的角色，用于对Redis消息进行清洗过滤与分发处理.
"""
QRedis = QSet.requireQueue('QRedis')
QMqttPub = QSet.requireQueue('QMqttPub')
while True:
message = QRedis.get()
logger.debug('Get redis raw message {0}'.format(message))
type = message.get('type', None)
if type == 'subscribe':
logger.info("redis subscribe successfully".format())
elif type == 'message':
try:
bytes = message['data']
data = Deserializer(bytes)
logger.debug("Get channel data: {data}".format(data=data))
if data.get('type', None) == 'sensor':
logger.debug(
"Get Sensor data package, forward to Mqtt channel")
QMqttPub.put(bytes)
except DataFormatError:
logger.error(
"can not deserialize channel data: {channel}-{data}".
format(channel=message['channel'], data=message['data']))
def PollingRedisMessage(hPubsub):
"""轮询redis消息
注意该消息队列将在单独的线程中执行
:param hPubsub: redis pubsub对象
:type hPubsub: obj
:param queueSet: IPC消息队列集合
"""
logger.info("start new thread to polling redis message")
QFatal = QSet.requireQueue('QFatal')
try:
QRedis = QSet.requireQueue('QRedis')
except QSet.QueueDoesNoeExist:
QFatal.put(sys.exc_info())
while True:
# here the get_message() method using select systemcall, so it is very fast. just use this infinite loop to polling is ok
message = hPubsub.get_message()
if message is not None:
QRedis.put(message)
# be nice to system
time.sleep(0.01)
def run():
args = parser.parse_args()
conf = yaml.safe_load(args.config)
logger.info("system start")
BASE_DIR = os.path.dirname(__file__)
RPC_DIR = os.path.abspath("{base}/{rpc}".format(
base=BASE_DIR, rpc=conf['rpc']['execPath']))
logger.info("run at {0}".format(BASE_DIR))
logger.info("RPC exec path at {0}".format(RPC_DIR))
hRedisPubSub = getRedisClient(conf)
print(hRedisPubSub)
mqttClient = getMqttclient(conf)
print(mqttClient)
# this thread to handle mqtt network communication
QSet.setMultipleQueue(
QFatal=Queue(),
QRedis=Queue(),
QMqttPub=Queue(),
QRPC=Queue(),
)
QFatal = QSet.requireQueue('QFatal')
# this thread to polling Redis message
hPollingThread = threading.Thread(
target=functools.partial(PollingRedisMessage, hRedisPubSub))
hPollingThread.start()
# this thread to filter data
hFilterThread = threading.Thread(target=DataHandleFlow)
hFilterThread.start()
# this thread to publish data to mqtt
hMqttThread = threading.Thread(
target=functools.partial(MqttPublish, mqttClient))
hMqttThread.start()
# this thread to get RPC Info
hRpcThread = threading.Thread(
target=functools.partial(RpcExectuator, RPC_DIR))
hRpcThread.start()
execInfo = QFatal.get()
logging.critical("Fatal Error, exit whole program:{0}".format(execInfo))
def onMQTTDataReceived(client: Client, userdata, message: MQTTMessage):
"""接收到Mqtt消息后的回调.
:param client: paho mqtt对象
:type client: obj
:param userdata: 用户自定义数据
:type userdata: obj
:param message: Mqtt broker转发的消息
:type message: obj
"""
QRPC = QSet.requireQueue('QRPC')
try:
msgObj = Deserializer(message.payload)
QRPC.put(msgObj)
logger.debug("get message {0}".format(msgObj))
except DataFormatError:
logger.error("mqtt message deserialize failed, raw:{0}".format(
message.payload))
def MqttPublish(mqttClient):
"""用于发布Mqtt消息的线程
扮演Sink的角色。该线程从QMqttPub队列中订阅消息,然后将获取的消息推送到服务器。
请参见 pollingRedisMessage / dataHandle
:param mqttClient: paho mqtt client对象
:type mqttClient: object
"""
QMqttPub = QSet.requireQueue('QMqttPub')
while True:
message = QMqttPub.get() # type:bytes
# TODO: hardcoded mqtt topic name
remoteTopic = 'testHome/data'
logger.debug(
"publish message to remote topic {topic}, data:{data}".format(
topic=remoteTopic, data=message))
mqttClient.publish(remoteTopic, payload=message, qos=2, retain=False)