def _exec(method, url, output='text', **kwargs):
""" Available output types: text (default), json, binary """
if 'username' in kwargs and 'password' in kwargs:
kwargs['auth'] = (kwargs.pop('username'), kwargs.pop('password'))
method = getattr(requests, method)
response = method(url, **kwargs)
response.raise_for_status()
if output == 'json':
output = response.json()
if output == 'binary':
output = response.content
else:
output = response.text
# noinspection PyBroadException
try:
# If the response is a Platypush JSON, extract it
output = Message.build(output)
except:
pass
return output
def on_message(client, userdata, msg):
def response_thread(msg):
set_thread_name('MQTTProcessor')
response = self.get_message_response(msg)
if not response:
return
response_topic = '{}/responses/{}'.format(self.topic, msg.id)
self.logger.info('Processing response on the MQTT topic {}: {}'.
format(response_topic, response))
self.send_message(response)
msg = msg.payload.decode('utf-8')
try: msg = Message.build(json.loads(msg))
except: pass
if not msg: return
self.logger.info('Received message on the MQTT backend: {}'.format(msg))
try:
self.on_message(msg)
except Exception as e:
self.logger.exception(e)
return
if isinstance(msg, Request):
threading.Thread(target=response_thread,
name='MQTTProcessor',
args=(msg,)).start()
def on_message(self, msg):
"""
Callback when a message is received on the backend.
It parses and posts the message on the main bus.
It should be called by the derived classes whenever
a new message should be processed.
:param msg: Received message. It can be either a key-value dictionary, a platypush.message.Message object,
or a string/byte UTF-8 encoded string
"""
msg = Message.build(msg)
if not getattr(msg, 'target') or msg.target != self.device_id:
return # Not for me
self.logger.debug('Message received on the {} backend: {}'.format(
self.__class__.__name__, msg))
if self._is_expected_response(msg):
# Expected response, trigger the response handler
clear_timeout()
self._request_context['on_response'](msg)
self.stop()
return
if isinstance(msg, StopEvent) and msg.targets_me():
self.logger.info('Received STOP event on {}'.format(
self.__class__.__name__))
self._stop = True
else:
msg.backend = self # Augment message to be able to process responses
self.bus.post(msg)
def get_message(self, queue_name=None):
queue = queue_name or self.queue
msg = self.redis.blpop(queue)[1].decode('utf-8')
try:
msg = Message.build(json.loads(msg))
except:
try:
import ast
msg = Message.build(ast.literal_eval(msg))
except:
try:
msg = json.loads(msg)
except Exception as e:
self.logger.exception(e)
return msg
def send(self,
url,
msg,
ssl_cert=None,
ssl_key=None,
ssl_cafile=None,
ssl_capath=None,
*args,
**kwargs):
"""
Sends a message to a websocket.
:param url: Websocket URL, e.g. ws://localhost:8765 or wss://localhost:8765
:type topic: str
:param msg: Message to be sent. It can be a list, a dict, or a Message object
:param ssl_cert: Path to the SSL certificate to be used, if the SSL connection requires client authentication as well (default: None)
:type ssl_cert: str
:param ssl_key: Path to the SSL key to be used, if the SSL connection requires client authentication as well (default: None)
:type ssl_key: str
:param ssl_cafile: Path to the certificate authority file if required by the SSL configuration (default: None)
:type ssl_cafile: str
:param ssl_capath: Path to the certificate authority directory if required by the SSL configuration (default: None)
:type ssl_capath: str
"""
async def send():
websocket_args = {}
if ssl_cert:
websocket_args['ssl'] = get_ssl_client_context(
ssl_cert=ssl_cert,
ssl_key=ssl_key,
ssl_cafile=ssl_cafile,
ssl_capath=ssl_capath)
async with websockets.connect(url, **websocket_args) as websocket:
try:
await websocket.send(str(msg))
except websockets.exceptions.ConnectionClosed:
self.logger.warning('Error on websocket {}: {}'.format(
url, e))
try:
msg = json.dumps(msg)
except:
pass
try:
msg = Message.build(json.loads(msg))
except:
pass
loop = get_or_create_event_loop()
loop.run_until_complete(send())
def get_message_response(msg):
redis = Redis(**bus().redis_args)
response = redis.blpop(get_redis_queue_name_by_message(msg), timeout=60)
if response and len(response) > 1:
response = Message.build(response[1])
else:
response = None
return response
def get_message(self, queue_name=None):
queue = queue_name or self.queue
data = self.redis.blpop(queue, timeout=1)
if data is None:
return
msg = data[1].decode()
try:
msg = Message.build(msg)
except Exception as e:
self.logger.debug(str(e))
try:
import ast
msg = Message.build(ast.literal_eval(msg))
except Exception as ee:
self.logger.debug(str(ee))
try:
msg = json.loads(msg)
except Exception as eee:
self.logger.exception(eee)
return msg
def _f():
processed_bytes = 0
open_brackets = 0
msg = b''
prev_ch = None
redis = self._get_redis()
while True:
if processed_bytes > self._MAX_REQ_SIZE:
self.logger.warning(
'Ignoring message longer than {} bytes from {}'.format(
self._MAX_REQ_SIZE, address[0]))
return
ch = sock.recv(1)
processed_bytes += 1
if ch == b'':
break
if ch == b'{' and prev_ch != b'\\':
open_brackets += 1
if not open_brackets:
continue
msg += ch
if ch == b'}' and prev_ch != b'\\':
open_brackets -= 1
if not open_brackets:
break
prev_ch = ch
if msg == b'':
return
msg = Message.build(msg)
self.logger.info('Received request from {}: {}'.format(
msg, address[0]))
self.on_message(msg)
response = self.get_message_response(msg)
self.logger.info(
'Processing response on the TCP backend: {}'.format(response))
if response:
sock.send(str(response).encode())
def get(self):
""" Reads one message from the Redis queue """
try:
if self.should_stop():
return
msg = self.redis.blpop(self.redis_queue, timeout=1)
if not msg or msg[1] is None:
return
msg = msg[1].decode('utf-8')
return Message.build(msg)
except Exception as e:
logger.exception(e)
def get_message_response(self, msg):
try:
redis = self._get_redis()
response = redis.blpop(get_redis_queue_name_by_message(msg),
timeout=60)
if response and len(response) > 1:
response = Message.build(response[1])
else:
response = None
return response
except Exception as e:
self.logger.error(
'Error while processing response to {}: {}'.format(
msg, str(e)))
def _on_record(self, record):
if record.topic != self.topic: return
msg = record.value.decode('utf-8')
is_platypush_message = False
try:
msg = Message.build(msg)
is_platypush_message = True
except:
pass
self.logger.info('Received message on Kafka backend: {}'.format(msg))
if is_platypush_message:
self.on_message(msg)
else:
self.on_message(KafkaMessageEvent(msg=msg))
def send_message(msg, wait_for_response=True):
msg = Message.build(msg)
if isinstance(msg, Request):
msg.origin = 'http'
if Config.get('token'):
msg.token = Config.get('token')
bus().post(msg)
if isinstance(msg, Request) and wait_for_response:
response = get_message_response(msg)
logger().debug(
'Processing response on the HTTP backend: {}'.format(response))
return response
def get(self):
""" Reads one message from the Redis queue """
msg = None
try:
msg = self.redis.blpop(self.redis_queue)
if not msg or msg[1] is None:
return
msg = msg[1].decode('utf-8')
try:
msg = json.loads(msg)
except json.decoder.JSONDecodeError:
msg = ast.literal_eval(msg)
msg = Message.build(msg)
except Exception as e:
logger.exception(e)
return msg
def send_request(self):
set_timeout(seconds=self.timeout,
on_timeout=self.on_timeout('Receiver response timed out'))
response = requests.post(
u'http://localhost:8123/execute',
json = {
'type': 'request',
'target': Config.get('device_id'),
'action': 'shell.exec',
'args': { 'cmd':'echo ping' }
}
)
clear_timeout()
response = Message.build(response.json())
self.assertTrue(isinstance(response, Response))
self.assertEqual(response.output.strip(), 'ping')
self.receiver.stop_app()
def handler(_, __, msg):
# noinspection PyShadowingNames
def response_thread(msg):
set_thread_name('MQTTProcessor')
response = self.get_message_response(msg)
if not response:
return
response_topic = '{}/responses/{}'.format(self.topic, msg.id)
self.logger.info(
'Processing response on the MQTT topic {}: {}'.format(
response_topic, response))
self.send_message(response, topic=response_topic)
msg = msg.payload.decode('utf-8')
# noinspection PyBroadException
try:
msg = json.loads(msg)
msg = Message.build(msg)
except Exception as e:
self.logger.debug(str(e))
if not msg:
return
self.logger.info(
'Received message on the MQTT backend: {}'.format(msg))
try:
self.on_message(msg)
except Exception as e:
self.logger.exception(e)
return
if isinstance(msg, Request):
threading.Thread(target=response_thread,
name='MQTTProcessor',
args=(msg, )).start()
async def serve_client(websocket, path):
self.active_websockets.add(websocket)
self.logger.debug('New websocket connection from {}'.format(
websocket.remote_address[0]))
try:
while True:
if self.client_timeout:
msg = await asyncio.wait_for(
websocket.recv(), timeout=self.client_timeout)
else:
msg = await websocket.recv()
msg = Message.build(msg)
self.logger.info('Received message from {}: {}'.format(
websocket.remote_address[0], msg))
self.on_message(msg)
if isinstance(msg, Request):
response = self.get_message_response(msg) or Response()
self.logger.info(
'Processing response on the websocket backend: {}'.
format(response))
await websocket.send(str(response))
except websockets.exceptions.ConnectionClosed as e:
self.active_websockets.remove(websocket)
self.logger.debug(
'Websocket client {} closed connection'.format(
websocket.remote_address[0]))
except asyncio.TimeoutError as e:
self.active_websockets.remove(websocket)
self.logger.debug(
'Websocket connection to {} timed out'.format(
websocket.remote_address[0]))
except Exception as e:
self.logger.exception(e)
def _should_skip_last_received_msg(self, msg):
if not isinstance(msg, dict):
return True # We received something weird
is_duplicate = False
last_msg = self._last_received_msg[msg['type']]
if last_msg:
msg = Message.parse(msg)
if str(msg) == str(last_msg['body']) \
and time.time() - last_msg['time'] <= 2:
# Duplicate message sent on the Pushbullet socket within
# two seconds, ignore it
logging.debug(
'Ignoring duplicate message received on the socket')
is_duplicate = True
self._last_received_msg[msg['type']] = {
'body': msg,
'time': time.time()
}
return is_duplicate
def send_message(self, topic, msg, host=None, port=1883, tls_cafile=None,
tls_certfile=None, tls_keyfile=None,
tls_version=None, tls_ciphers=None, username=None,
password=None, *args, **kwargs):
"""
Sends a message to a topic/channel.
:param topic: Topic/channel where the message will be delivered
:type topic: str
:param msg: Message to be sent. It can be a list, a dict, or a Message object
:param host: MQTT broker hostname/IP
:type host: str
:param port: MQTT broker port (default: 1883)
:type port: int
:param tls_cafile: If TLS/SSL is enabled on the MQTT server and the certificate requires a certificate authority to authenticate it, `ssl_cafile` will point to the provided ca.crt file (default: None)
:type tls_cafile: str
:param tls_certfile: If TLS/SSL is enabled on the MQTT server and a client certificate it required, specify it here (default: None)
:type tls_certfile: str
:param tls_keyfile: If TLS/SSL is enabled on the MQTT server and a client certificate key it required, specify it here (default: None)
:type tls_keyfile: str
:param tls_version: If TLS/SSL is enabled on the MQTT server and it requires a certain TLS version, specify it here (default: None)
:type tls_version: str
:param tls_ciphers: If TLS/SSL is enabled on the MQTT server and an explicit list of supported ciphers is required, specify it here (default: None)
:type tls_ciphers: str
:param username: Specify it if the MQTT server requires authentication (default: None)
:type username: str
:param password: Specify it if the MQTT server requires authentication (default: None)
:type password: str
"""
import paho.mqtt.publish as publisher
if not host and not self.host:
raise RuntimeError('No host specified and no default host configured')
publisher_args = {
'hostname': host or self.host,
'port': port or self.port,
}
if host:
if username and password:
publisher_args['auth'] = {
'username': username,
'password': password,
}
else:
if self.username and self.password:
publisher_args['auth'] = {
'username': username,
'password': password,
}
if host:
if tls_cafile:
publisher_args['tls'] = { 'ca_certs': tls_cafile }
if tls_certfile:
publisher_args['tls']['certfile'] = tls_certfile
if tls_keyfile:
publisher_args['tls']['keyfile'] = tls_keyfile
if tls_version:
publisher_args['tls']['tls_version'] = tls_version
if tls_ciphers:
publisher_args['tls']['ciphers'] = tls_ciphers
else:
if self.tls_cafile:
publisher_args['tls'] = { 'ca_certs': self.tls_cafile }
if self.tls_certfile:
publisher_args['tls']['certfile'] = self.tls_certfile
if self.tls_keyfile:
publisher_args['tls']['keyfile'] = self.tls_keyfile
if self.tls_version:
publisher_args['tls']['tls_version'] = self.tls_version
if self.tls_ciphers:
publisher_args['tls']['ciphers'] = self.tls_ciphers
try: msg = json.dumps(msg)
except: pass
try: msg = Message.build(json.loads(msg))
except: pass
publisher.single(topic, str(msg), **publisher_args)
def _get_next_message(self):
fifo = self.response_fifo if self._request_context else self.request_fifo
with open(fifo, 'rb', 0) as f:
msg = f.readline()
return Message.build(msg) if len(msg) else None
def _parse_msg(cls, msg: Union[dict, list]):
import json
return Message.build(json.loads(json.dumps(msg)))
def _parse_msg(cls, msg: Union[str, dict]) -> dict:
return Message.build(json.loads(json.dumps(msg)))
def publish(self,
topic: str,
msg: Any,
host: Optional[str] = None,
port: int = 1883,
reply_topic: Optional[str] = None,
timeout: int = 60,
tls_cafile: Optional[str] = None,
tls_certfile: Optional[str] = None,
tls_keyfile: Optional[str] = None,
tls_version: Optional[str] = None,
tls_ciphers: Optional[str] = None,
username: Optional[str] = None,
password: Optional[str] = None):
"""
Sends a message to a topic.
:param topic: Topic/channel where the message will be delivered
:param msg: Message to be sent. It can be a list, a dict, or a Message object.
:param host: MQTT broker hostname/IP.
:param port: MQTT broker port (default: 1883).
:param reply_topic: If a ``reply_topic`` is specified, then the action will wait for a response on this topic.
:param timeout: If ``reply_topic`` is set, use this parameter to specify the maximum amount of time to
wait for a response (default: 60 seconds).
:param tls_cafile: If TLS/SSL is enabled on the MQTT server and the certificate requires a certificate authority
to authenticate it, `ssl_cafile` will point to the provided ca.crt file (default: None).
:param tls_certfile: If TLS/SSL is enabled on the MQTT server and a client certificate it required, specify it
here (default: None).
:param tls_keyfile: If TLS/SSL is enabled on the MQTT server and a client certificate key it required, specify
it here (default: None).
:param tls_version: If TLS/SSL is enabled on the MQTT server and it requires a certain TLS version, specify it
here (default: None).
:param tls_ciphers: If TLS/SSL is enabled on the MQTT server and an explicit list of supported ciphers is
required, specify it here (default: None).
:param username: Specify it if the MQTT server requires authentication (default: None).
:param password: Specify it if the MQTT server requires authentication (default: None).
"""
from paho.mqtt.client import Client
if not host and not self.host:
raise RuntimeError(
'No host specified and no default host configured')
if not host:
tls_cafile = self.tls_cafile
tls_certfile = self.tls_certfile
tls_keyfile = self.tls_keyfile
tls_version = self.tls_version
tls_ciphers = self.tls_ciphers
username = self.username
password = self.password
client = Client()
if username and password:
client.username_pw_set(username, password)
if tls_cafile:
client.tls_set(ca_certs=tls_cafile,
certfile=tls_certfile,
keyfile=tls_keyfile,
tls_version=tls_version,
ciphers=tls_ciphers)
# Try to parse it as a platypush message or dump it to JSON from a dict/list
if isinstance(msg, dict) or isinstance(msg, list):
msg = json.dumps(msg)
# noinspection PyBroadException
try:
msg = Message.build(json.loads(msg))
except:
pass
client.connect(host, port, keepalive=timeout)
response_buffer = io.BytesIO()
try:
response_received = threading.Event()
if reply_topic:
client.on_message = self._response_callback(
reply_topic=reply_topic,
event=response_received,
buffer=response_buffer)
client.subscribe(reply_topic)
client.publish(topic, str(msg))
if not reply_topic:
return
client.loop_start()
ok = response_received.wait(timeout=timeout)
if not ok:
raise TimeoutError('Response timed out')
return response_buffer.getvalue()
finally:
response_buffer.close()
# noinspection PyBroadException
try:
client.loop_stop()
except:
pass
client.disconnect()
def parse_response(response):
response = Message.build(response.json())
assert isinstance(response,
Response), 'Expected Response type, got {}'.format(
response.__class__.__name__)
return response
def publish(self,
topic: str,
msg: Any,
host: Optional[str] = None,
port: Optional[int] = None,
reply_topic: Optional[str] = None,
timeout: int = 60,
tls_cafile: Optional[str] = None,
tls_certfile: Optional[str] = None,
tls_keyfile: Optional[str] = None,
tls_version: Optional[str] = None,
tls_ciphers: Optional[str] = None,
tls_insecure: Optional[bool] = None,
username: Optional[str] = None,
password: Optional[str] = None):
"""
Sends a message to a topic.
:param topic: Topic/channel where the message will be delivered
:param msg: Message to be sent. It can be a list, a dict, or a Message object.
:param host: MQTT broker hostname/IP (default: default host configured on the plugin).
:param port: MQTT broker port (default: default port configured on the plugin).
:param reply_topic: If a ``reply_topic`` is specified, then the action will wait for a response on this topic.
:param timeout: If ``reply_topic`` is set, use this parameter to specify the maximum amount of time to
wait for a response (default: 60 seconds).
:param tls_cafile: If TLS/SSL is enabled on the MQTT server and the certificate requires a certificate authority
to authenticate it, `ssl_cafile` will point to the provided ca.crt file (default: None).
:param tls_certfile: If TLS/SSL is enabled on the MQTT server and a client certificate it required, specify it
here (default: None).
:param tls_keyfile: If TLS/SSL is enabled on the MQTT server and a client certificate key it required, specify
it here (default: None).
:param tls_version: If TLS/SSL is enabled on the MQTT server and it requires a certain TLS version, specify it
here (default: None). Supported versions: ``tls`` (automatic), ``tlsv1``, ``tlsv1.1``, ``tlsv1.2``.
:param tls_insecure: Set to True to ignore TLS insecure warnings (default: False).
:param tls_ciphers: If TLS/SSL is enabled on the MQTT server and an explicit list of supported ciphers is
required, specify it here (default: None).
:param username: Specify it if the MQTT server requires authentication (default: None).
:param password: Specify it if the MQTT server requires authentication (default: None).
"""
response_buffer = io.BytesIO()
client = None
try:
# Try to parse it as a platypush message or dump it to JSON from a dict/list
if isinstance(msg, (dict, list)):
msg = json.dumps(msg)
try:
msg = Message.build(json.loads(msg))
except Exception as e:
self.logger.debug(f'Not a valid JSON: {str(e)}')
host = host or self.host
port = port or self.port or 1883
assert host, 'No host specified'
client = self._get_client(tls_cafile=tls_cafile,
tls_certfile=tls_certfile,
tls_keyfile=tls_keyfile,
tls_version=tls_version,
tls_ciphers=tls_ciphers,
tls_insecure=tls_insecure,
username=username,
password=password)
client.connect(host, port, keepalive=timeout)
response_received = threading.Event()
if reply_topic:
client.on_message = self._response_callback(
reply_topic=reply_topic,
event=response_received,
buffer=response_buffer)
client.subscribe(reply_topic)
client.publish(topic, str(msg))
if not reply_topic:
return
client.loop_start()
ok = response_received.wait(timeout=timeout)
if not ok:
raise TimeoutError('Response timed out')
return response_buffer.getvalue()
finally:
response_buffer.close()
if client:
try:
client.loop_stop()
except Exception as e:
self.logger.warning(f'Could not stop client loop: {e}')
client.disconnect()