def test_multiple_enqueue_dequeue(self):
obj = {"foo": "bar"}
s = Sender("/a.b.c")
r = Receiver("/a.b.c")
s.start()
for _ in range(10):
s.send(obj)
expected_message_count = 10
def on_message(msg):
assert msg == obj
nonlocal expected_message_count
if expected_message_count == 1:
r.stop()
expected_message_count -= 1
r.start()
r.on_message = on_message
thread = Thread(target=r.run)
thread.start()
thread.join()
class Event(object):
def __init__(self, name, description, params, queue_template):
self.unique_id = str(uuid4())
self.name = name
self.description = description
self.params = params
self.queue = queue_template.format(self.unique_id)
self.sender = Sender(self.queue)
def start(self):
self.sender.start()
def build_announcement(self):
data = {
"name": self.name,
"description": self.description,
"params": self.params,
"id": self.unique_id,
"queue": self.queue
}
return Message("enqueue", data)
@property
def schema(self):
schema = {
"type": "object",
"properties": self.params,
"additionalProperties": False
}
if self.params:
schema["required"] = list(self.params.keys())
return schema
def fire(self, **kwargs):
self.sender.send(kwargs)
def on_messaging(self, obj):
queue = obj["queue"]
data = obj["data"]
sender = Sender(queue)
sender.start()
logger.info("Sent Message to %s: %s", queue, str(data))
sender.send(data)
sender.close()
def start_echo_receiver(cls, queue):
sender = Sender(queue)
sender.start()
def reply(msg):
sender.send(msg)
receiver = Receiver(queue)
receiver.on_message = reply
receiver.start()
def test_keyed_sticky(self):
def make_receiver(count, obj, sem, r):
def on_message(msg):
obj.update(msg)
sem.release()
nonlocal count
count -= 1
if not count:
r.stop()
return on_message
s1 = Sender("/x.keyedsticky")
s2 = Sender("/x.keyedsticky")
obj1 = {}
obj2 = {}
sem1 = Semaphore(0)
sem2 = Semaphore(0)
r1 = Receiver("/x.keyedsticky")
r2 = Receiver("/x.keyedsticky")
r1.on_message = make_receiver(4, obj1, sem1, r1)
r1.start()
Thread(target=r1.run).start()
assert sem1.acquire(timeout=10) # Won't timeout for the first message.
assert obj1 == {}
s1.start()
s1.send({"foo": "bar"}, headers={"KEY": "1"})
assert sem1.acquire(timeout=10) # Must not timeout.
assert obj1 == {"1": {"foo": "bar"}}
r2.on_message = make_receiver(3, obj2, sem2, r2)
r2.start()
Thread(target=r2.run).start()
assert sem2.acquire(timeout=10) # Must not timeout.
assert obj2 == {"1": {"foo": "bar"}}
s2.start()
s2.send({"baz": "grr"}, headers={"KEY": "2"})
assert sem1.acquire(timeout=10)
assert sem2.acquire(timeout=10)
assert obj1 == {"1": {"foo": "bar"}, "2": {"baz": "grr"}}
assert obj2 == {"1": {"foo": "bar"}, "2": {"baz": "grr"}}
s2.send({"foo": "hello"}, headers={"KEY": "1"})
assert sem1.acquire(timeout=10)
assert sem2.acquire(timeout=10)
assert obj1 == {"1": {"foo": "hello"}, "2": {"baz": "grr"}}
assert obj2 == {"1": {"foo": "hello"}, "2": {"baz": "grr"}}
class StatusService(object):
def on_service_start(self, *args, **kwargs):
queue = self.get_service_queue_name("status")
create_status_queue(queue)
self.status_sender = Sender(queue)
self.status_sender.start()
super().on_service_start(*args, **kwargs)
def update_status(self, msg):
self.status_sender.send(msg)
def on_service_stop(self):
self.status_sender.close()
def express_event(self, name, description, params):
queue_template = self.get_service_queue_name("event/{}")
event = Event(name, description, params, queue_template)
queue_name = self.get_service_queue_name("events")
event_sender = Sender(queue_name)
event_sender.start()
headers = {"KEY": event.unique_id}
event_sender.send(event.build_announcement(), headers=headers)
self.create_event_queue(event)
event.start()
return event
def test_sticky_simple_enqueue_dequeue(self):
def make_receiver(count, msgs, sem, r):
def on_message(msg):
msgs.append(msg)
sem.release()
nonlocal count
count -= 1
if not count:
r.stop()
return on_message
sem1 = Semaphore(0)
sem2 = Semaphore(0)
msgs1 = []
msgs2 = []
op = {"foo": "bar"}
s = Sender("/a.sticky")
s.start()
r1 = Receiver("/a.sticky")
r1.on_message = make_receiver(2, msgs1, sem1, r1)
r1.start()
Thread(target=r1.run).start()
assert not sem1.acquire(timeout=2) # Assert that this times out.
assert len(msgs1) == 0
s.send(op)
assert sem1.acquire(timeout=10) # This shouldn't timeout.
assert len(msgs1) == 1
assert not sem1.acquire(timeout=2) # This should timeout again.
assert len(msgs1) == 1
r2 = Receiver("/a.sticky")
r2.on_message = make_receiver(2, msgs2, sem2, r2)
r2.start()
Thread(target=r2.run).start()
assert sem2.acquire(timeout=10) # This shouldn't timeout.
assert len(msgs2) == 1
assert not sem2.acquire(timeout=2) # This should timeout.
s.send(op)
assert sem1.acquire(timeout=10)
assert sem2.acquire(timeout=2)
assert len(msgs1) == 2
assert len(msgs2) == 2
def express_capability(self, name, description, params, handler):
check_handler_param(params, handler)
queue_template = self.get_service_queue_name("capability/{}")
capability = Capability(name, description, params, queue_template)
queue_name = self.get_service_queue_name("capabilities")
capability_sender = Sender(queue_name)
capability_sender.start()
headers = {"KEY": capability.unique_id}
capability_sender.send(capability, headers=headers)
self.create_capability_queue(capability)
thread, receiver = self.start_capability_receiver(capability, handler)
with self.capabilities_queue_lock:
self.capabilities_queue_map[capability.unique_id] = (thread,
receiver)
def test_simple_enqueue_dequeue(self):
msgs = []
s = Sender("/a.b.c")
r = Receiver("/a.b.c")
s.start()
r.start()
r.on_message = lambda msg: msgs.append(msg) or r.stop()
s.send({"foo": "bar"})
thread = Thread(target=r.run)
thread.start()
thread.join()
assert msgs == [{"foo": "bar"}]
def test_express_simple_capability_with_bad_schema(self):
receiver = Receiver("/services/test/capabilities")
receiver.start()
obj = receiver.receive()
assert len(obj.task) == 1
value = next(iter(obj.task.values()))
value.pop("id")
queue = value.pop("queue")
assert value == {
"name": "test",
"description": "testdesc",
"params": {
"arg1": {
"type": "string"
}
},
}
sender = Sender(queue)
sender.start()
with pytest.raises(SchemaValidationFailed):
sender.send({"arg2": "new-value"})
def test_express_simple_capability_with_correct_schema(self):
receiver = Receiver("/services/test/capabilities")
receiver.start()
obj = receiver.receive()
assert len(obj.task) == 1
value = next(iter(obj.task.values()))
value.pop("id")
queue = value.pop("queue")
assert value == {
"name": "test",
"description": "testdesc",
"params": {
"arg1": {
"type": "string"
}
},
}
sender = Sender(queue)
sender.start()
sender.send({"arg1": "new-value"})
assert self.service.get_value() == "new-value"
def test_keyed_enqueue_without_key_header(self):
s = Sender("/x.keyedsticky")
s.start()
with pytest.raises(RequiredFieldsMissing):
s.send({"foo": "bar"})
def test_enqueue_with_bad_schema(self):
s = Sender("/a.b.c")
s.start()
with pytest.raises(SchemaValidationFailed):
s.send({"foo": [1, 2]})
def test_enqueue_to_unknown_queue(self):
s = Sender("unknown.queue")
s.start()
with pytest.raises(QueueNotFound):
s.send({"a": "b"})
def test_enqueue_without_task(self):
s = Sender("/a.b.c")
s.start()
with pytest.raises(RequiredFieldsMissing):
s.send(None)
class RPCServer(RPC):
MAX_RPC_WORKERS = 5
def __init__(self, name, description, apis, service):
super(RPCServer, self).__init__(name, description, apis)
self.service = service
self.unique_id = "rpc-" + str(uuid4())
self.queue = service.get_service_queue_name("apis/" + self.unique_id)
self.request_queue = self.queue + "/request"
self.response_queue = self.queue + "/response"
self.sender = Sender(self.response_queue)
self.receiver = RPCReceiver(self, self.request_queue)
self.receiver_thread = Thread(target=self.receiver.run)
self.executor = ThreadPoolExecutor(self.MAX_RPC_WORKERS)
def start(self):
creator = Creator()
creator.start()
creator.create({
"queue_name": self.request_queue,
"request_schema": self.request_schema
})
creator.create({
"queue_name": self.response_queue,
"request_schema": self.response_schema
})
self.sender.start()
self.receiver.start()
self.receiver_thread.start()
self.service.app.register_rpc_server(self)
def stop(self):
# TODO: Delete the queue, too.
self.receiver.stop()
self.receiver_thread.join()
self.executor.shutdown()
def on_rpc_message(self, obj):
def make_done_callback(request_id, cmd):
def callback(future):
ex = future.exception()
if ex:
logger.warn("Internal API raised Exception.", ex)
self.sender.send({
"id": request_id,
"command": cmd,
"error": "Internal API Error."
})
return
self.sender.send({
"id": request_id,
"command": cmd,
"result": future.result()
})
return callback
request_id = obj["id"]
cmd = obj["command"]
try:
api = self[cmd]
except KeyError:
self.sender.send({
"id": request_id,
"result": False,
"error": "API not found."
})
return
args = obj.get("args", [])
kwargs = obj.get("kwargs", {})
future = self.executor.submit(api, *args, **kwargs)
future.add_done_callback(make_done_callback(request_id, cmd))
@property
def info_message(self):
return {
"name": self.name,
"description": self.description,
"apis": {name: api.info
for name, api in self.apis.items()},
"uri": self.queue
}
def register_service(self):
sender = Sender("/root/services")
sender.start()
headers = {"KEY": self.get_service_queue_name("")}
sender.send({}, headers=headers)
class DahuaMessagingBridge(object):
def __init__(self, service, host, username, pwd, cam):
self.host = host
self.cam = cam
url = "rtsp://{}:{}@{}:554/cam/realmonitor".format(username, pwd, host)
url = url + "?channel={}&subtype=0".format(cam)
self.queue = service.get_service_queue_name("stream/dahua" + str(cam))
self.sender = Sender(self.queue)
self.stream = DahuaStream(url, self.send_frame)
self.running = False
def start(self):
creator = Creator()
creator.start()
try:
creator.create({
"queue_name": self.queue,
"queue_type": "sticky",
"request_schema": {
"type": "string"
}
})
except QueueAlreadyExists:
pass
self.sender.start()
self.running = True
def stop(self):
if not self.running:
return
self.running = False
if self.stream.capture is not None:
self.stream.stop()
self.sender.close()
def send_frame(self, obj):
self.sender.send(obj)
def fingerprint(self):
mac = netutils.get_mac_address(self.host).replace(":", "-")
return "rtsp-{}-cam-{}".format(mac, self.cam)
@property
def info_message(self):
return {
"id": self.fingerprint(),
"queue": self.queue,
"active": self.stream.capture is not None
}
@staticmethod
def discover(service, username, pwd):
logger.info("Scanning for Dahua Cameras.")
res = []
for ip_obj in netutils.iter_ipv4_addresses():
net = IPv4Network(ip_obj["addr"] + "/" + ip_obj["netmask"],
strict=False)
if net.is_loopback:
continue
for ip in net.hosts():
if ip <= IPv4Address("192.168.1.115"):
continue
if ip > IPv4Address("192.168.1.120"):
continue
if DahuaMessagingBridge.check_ip(ip):
res.append(str(ip))
logger.info("Found %d Dahua Cameras.", len(res))
return [
DahuaMessagingBridge(service, str(x), username, pwd, y)
for x in res for y in range(1, 5)
]
@staticmethod
def check_ip(host):
if not DahuaMessagingBridge.checK_socket(host, 554):
return False
return "WEB SERVICE" in requests.get("http://{}/".format(host)).text
@staticmethod
def checK_socket(ip, port):
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(0.2)
try:
sock.connect((str(ip), port))
return True
except IOError:
return False
finally:
sock.close()
def push_update(self):
sender = Sender(self.APPLICATION_INFO_QUEUE)
sender.start()
sender.send(self.info_message, headers={"KEY": self.unique_id})
sender.close()
class RPCClient(RPC):
def __init__(self, rpc_info):
name = rpc_info["name"]
description = rpc_info["description"]
apis = [self.get_api_call(x) for x in rpc_info["apis"].values()]
super(RPCClient, self).__init__(name, description, apis)
self.sender = Sender(rpc_info["uri"] + "/request")
self.receiver = RPCReceiver(self, rpc_info["uri"] + "/response")
self.receiver_thread = Thread(target=self.receiver.run)
self.callbacks = {}
self.callbacks_lock = RLock()
def start(self):
self.sender.start()
self.receiver.start()
self.receiver_thread.start()
def stop(self):
self.sender.close()
self.receiver.stop()
self.receiver_thread.join()
def get_api_call(self, obj):
def make_blocking_callback(event, response_arr):
def callback(obj):
response_arr.append(obj)
event.set()
return callback
def on_invoke(obj, block, callback):
msg_id = obj["id"]
if block:
res_arr = []
event = Event()
callback = make_blocking_callback(event, res_arr)
if callback:
with self.callbacks_lock:
self.callbacks[msg_id] = callback
self.sender.send(obj)
if not block:
return
event.wait()
if "result" in res_arr[0]:
return res_arr[0]["result"]
else:
raise RemoteAPIError(res_arr[0].get("error"))
return ClientAPI.from_info(obj, on_invoke)
def on_rpc_message(self, msg):
with self.callbacks_lock:
callback = self.callbacks.pop(msg["id"])
if not callback:
return
callback(msg)