def test_add_attribute(self):
b = Block("blockname")
attr = MagicMock()
attr.name = "attr"
b.add_attribute(attr)
attr.set_parent.assert_called_once_with(b)
self.assertEqual({"attr":attr}, b._attributes)
self.assertIs(attr, b.attr)
def test_add_attribute(self):
b = Block("blockname")
attr = MagicMock()
attr.name = "attr"
b.add_attribute(attr)
attr.set_parent.assert_called_once_with(b)
self.assertEqual({"attr": attr}, b._attributes)
self.assertIs(attr, b.attr)
def test_add_attribute(self):
b = Block()
b.name = 'block'
b.on_changed = MagicMock(side_effect=b.on_changed)
attr = MagicMock()
b.add_attribute("attr", attr)
attr.set_parent.assert_called_once_with(b, "attr")
self.assertEqual({"attr":attr}, b.attributes)
self.assertIs(attr, b.attr)
b.on_changed.assert_called_with(
[[attr.name], attr.to_dict.return_value], True)
def test_add_attribute(self):
b = Block()
b.name = 'block'
b.on_changed = MagicMock(side_effect=b.on_changed)
attr = MagicMock()
b.add_attribute("attr", attr)
attr.set_parent.assert_called_once_with(b, "attr")
self.assertEqual({"attr": attr}, b.attributes)
self.assertIs(attr, b.attr)
b.on_changed.assert_called_with(
[[attr.name], attr.to_dict.return_value], True)
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def test_given_request_then_pass_to_correct_method(self):
endpoint = ["TestBlock", "get_things"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.block.handle_request(request)
self.method.get_response.assert_called_once_with(request)
response = self.method.get_response.return_value
self.block.parent.block_respond.assert_called_once_with(
response, request.response_queue)
def test_given_put_then_update_attribute(self):
endpoint = ["TestBlock", "test_attribute", "value"]
value = "5"
request = Put(MagicMock(), MagicMock(), endpoint, value)
self.block.handle_request(request)
self.attribute.put.assert_called_once_with(value)
self.attribute.set_value.assert_called_once_with(value)
response = self.block.parent.block_respond.call_args[0][0]
self.assertEqual("malcolm:core/Return:1.0", response.typeid)
self.assertIsNone(response.value)
response_queue = self.block.parent.block_respond.call_args[0][1]
self.assertEqual(request.response_queue, response_queue)
def test_invalid_request_fails(self):
request = MagicMock()
request.type_ = "Get"
self.assertRaises(AssertionError, self.block.handle_request, request)
def test_invalid_request_fails(self):
endpoint = ["a","b","c","d"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
request = Put(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def test_given_request_then_pass_to_correct_method(self):
endpoint = ["TestBlock", "get_things"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.block.handle_request(request)
self.method.get_response.assert_called_once_with(request)
response = self.method.get_response.return_value
self.block.parent.block_respond.assert_called_once_with(
response, request.response_queue)
def test_given_put_then_update_attribute(self):
endpoint = ["TestBlock", "test_attribute", "value"]
value = "5"
request = Put(MagicMock(), MagicMock(), endpoint, value)
self.block.handle_request(request)
self.attribute.put.assert_called_once_with(value)
self.attribute.set_value.assert_called_once_with(value)
response = self.block.parent.block_respond.call_args[0][0]
self.assertEqual("malcolm:core/Return:1.0", response.typeid)
self.assertIsNone(response.value)
response_queue = self.block.parent.block_respond.call_args[0][1]
self.assertEqual(request.response_queue, response_queue)
def test_invalid_request_fails(self):
request = MagicMock()
request.type_ = "Get"
self.assertRaises(AssertionError, self.block.handle_request, request)
def test_invalid_request_fails(self):
endpoint = ["a", "b", "c", "d"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
request = Put(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block()
block.set_parent(MagicMock(), "Test")
block.add_method('method_one', m1)
block.add_method('method_two', m2)
block.add_attribute('attr_one', a1)
block.add_attribute('attr_two', a2)
m1.reset_mock()
m2.reset_mock()
a1.reset_mock()
a2.reset_mock()
expected_dict = OrderedDict()
expected_dict['typeid'] = "malcolm:core/Block:1.0"
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block()
block.set_parent(MagicMock(), "Test")
block.add_method('method_one', m1)
block.add_method('method_two', m2)
block.add_attribute('attr_one', a1)
block.add_attribute('attr_two', a2)
m1.reset_mock()
m2.reset_mock()
a1.reset_mock()
a2.reset_mock()
expected_dict = OrderedDict()
expected_dict['typeid'] = "malcolm:core/Block:1.0"
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
def test_replace_children(self):
b = Block()
b.name = "blockname"
b.methods["m1"] = 2
b.attributes["a1"] = 3
setattr(b, "m1", 2)
setattr(b, "a1", 3)
attr_meta = StringMeta(description="desc")
attr = Attribute(attr_meta)
b.add_attribute('attr', attr)
method = Method(description="desc")
b.add_method('method', method)
b.on_changed = MagicMock(wrap=b.on_changed)
b.replace_children({'attr': attr, 'method': method})
self.assertEqual(b.attributes, dict(attr=attr))
self.assertEqual(b.methods, dict(method=method))
b.on_changed.assert_called_once_with([[], b.to_dict()], True)
self.assertFalse(hasattr(b, "m1"))
self.assertFalse(hasattr(b, "a1"))
def test_replace_children(self):
b = Block()
b.name = "blockname"
b.methods["m1"] = 2
b.attributes["a1"] = 3
setattr(b, "m1", 2)
setattr(b, "a1", 3)
attr_meta = StringMeta(description="desc")
attr = Attribute(attr_meta)
b.add_attribute('attr', attr)
method = Method(description="desc")
b.add_method('method', method)
b.on_changed = MagicMock(wrap=b.on_changed)
b.replace_children({'attr':attr, 'method':method})
self.assertEqual(b.attributes, dict(attr=attr))
self.assertEqual(b.methods, dict(method=method))
b.on_changed.assert_called_once_with(
[[], b.to_dict()], True)
self.assertFalse(hasattr(b, "m1"))
self.assertFalse(hasattr(b, "a1"))
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.name = "method_one"
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.name = "method_two"
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1.name = "attr_one"
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2.name = "attr_two"
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block("Test")
block.add_method(m1)
block.add_method(m2)
block.add_attribute(a1)
block.add_attribute(a2)
expected_dict = OrderedDict()
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.name = "method_one"
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.name = "method_two"
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1.name = "attr_one"
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2.name = "attr_two"
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block("Test")
block.add_method(m1)
block.add_method(m2)
block.add_attribute(a1)
block.add_attribute(a2)
expected_dict = OrderedDict()
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block name -> block
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
self._subscriptions = OrderedDict() # block name -> list of subs
self._last_changes = OrderedDict() # block name -> list of changes
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
BlockNotify: self._handle_block_notify,
BlockChanged: self._handle_block_changed,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception:
self.log_exception("Exception while handling %s", request)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s in self._other_spawned:
s.wait(timeout=timeout)
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
self._other_spawned.append(
self.sync_factory.spawn(block.handle_request, request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
Lock: New lock
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
spawned = self.sync_factory.spawn(function, *args, **kwargs)
self._other_spawned.append(spawned)
return spawned
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
a = Attribute(StringArrayMeta(
description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(StringArrayMeta(
description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block.remoteBlocks.set_value(remotes)
def notify_subscribers(self, block_name):
self.q.put(BlockNotify(name=block_name))
def _handle_block_notify(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
for delta in self._last_changes.setdefault(request.name, []):
self._block_state_cache.delta_update(delta)
for subscription in self._subscriptions.setdefault(request.name, []):
endpoint = subscription.endpoint
# find stuff that's changed that is relevant to this subscriber
changes = []
for change in self._last_changes[request.name]:
change_path = change[0]
# look for a change_path where the beginning matches the
# endpoint path, then strip away the matching part and add
# to the change set
i = 0
for (cp_element, ep_element) in zip(change_path, endpoint):
if cp_element != ep_element:
break
i += 1
else:
# change has matching path, so keep it
# but strip off the end point path
filtered_change = [change_path[i:]] + change[1:]
changes.append(filtered_change)
if len(changes) > 0:
if subscription.delta:
# respond with the filtered changes
response = Delta(
subscription.id_, subscription.context, changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(endpoint)
response = Update(
subscription.id_, subscription.context, d)
self.log_debug("Responding to subscription %s", response)
subscription.response_queue.put(response)
self._last_changes[request.name] = []
def on_changed(self, change, notify=True):
self.q.put(BlockChanged(change=change))
if notify:
block_name = change[0][0]
self.notify_subscribers(block_name)
def _handle_block_changed(self, request):
"""Record changes to made to a block"""
# update changes
path = request.change[0]
block_changes = self._last_changes.setdefault(path[0], [])
block_changes.append(request.change)
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, name, block):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
"""
assert name not in self._blocks, \
"There is already a block called %s" % name
block.set_parent(self, name)
self.q.put(BlockAdd(block=block))
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
block = request.block
assert block.name not in self._blocks, \
"There is already a block called %s" % block.name
self._blocks[block.name] = block
self._block_state_cache[block.name] = block.to_dict()
block.lock = self.create_lock()
# Regenerate list of blocks
self.process_block.blocks.set_value(list(self._blocks))
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
subs = self._subscriptions.setdefault(request.endpoint[0], [])
subs.append(request)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
response = Return(request.id_, request.context, d)
request.response_queue.put(response)
class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block name -> block
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
self._subscriptions = OrderedDict() # block name -> list of subs
self._last_changes = OrderedDict() # block name -> list of changes
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
BlockNotify: self._handle_block_notify,
BlockChanged: self._handle_block_changed,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception:
self.log_exception("Exception while handling %s", request)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s in self._other_spawned:
s.wait(timeout=timeout)
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
self._other_spawned.append(
self.sync_factory.spawn(block.handle_request, request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
Lock: New lock
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
spawned = self.sync_factory.spawn(function, *args, **kwargs)
self._other_spawned.append(spawned)
return spawned
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
a = Attribute(
StringArrayMeta(description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(
StringArrayMeta(description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block.remoteBlocks.set_value(remotes)
def notify_subscribers(self, block_name):
self.q.put(BlockNotify(name=block_name))
def _handle_block_notify(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
for delta in self._last_changes.setdefault(request.name, []):
self._block_state_cache.delta_update(delta)
for subscription in self._subscriptions.setdefault(request.name, []):
endpoint = subscription.endpoint
# find stuff that's changed that is relevant to this subscriber
changes = []
for change in self._last_changes[request.name]:
change_path = change[0]
# look for a change_path where the beginning matches the
# endpoint path, then strip away the matching part and add
# to the change set
i = 0
for (cp_element, ep_element) in zip(change_path, endpoint):
if cp_element != ep_element:
break
i += 1
else:
# change has matching path, so keep it
# but strip off the end point path
filtered_change = [change_path[i:]] + change[1:]
changes.append(filtered_change)
if len(changes) > 0:
if subscription.delta:
# respond with the filtered changes
response = Delta(subscription.id_, subscription.context,
changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(endpoint)
response = Update(subscription.id_, subscription.context,
d)
self.log_debug("Responding to subscription %s", response)
subscription.response_queue.put(response)
self._last_changes[request.name] = []
def on_changed(self, change, notify=True):
self.q.put(BlockChanged(change=change))
if notify:
block_name = change[0][0]
self.notify_subscribers(block_name)
def _handle_block_changed(self, request):
"""Record changes to made to a block"""
# update changes
path = request.change[0]
block_changes = self._last_changes.setdefault(path[0], [])
block_changes.append(request.change)
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, name, block):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
"""
assert name not in self._blocks, \
"There is already a block called %s" % name
block.set_parent(self, name)
self.q.put(BlockAdd(block=block))
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
block = request.block
assert block.name not in self._blocks, \
"There is already a block called %s" % block.name
self._blocks[block.name] = block
self._block_state_cache[block.name] = block.to_dict()
block.lock = self.create_lock()
# Regenerate list of blocks
self.process_block.blocks.set_value(list(self._blocks))
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
subs = self._subscriptions.setdefault(request.endpoint[0], [])
subs.append(request)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
response = Return(request.id_, request.context, d)
request.response_queue.put(response)