def __init__(self, process, parts, params):
"""
Args:
process (Process): The process this should run under
params (Map): The parameters specified in method_takes()
parts (list): [Part]
"""
self.process = process
self.params = params
self.mri = params.mri
controller_name = "%s(%s)" % (type(self).__name__, self.mri)
self.set_logger_name(controller_name)
self.block = Block()
self.log_debug("Creating block %r as %r", self.block, self.mri)
self.lock = process.create_lock()
# {part: task}
self.part_tasks = {}
# dictionary of dictionaries
# {state (str): {Meta/MethodMeta/Attribute: writeable (bool)}
self.children_writeable = {}
# dict {hook: name}
self.hook_names = self._find_hooks()
self.parts = self._setup_parts(parts, controller_name)
self._set_block_children()
self._do_transition(sm.DISABLED, "Disabled")
self.block.set_process_path(process, [self.mri])
process.add_block(self.block, self)
self.do_initial_reset()
def setUp(self):
s = SyncFactory("sync")
self.p = Process("process", s)
self.b = Block("blockname")
self.comms = MagicMock()
serialized = dict(say_hello=dict(
description="Says hello",
takes=dict(
elements=dict(name=dict(
description="A name",
metaOf="malcolm:core/String:1.0",
), ),
required=["name"],
),
defaults={},
returns=dict(
elements=dict(greeting=dict(
description="A greeting",
metaOf="malcolm:core/String:1.0",
), ),
required=["response"],
),
), )
def f(request):
request.respond_with_return(serialized)
self.comms.q.put.side_effect = f
self.cc = ClientController(self.p, self.b, self.comms)
def __init__(self, block_name, process, parts=None, params=None):
"""
Args:
process (Process): The process this should run under
"""
controller_name = "%s(%s)" % (type(self).__name__, block_name)
self.set_logger_name(controller_name)
self.block = Block()
self.log_debug("Creating block %r as %r" % (self.block, block_name))
self.block_name = block_name
self.params = params
self.process = process
self.lock = process.create_lock()
# {part: task}
self.part_tasks = {}
# dictionary of dictionaries
# {state (str): {MethodMeta: writeable (bool)}
self.methods_writeable = {}
# dict {hook: name}
self.hook_names = self._find_hooks()
self.parts = self._setup_parts(parts, controller_name)
self._set_block_children()
self._do_transition(sm.DISABLED, "Disabled")
self.block.set_parent(process, block_name)
process.add_block(self.block)
self.do_initial_reset()
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def test_add_block(self):
p = Process("proc", MagicMock())
b = Block()
b.set_process_path(p, ("name",))
c = MagicMock()
p.add_block(b, c)
self.assertEqual(p._blocks["name"], b)
self.assertEqual(p._controllers["name"], c)
def test_add_attribute(self):
b = Block("blockname")
attr = MagicMock()
attr.name = "attr"
b.add_attribute(attr)
attr.set_parent.assert_called_once_with(b)
self.assertEqual({"attr":attr}, b._attributes)
self.assertIs(attr, b.attr)
def test_add_attribute(self):
b = Block("blockname")
attr = MagicMock()
attr.name = "attr"
b.add_attribute(attr)
attr.set_parent.assert_called_once_with(b)
self.assertEqual({"attr": attr}, b._attributes)
self.assertIs(attr, b.attr)
def test_add_block(self):
p = Process("proc", MagicMock())
b = Block()
b.set_process_path(p, ("name", ))
c = MagicMock()
p.add_block(b, c)
self.assertEqual(p._blocks["name"], b)
self.assertEqual(p._controllers["name"], c)
def setUp(self):
super(TestSystemWSCommsServerAndClient, self).setUp()
self.process2 = Process("proc2", self.sf)
self.block2 = Block()
ClientController(self.process2, self.block2, 'hello')
self.cc = WSClientComms("cc", self.process2,
"ws://localhost:%s/ws" % self.socket)
self.process2.start()
self.cc.start()
def create_process_block(self):
self.process_block = Block()
a = Attribute(
StringArrayMeta(description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(
StringArrayMeta(description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def test_put_changes_value(self):
b = Block()
b.on_changed = Mock(wraps=b.on_changed)
c = CounterController(Mock(), b, 'block')
b.on_changed.reset_mock()
c.counter.put(32)
self.assertEqual(c.counter.value, 32)
c.block.on_changed.assert_called_once_with([["counter", "value"], 32],
True)
def test_add_block_calls_handle(self):
s = SyncFactory("sched")
p = Process("proc", s)
b = Block()
b.set_parent(p, "myblock")
p.add_block(b)
p.start()
p.stop()
self.assertEqual(len(p._blocks), 2)
self.assertEqual(p._blocks, dict(myblock=b, proc=p.process_block))
def test_add_method_registers(self):
b = Block("blockname")
m = MagicMock()
m.name = "mymethod"
b.add_method(m)
self.assertEqual(b._methods.keys(), ["mymethod"])
self.assertFalse(m.called)
m.return_value = 42
self.assertEqual(b.mymethod(), 42)
m.assert_called_once_with()
def test_add_method_registers(self):
b = Block("blockname")
m = MagicMock()
m.name = "mymethod"
b.add_method(m)
self.assertEqual(b._methods.keys(), ["mymethod"])
self.assertFalse(m.called)
m.return_value = 42
self.assertEqual(b.mymethod(), 42)
m.assert_called_once_with()
def test_add_attribute(self):
b = Block()
b.name = 'block'
b.on_changed = MagicMock(side_effect=b.on_changed)
attr = MagicMock()
b.add_attribute("attr", attr)
attr.set_parent.assert_called_once_with(b, "attr")
self.assertEqual({"attr":attr}, b.attributes)
self.assertIs(attr, b.attr)
b.on_changed.assert_called_with(
[[attr.name], attr.to_dict.return_value], True)
def test_add_block_calls_handle(self):
s = SyncFactory("sched")
p = Process("proc", s)
b = Block()
c = MagicMock()
b.set_process_path(p, ("myblock",))
p.add_block(b, c)
p.start()
p.stop()
self.assertEqual(len(p._blocks), 2)
self.assertEqual(p._blocks, dict(myblock=b, proc=p.process_block))
def test_add_method_registers(self):
b = Block()
b.on_changed = MagicMock(side_effect=b.on_changed)
m = MagicMock()
b.add_method("mymethod", m)
self.assertEqual(list(b.methods), ["mymethod"])
self.assertFalse(m.called)
b.on_changed.assert_called_with([[m.name], m.to_dict.return_value], True)
m.return_value = 42
self.assertEqual(b.mymethod(), 42)
m.assert_called_once_with()
def create_process_block(self):
self.process_block = Block()
# TODO: add a meta here
children = OrderedDict()
children["blocks"] = StringArrayMeta(
description="Blocks hosted by this Process").make_attribute([])
children["remoteBlocks"] = StringArrayMeta(
description="Blocks reachable via ClientComms").make_attribute([])
self.process_block.replace_endpoints(children)
self.process_block.set_process_path(self, [self.name])
self.add_block(self.process_block, self)
def test_server_with_malcolm_client(self):
self.cc = WSClientComms("cc", self.process, "ws://localhost:8888/ws")
self.cc.start()
# Wait for comms to be connected
while not self.cc.conn.done():
time.sleep(0.001)
# Don't add to process as we already have a block of that name
block2 = Block("hello")
ClientController(self.process, block2, self.cc)
ret = block2.say_hello("me2")
self.assertEqual(ret, dict(greeting="Hello me2"))
self.cc.stop()
def test_get_block(self):
p = Process("proc", MagicMock())
p.process_block["remoteBlocks"].set_value(['name1'])
b1 = p.get_block("name1")
self.assertEqual(b1.status, "Waiting for connection...")
self.assertEqual(p.get_block("name1"), b1)
b2 = Block()
b2.set_process_path(p, ("name2",))
c = MagicMock()
p.add_block(b2, c)
self.assertEqual(p.get_block("name2"), b2)
self.assertEqual(p.get_controller("name2"), c)
def test_get_block(self):
p = Process("proc", MagicMock())
p.process_block["remoteBlocks"].set_value(['name1'])
b1 = p.get_block("name1")
self.assertEqual(b1.status, "Waiting for connection...")
self.assertEqual(p.get_block("name1"), b1)
b2 = Block()
b2.set_process_path(p, ("name2", ))
c = MagicMock()
p.add_block(b2, c)
self.assertEqual(p.get_block("name2"), b2)
self.assertEqual(p.get_controller("name2"), c)
def test_server_with_malcolm_client(self):
self.cc = WSClientComms("cc", self.process, "ws://localhost:8888/ws")
self.cc.start()
# Wait for comms to be connected
while not self.cc.conn.done():
time.sleep(0.001)
# Don't add to process as we already have a block of that name
block2 = Block("hello")
ClientController(self.process, block2, self.cc)
ret = block2.say_hello("me2")
self.assertEqual(ret, dict(greeting="Hello me2"))
self.cc.stop()
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def test_given_request_then_pass_to_correct_method(self):
endpoint = ["TestBlock", "get_things"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.block.handle_request(request)
self.method.get_response.assert_called_once_with(request)
response = self.method.get_response.return_value
self.block.parent.block_respond.assert_called_once_with(
response, request.response_queue)
def test_given_put_then_update_attribute(self):
endpoint = ["TestBlock", "test_attribute", "value"]
value = "5"
request = Put(MagicMock(), MagicMock(), endpoint, value)
self.block.handle_request(request)
self.attribute.put.assert_called_once_with(value)
self.attribute.set_value.assert_called_once_with(value)
response = self.block.parent.block_respond.call_args[0][0]
self.assertEqual("malcolm:core/Return:1.0", response.typeid)
self.assertIsNone(response.value)
response_queue = self.block.parent.block_respond.call_args[0][1]
self.assertEqual(request.response_queue, response_queue)
def test_invalid_request_fails(self):
request = MagicMock()
request.type_ = "Get"
self.assertRaises(AssertionError, self.block.handle_request, request)
def test_invalid_request_fails(self):
endpoint = ["a","b","c","d"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
request = Put(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def test_given_request_then_pass_to_correct_method(self):
endpoint = ["TestBlock", "get_things"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.block.handle_request(request)
self.method.get_response.assert_called_once_with(request)
response = self.method.get_response.return_value
self.block.parent.block_respond.assert_called_once_with(
response, request.response_queue)
def test_given_put_then_update_attribute(self):
endpoint = ["TestBlock", "test_attribute", "value"]
value = "5"
request = Put(MagicMock(), MagicMock(), endpoint, value)
self.block.handle_request(request)
self.attribute.put.assert_called_once_with(value)
self.attribute.set_value.assert_called_once_with(value)
response = self.block.parent.block_respond.call_args[0][0]
self.assertEqual("malcolm:core/Return:1.0", response.typeid)
self.assertIsNone(response.value)
response_queue = self.block.parent.block_respond.call_args[0][1]
self.assertEqual(request.response_queue, response_queue)
def test_invalid_request_fails(self):
request = MagicMock()
request.type_ = "Get"
self.assertRaises(AssertionError, self.block.handle_request, request)
def test_invalid_request_fails(self):
endpoint = ["a", "b", "c", "d"]
request = Post(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
request = Put(MagicMock(), MagicMock(), endpoint)
self.assertRaises(ValueError, self.block.handle_request, request)
def test_init(self):
block = Block()
block.add_method = Mock(wraps=block.add_method)
c = CounterController(Mock(), block, 'block')
self.assertIs(block, c.block)
self.assertEquals(3, len(block.add_method.call_args_list))
method_1 = block.add_method.call_args_list[0][0][1]
method_2 = block.add_method.call_args_list[1][0][1]
method_3 = block.add_method.call_args_list[2][0][1]
self.assertEquals("disable", method_1.name)
self.assertEquals(c.disable, method_1.func)
self.assertEquals("increment", method_2.name)
self.assertEquals(c.increment, method_2.func)
self.assertEquals("reset", method_3.name)
self.assertEquals(c.reset, method_3.func)
def test_increment_increments(self):
c = CounterController(Mock(), Block(), 'block')
self.assertEquals(0, c.counter.value)
c.increment()
self.assertEquals(1, c.counter.value)
c.increment()
self.assertEquals(2, c.counter.value)
def setUp(self):
self.sync = SyncFactory("threads")
self.process = Process("proc", self.sync)
self.block = Block("block")
self.host = socket.gethostname().split('.')[0]
self.prefix = "%s-AD-SIM-01" % self.host
pass
def test_lock_released(self):
b = Block()
b.name = "blockname"
b.lock.acquire = MagicMock(wrap=b.lock.acquire)
b.lock.release = MagicMock(wrap=b.lock.release)
lock_methods = MagicMock()
lock_methods.attach_mock(b.lock.acquire, "acquire")
lock_methods.attach_mock(b.lock.release, "release")
with b.lock:
with b.lock_released():
pass
self.assertEquals(
[call.acquire(), call.release(), call.acquire(), call.release()],
lock_methods.method_calls)
def __init__(self, process, parts, params):
"""
Args:
process (Process): The process this should run under
params (Map): The parameters specified in method_takes()
parts (list): [Part]
"""
self.process = process
self.params = params
self.mri = params.mri
controller_name = "%s(%s)" % (type(self).__name__, self.mri)
self.set_logger_name(controller_name)
self.block = Block()
self.log_debug("Creating block %r as %r", self.block, self.mri)
self.lock = process.create_lock()
# {part: task}
self.part_tasks = {}
# dictionary of dictionaries
# {state (str): {Meta/MethodMeta/Attribute: writeable (bool)}
self.children_writeable = {}
# dict {hook: name}
self.hook_names = self._find_hooks()
self.parts = self._setup_parts(parts, controller_name)
self._set_block_children()
self._do_transition(sm.DISABLED, "Disabled")
self.block.set_process_path(process, [self.mri])
process.add_block(self.block, self)
self.do_initial_reset()
class TestSystemWSCommsServerAndClient(TestSystemWSCommsServerOnly):
socket = 8882
def setUp(self):
super(TestSystemWSCommsServerAndClient, self).setUp()
self.process2 = Process("proc2", self.sf)
self.block2 = Block()
ClientController(self.process2, self.block2, 'hello')
self.cc = WSClientComms("cc", self.process2,
"ws://localhost:%s/ws" % self.socket)
self.process2.start()
self.cc.start()
def tearDown(self):
super(TestSystemWSCommsServerAndClient, self).tearDown()
self.cc.stop()
self.cc.wait()
self.process2.stop()
def test_server_with_malcolm_client(self):
# Normally we would wait for it to be connected here, but it isn't
# attached to a process so just sleep for a bit
time.sleep(0.5)
ret = self.block2.say_hello("me2")
self.assertEqual(ret, dict(greeting="Hello me2"))
def setUp(self):
s = SyncFactory("sync")
self.p = Process("process", s)
self.b = Block("blockname")
self.comms = MagicMock()
serialized = dict(
say_hello=dict(
description="Says hello",
takes=dict(
elements=dict(
name=dict(
description="A name",
metaOf="malcolm:core/String:1.0",
),
),
required=["name"],
),
defaults={},
returns=dict(
elements=dict(
greeting=dict(
description="A greeting",
metaOf="malcolm:core/String:1.0",
),
),
required=["response"],
),
),
)
def f(request):
request.respond_with_return(serialized)
self.comms.q.put.side_effect = f
self.cc = ClientController(self.p, self.b, self.comms)
def setUp(self):
self.b = Block()
self.b.name = "block"
self.c = DummyController(MagicMock(), self.b, 'block')
for attr in ["busy", "state", "status"]:
attr = self.b.attributes[attr]
attr.set_value = MagicMock(side_effect=attr.set_value)
class TestSystemWSCommsServerAndClient(TestSystemWSCommsServerOnly):
socket = 8882
def setUp(self):
super(TestSystemWSCommsServerAndClient, self).setUp()
self.process2 = Process("proc2", self.sf)
self.block2 = Block()
ClientController(self.process2, self.block2, 'hello')
self.cc = WSClientComms("cc", self.process2, "ws://localhost:%s/ws" %
self.socket)
self.process2.start()
self.cc.start()
def tearDown(self):
super(TestSystemWSCommsServerAndClient, self).tearDown()
self.cc.stop()
self.cc.wait()
self.process2.stop()
def test_server_with_malcolm_client(self):
# Normally we would wait for it to be connected here, but it isn't
# attached to a process so just sleep for a bit
time.sleep(0.5)
ret = self.block2.say_hello("me2")
self.assertEqual(ret, dict(greeting="Hello me2"))
def setUp(self):
self.sf = SyncFactory("sync")
self.process = Process("proc", self.sf)
block = Block()
HelloController(self.process, block, 'hello')
self.sc = WSServerComms("sc", self.process, self.socket)
self.process.start()
self.sc.start()
def setUp(self):
self.block = Block()
self.block.set_parent(MagicMock(), "TestBlock")
self.method = MagicMock()
self.attribute = MagicMock()
self.response = MagicMock()
self.block.add_method('get_things', self.method)
self.block.add_attribute('test_attribute', self.attribute)
def setUp(self):
super(TestSystemWSCommsServerAndClient, self).setUp()
self.process2 = Process("proc2", self.sf)
self.block2 = Block()
ClientController(self.process2, self.block2, 'hello')
self.cc = WSClientComms("cc", self.process2, "ws://localhost:%s/ws" %
self.socket)
self.process2.start()
self.cc.start()
def setUp(self):
sync_factory = SyncFactory("sync")
self.process = Process("proc", sync_factory)
block = Block("hello")
self.process.add_block(block)
HelloController(block)
self.sc = WSServerComms("sc", self.process, 8888)
self.process.start()
self.sc.start()
def test_lock_released(self):
b = Block()
b.name = "blockname"
b.lock.acquire = MagicMock(wrap=b.lock.acquire)
b.lock.release = MagicMock(wrap=b.lock.release)
lock_methods = MagicMock()
lock_methods.attach_mock(b.lock.acquire, "acquire")
lock_methods.attach_mock(b.lock.release, "release")
with b.lock:
with b.lock_released():
pass
self.assertEquals(
[call.acquire(),
call.release(),
call.acquire(),
call.release()], lock_methods.method_calls)
def create_process_block(self):
self.process_block = Block()
a = Attribute(StringArrayMeta(
description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(StringArrayMeta(
description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def setUp(self):
# Serialized version of the block we want
source = Block()
HelloController(MagicMock(), source, "blockname")
self.serialized = source.to_dict()
# Setup client controller prerequisites
self.b = Block()
self.b.name = "blockname"
self.p = MagicMock()
self.comms = MagicMock()
self.cc = ClientController(self.p, self.b, "blockname")
# get process to give us comms
self.p.get_client_comms.return_value = self.comms
# tell our controller which blocks the process can talk to
response = MagicMock(id_=self.cc.REMOTE_BLOCKS_ID, value=["blockname"])
self.cc.put(response)
# tell our controller the serialized state of the block
response = MagicMock(id_=self.cc.BLOCK_ID, changes=[[[], self.serialized]])
self.cc.put(response)
class TestClientController(unittest.TestCase):
def setUp(self):
s = SyncFactory("sync")
self.p = Process("process", s)
self.b = Block("blockname")
self.comms = MagicMock()
serialized = dict(
say_hello=dict(
description="Says hello",
takes=dict(
elements=dict(
name=dict(
description="A name",
metaOf="malcolm:core/String:1.0",
),
),
required=["name"],
),
defaults={},
returns=dict(
elements=dict(
greeting=dict(
description="A greeting",
metaOf="malcolm:core/String:1.0",
),
),
required=["response"],
),
),
)
def f(request):
request.respond_with_return(serialized)
self.comms.q.put.side_effect = f
self.cc = ClientController(self.p, self.b, self.comms)
def test_methods_created(self):
self.assertEqual(self.b._methods.keys(), ["say_hello"])
m = self.b._methods["say_hello"]
self.assertEqual(m.name, "say_hello")
self.assertEqual(m.takes.elements.keys(), ["name"])
self.assertEqual(type(m.takes.elements["name"]), StringMeta)
self.assertEqual(m.returns.elements.keys(), ["greeting"])
self.assertEqual(type(m.returns.elements["greeting"]), StringMeta)
self.assertEqual(m.defaults, {})
def test_call_method(self):
def f(request):
request.respond_with_return(dict(
greeting="Hello %s" % request.parameters["name"]))
self.comms.q.put.side_effect = f
ret = self.b.say_hello(name="me")
self.assertEqual(ret["greeting"], "Hello me")
def create_process_block(self):
self.process_block = Block()
# TODO: add a meta here
children = OrderedDict()
children["blocks"] = StringArrayMeta(
description="Blocks hosted by this Process"
).make_attribute([])
children["remoteBlocks"] = StringArrayMeta(
description="Blocks reachable via ClientComms"
).make_attribute([])
self.process_block.replace_endpoints(children)
self.process_block.set_process_path(self, [self.name])
self.add_block(self.process_block, self)
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block()
block.set_parent(MagicMock(), "Test")
block.add_method('method_one', m1)
block.add_method('method_two', m2)
block.add_attribute('attr_one', a1)
block.add_attribute('attr_two', a2)
m1.reset_mock()
m2.reset_mock()
a1.reset_mock()
a2.reset_mock()
expected_dict = OrderedDict()
expected_dict['typeid'] = "malcolm:core/Block:1.0"
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
class TestClientController(unittest.TestCase):
def setUp(self):
s = SyncFactory("sync")
self.p = Process("process", s)
self.b = Block("blockname")
self.comms = MagicMock()
serialized = dict(say_hello=dict(
description="Says hello",
takes=dict(
elements=dict(name=dict(
description="A name",
metaOf="malcolm:core/String:1.0",
), ),
required=["name"],
),
defaults={},
returns=dict(
elements=dict(greeting=dict(
description="A greeting",
metaOf="malcolm:core/String:1.0",
), ),
required=["response"],
),
), )
def f(request):
request.respond_with_return(serialized)
self.comms.q.put.side_effect = f
self.cc = ClientController(self.p, self.b, self.comms)
def test_methods_created(self):
self.assertEqual(self.b._methods.keys(), ["say_hello"])
m = self.b._methods["say_hello"]
self.assertEqual(m.name, "say_hello")
self.assertEqual(m.takes.elements.keys(), ["name"])
self.assertEqual(type(m.takes.elements["name"]), StringMeta)
self.assertEqual(m.returns.elements.keys(), ["greeting"])
self.assertEqual(type(m.returns.elements["greeting"]), StringMeta)
self.assertEqual(m.defaults, {})
def test_call_method(self):
def f(request):
request.respond_with_return(
dict(greeting="Hello %s" % request.parameters["name"]))
self.comms.q.put.side_effect = f
ret = self.b.say_hello(name="me")
self.assertEqual(ret["greeting"], "Hello me")
def test_configure(self):
params = MagicMock()
with patch("malcolm.vmetas.pointgeneratormeta.CompoundGenerator",
spec=True) as cg_mock:
params.generator = cg_mock()
params.exposure = 1
params.axis_name = "x"
block = MagicMock(wraps=Block())
sptc = ScanPointTickerController(MagicMock(), block, 'block')
sptc.configure(params)
self.assertEqual(params.generator, sptc.generator.value)
self.assertEqual(params.axis_name, sptc.axis_name.value)
self.assertEqual(params.exposure, sptc.exposure.value)
block.notify_subscribers.assert_called_once_with()
def test_add_attribute(self):
b = Block()
b.name = 'block'
b.on_changed = MagicMock(side_effect=b.on_changed)
attr = MagicMock()
b.add_attribute("attr", attr)
attr.set_parent.assert_called_once_with(b, "attr")
self.assertEqual({"attr": attr}, b.attributes)
self.assertIs(attr, b.attr)
b.on_changed.assert_called_with(
[[attr.name], attr.to_dict.return_value], True)
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block("TestBlock")
self.method = MagicMock()
self.method.name = "get_things"
self.response = MagicMock()
self.block.add_method(self.method)
def test_given_request_then_pass_to_correct_method(self):
request = MagicMock()
request.POST = "Post"
request.type_ = "Post"
request.endpoint = ["TestBlock", "device", "get_things"]
self.block.handle_request(request)
self.method.handle_request.assert_called_once_with(request)
def test_given_get_then_return_attribute(self):
self.block.state = MagicMock()
self.block.state.value = "Running"
request = MagicMock()
request.type_ = "Get"
request.endpoint = ["TestBlock", "state", "value"]
self.block.handle_request(request)
request.respond_with_return.assert_called_once_with("Running")
def test_given_get_block_then_return_self(self):
request = MagicMock()
request.type_ = "Get"
request.endpoint = ["TestBlock"]
expected_call = self.block.to_dict()
self.block.handle_request(request)
request.respond_with_return.assert_called_once_with(expected_call)
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block()
block.set_parent(MagicMock(), "Test")
block.add_method('method_one', m1)
block.add_method('method_two', m2)
block.add_attribute('attr_one', a1)
block.add_attribute('attr_two', a2)
m1.reset_mock()
m2.reset_mock()
a1.reset_mock()
a2.reset_mock()
expected_dict = OrderedDict()
expected_dict['typeid'] = "malcolm:core/Block:1.0"
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
class TestHandleRequest(unittest.TestCase):
def setUp(self):
self.block = Block("TestBlock")
self.method = MagicMock()
self.method.name = "get_things"
self.response = MagicMock()
self.block.add_method(self.method)
def test_given_request_then_pass_to_correct_method(self):
request = MagicMock()
request.POST = "Post"
request.type_ = "Post"
request.endpoint = ["TestBlock", "device", "get_things"]
self.block.handle_request(request)
self.method.handle_request.assert_called_once_with(request)
def test_given_get_then_return_attribute(self):
self.block.state = MagicMock()
self.block.state.value = "Running"
request = MagicMock()
request.type_ = "Get"
request.endpoint = ["TestBlock", "state", "value"]
self.block.handle_request(request)
request.respond_with_return.assert_called_once_with("Running")
def test_given_get_block_then_return_self(self):
request = MagicMock()
request.type_ = "Get"
request.endpoint = ["TestBlock"]
expected_call = self.block.to_dict()
self.block.handle_request(request)
request.respond_with_return.assert_called_once_with(expected_call)
def test_add_method_registers(self):
b = Block()
b.on_changed = MagicMock(side_effect=b.on_changed)
m = MagicMock()
b.add_method("mymethod", m)
self.assertEqual(list(b.methods), ["mymethod"])
self.assertFalse(m.called)
b.on_changed.assert_called_with([[m.name], m.to_dict.return_value],
True)
m.return_value = 42
self.assertEqual(b.mymethod(), 42)
m.assert_called_once_with()
def test_returns_dict(self):
method_dict = OrderedDict(takes=OrderedDict(one=OrderedDict()),
returns=OrderedDict(one=OrderedDict()),
defaults=OrderedDict())
m1 = MagicMock()
m1.name = "method_one"
m1.to_dict.return_value = method_dict
m2 = MagicMock()
m2.name = "method_two"
m2.to_dict.return_value = method_dict
a1 = MagicMock()
a1.name = "attr_one"
a1dict = OrderedDict(value="test", meta=MagicMock())
a1.to_dict.return_value = a1dict
a2 = MagicMock()
a2.name = "attr_two"
a2dict = OrderedDict(value="value", meta=MagicMock())
a2.to_dict.return_value = a2dict
block = Block("Test")
block.add_method(m1)
block.add_method(m2)
block.add_attribute(a1)
block.add_attribute(a2)
expected_dict = OrderedDict()
expected_dict['attr_one'] = a1dict
expected_dict['attr_two'] = a2dict
expected_dict['method_one'] = method_dict
expected_dict['method_two'] = method_dict
response = block.to_dict()
m1.to_dict.assert_called_once_with()
m2.to_dict.assert_called_once_with()
self.assertEqual(expected_dict, response)
class Controller(Loggable):
"""Implement the logic that takes a Block through its state machine"""
stateMachine = sm()
# Attributes for all controllers
state = None
status = None
busy = None
# BlockMeta for descriptions
meta = None
def __init__(self, process, parts, params):
"""
Args:
process (Process): The process this should run under
params (Map): The parameters specified in method_takes()
parts (list): [Part]
"""
self.process = process
self.params = params
self.mri = params.mri
controller_name = "%s(%s)" % (type(self).__name__, self.mri)
self.set_logger_name(controller_name)
self.block = Block()
self.log_debug("Creating block %r as %r", self.block, self.mri)
self.lock = process.create_lock()
# {part: task}
self.part_tasks = {}
# dictionary of dictionaries
# {state (str): {Meta/MethodMeta/Attribute: writeable (bool)}
self.children_writeable = {}
# dict {hook: name}
self.hook_names = self._find_hooks()
self.parts = self._setup_parts(parts, controller_name)
self._set_block_children()
self._do_transition(sm.DISABLED, "Disabled")
self.block.set_process_path(process, [self.mri])
process.add_block(self.block, self)
self.do_initial_reset()
def _find_hooks(self):
hook_names = {}
for n in dir(self):
attr = getattr(self, n)
if isinstance(attr, Hook):
assert attr not in hook_names, \
"Hook %s already in controller as %s" % (
n, hook_names[attr])
hook_names[attr] = n
return hook_names
def _setup_parts(self, parts, controller_name):
parts_dict = OrderedDict()
for part in parts:
part.set_logger_name("%s.%s" % (controller_name, part.name))
# Check part hooks into one of our hooks
for func_name, part_hook, _ in get_hook_decorated(part):
assert part_hook in self.hook_names, \
"Part %s func %s not hooked into %s" % (
part.name, func_name, self)
parts_dict[part.name] = part
return parts_dict
def do_initial_reset(self):
pass
def _set_block_children(self):
# reconfigure block with new children
child_list = [self.create_meta()]
child_list += list(self.create_attributes())
child_list += list(self.create_methods())
for part in self.parts.values():
child_list += list(part.create_attributes())
child_list += list(part.create_methods())
self.children_writeable = {}
writeable_functions = {}
children = OrderedDict()
for name, child, writeable_func in child_list:
if isinstance(child, Attribute):
states = child.meta.writeable_in
else:
states = child.writeable_in
children[name] = child
if states:
for state in states:
assert state in self.stateMachine.possible_states, \
"State %s is not one of the valid states %s" % \
(state, self.stateMachine.possible_states)
elif writeable_func is not None:
states = [
state for state in self.stateMachine.possible_states
if state not in (sm.DISABLING, sm.DISABLED)]
else:
continue
self.register_child_writeable(name, states)
if writeable_func:
writeable_functions[name] = functools.partial(
self.call_writeable_function, writeable_func)
self.block.replace_endpoints(children)
self.block.set_writeable_functions(writeable_functions)
def call_writeable_function(self, function, child, *args):
with self.lock:
if not child.writeable:
raise ValueError(
"Child %r is not writeable" % (child.process_path,))
result = function(*args)
return result
def create_meta(self):
self.meta = BlockMeta()
return "meta", self.meta, None
def create_attributes(self):
"""Method that should provide Attribute instances for Block
Yields:
tuple: (string name, Attribute, callable put_function).
"""
# Add the state, status and busy attributes
self.state = ChoiceMeta(
"State of Block", self.stateMachine.possible_states, label="State"
).make_attribute()
yield "state", self.state, None
self.status = StringMeta(
"Status of Block", label="Status"
).make_attribute()
yield "status", self.status, None
self.busy = BooleanMeta(
"Whether Block busy or not", label="Busy"
).make_attribute()
yield "busy", self.busy, None
def create_methods(self):
"""Method that should provide MethodMeta instances for Block
Yields:
tuple: (string name, MethodMeta, callable post_function).
"""
return get_method_decorated(self)
def transition(self, state, message):
"""
Change to a new state if the transition is allowed
Args:
state(str): State to transition to
message(str): Status message
"""
with self.lock:
if self.stateMachine.is_allowed(
initial_state=self.state.value, target_state=state):
self._do_transition(state, message)
else:
raise TypeError("Cannot transition from %s to %s" %
(self.state.value, state))
def _do_transition(self, state, message):
# transition is allowed, so set attributes
changes = []
changes.append([["state", "value"], state])
changes.append([["status", "value"], message])
changes.append([["busy", "value"],
state in self.stateMachine.busy_states])
# say which children are now writeable
for name in self.block:
try:
writeable = self.children_writeable[state][name]
except KeyError:
continue
child = self.block[name]
if isinstance(child, Attribute):
changes.append([[name, "meta", "writeable"], writeable])
elif isinstance(child, MethodMeta):
changes.append([[name, "writeable"], writeable])
for ename in child.takes.elements:
path = [name, "takes", "elements", ename, "writeable"]
changes.append([path, writeable])
self.log_debug("Transitioning to %s", state)
self.block.apply_changes(*changes)
def register_child_writeable(self, name, states):
"""
Set the states that the given method can be called in
Args:
name (str): Child name that will be set writeable or not
states (list[str]): states where method is writeable
"""
for state in self.stateMachine.possible_states:
writeable_dict = self.children_writeable.setdefault(state, {})
is_writeable = state in states
writeable_dict[name] = is_writeable
def create_part_tasks(self):
part_tasks = {}
for part_name, part in self.parts.items():
part_tasks[part] = Task("Task(%s)" % part_name, self.process)
return part_tasks
def run_hook(self, hook, part_tasks, *args, **params):
hook_queue, hook_runners = self.start_hook(
hook, part_tasks, *args, **params)
return_dict = self.wait_hook(hook_queue, hook_runners)
return return_dict
def start_hook(self, hook, part_tasks, *args, **params):
assert hook in self.hook_names, \
"Hook %s doesn't appear in controller hooks %s" % (
hook, self.hook_names)
# ask the hook to find the functions it should run
part_funcs = hook.find_hooked_functions(self.parts)
hook_runners = {}
self.log_debug("Run %s hook on %s",
self.hook_names[hook], [p.name for p in part_funcs])
# now start them off
hook_queue = self.process.create_queue()
for part, func_name in part_funcs.items():
task = weakref.proxy(part_tasks[part])
hook_runner = HookRunner(
hook_queue, part, func_name, task, *args, **params)
hook_runner.start()
hook_runners[part] = hook_runner
return hook_queue, hook_runners
def wait_hook(self, hook_queue, hook_runners):
# Wait for them all to finish
return_dict = {}
while hook_runners:
part, ret = hook_queue.get()
hook_runner = hook_runners.pop(part)
if isinstance(ret, AbortedError):
# If AbortedError, all tasks have already been stopped.
self.log_debug("Part %s Aborted", part.name)
# Do not wait on them otherwise we might get a deadlock...
raise ret
# Wait for the process to terminate
hook_runner.wait()
return_dict[part.name] = ret
self.log_debug("Part %s returned %r. Still waiting for %s",
part.name, ret, [p.name for p in hook_runners])
if isinstance(ret, Exception):
# Got an error, so stop and wait all hook runners
for h in hook_runners.values():
h.stop()
# Wait for them to finish
for h in hook_runners.values():
h.wait()
raise ret
return return_dict
class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block name -> block
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
self._subscriptions = OrderedDict() # block name -> list of subs
self._last_changes = OrderedDict() # block name -> list of changes
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
BlockNotify: self._handle_block_notify,
BlockChanged: self._handle_block_changed,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception:
self.log_exception("Exception while handling %s", request)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s in self._other_spawned:
s.wait(timeout=timeout)
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
self._other_spawned.append(
self.sync_factory.spawn(block.handle_request, request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
Lock: New lock
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
spawned = self.sync_factory.spawn(function, *args, **kwargs)
self._other_spawned.append(spawned)
return spawned
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
a = Attribute(StringArrayMeta(
description="Blocks hosted by this Process"))
self.process_block.add_attribute("blocks", a)
a = Attribute(StringArrayMeta(
description="Blocks reachable via ClientComms"))
self.process_block.add_attribute("remoteBlocks", a)
self.add_block(self.name, self.process_block)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block.remoteBlocks.set_value(remotes)
def notify_subscribers(self, block_name):
self.q.put(BlockNotify(name=block_name))
def _handle_block_notify(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
for delta in self._last_changes.setdefault(request.name, []):
self._block_state_cache.delta_update(delta)
for subscription in self._subscriptions.setdefault(request.name, []):
endpoint = subscription.endpoint
# find stuff that's changed that is relevant to this subscriber
changes = []
for change in self._last_changes[request.name]:
change_path = change[0]
# look for a change_path where the beginning matches the
# endpoint path, then strip away the matching part and add
# to the change set
i = 0
for (cp_element, ep_element) in zip(change_path, endpoint):
if cp_element != ep_element:
break
i += 1
else:
# change has matching path, so keep it
# but strip off the end point path
filtered_change = [change_path[i:]] + change[1:]
changes.append(filtered_change)
if len(changes) > 0:
if subscription.delta:
# respond with the filtered changes
response = Delta(
subscription.id_, subscription.context, changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(endpoint)
response = Update(
subscription.id_, subscription.context, d)
self.log_debug("Responding to subscription %s", response)
subscription.response_queue.put(response)
self._last_changes[request.name] = []
def on_changed(self, change, notify=True):
self.q.put(BlockChanged(change=change))
if notify:
block_name = change[0][0]
self.notify_subscribers(block_name)
def _handle_block_changed(self, request):
"""Record changes to made to a block"""
# update changes
path = request.change[0]
block_changes = self._last_changes.setdefault(path[0], [])
block_changes.append(request.change)
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, name, block):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
"""
assert name not in self._blocks, \
"There is already a block called %s" % name
block.set_parent(self, name)
self.q.put(BlockAdd(block=block))
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
block = request.block
assert block.name not in self._blocks, \
"There is already a block called %s" % block.name
self._blocks[block.name] = block
self._block_state_cache[block.name] = block.to_dict()
block.lock = self.create_lock()
# Regenerate list of blocks
self.process_block.blocks.set_value(list(self._blocks))
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
subs = self._subscriptions.setdefault(request.endpoint[0], [])
subs.append(request)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
response = Return(request.id_, request.context, d)
request.response_queue.put(response)
class TestClientController(unittest.TestCase):
def setUp(self):
# Serialized version of the block we want
source = Block()
HelloController(MagicMock(), source, "blockname")
self.serialized = source.to_dict()
# Setup client controller prerequisites
self.b = Block()
self.b.name = "blockname"
self.p = MagicMock()
self.comms = MagicMock()
self.cc = ClientController(self.p, self.b, "blockname")
# get process to give us comms
self.p.get_client_comms.return_value = self.comms
# tell our controller which blocks the process can talk to
response = MagicMock(id_=self.cc.REMOTE_BLOCKS_ID, value=["blockname"])
self.cc.put(response)
# tell our controller the serialized state of the block
response = MagicMock(id_=self.cc.BLOCK_ID, changes=[[[], self.serialized]])
self.cc.put(response)
def test_init(self):
self.assertEqual(self.p.q.put.call_count, 1)
req = self.p.q.put.call_args[0][0]
self.assertEqual(req.typeid, "malcolm:core/Subscribe:1.0")
self.assertEqual(req.endpoint, [self.p.name, "remoteBlocks", "value"])
self.assertEqual(req.response_queue, self.cc)
self.p.get_client_comms.assert_called_with("blockname")
self.assertEqual(self.comms.q.put.call_count, 1)
req = self.comms.q.put.call_args[0][0]
self.assertEqual(req.typeid, "malcolm:core/Subscribe:1.0")
self.assertEqual(req.delta, True)
self.assertEqual(req.response_queue, self.cc)
self.assertEqual(req.endpoint, ["blockname"])
def test_methods_created(self):
self.assertEqual(list(self.b.methods), ["disable", "reset", "say_hello"])
m = self.b.methods["say_hello"]
self.assertEqual(m.name, "say_hello")
self.assertEqual(list(m.takes.elements), ["name"])
self.assertEqual(type(m.takes.elements["name"]), StringMeta)
self.assertEqual(list(m.returns.elements), ["greeting"])
self.assertEqual(type(m.returns.elements["greeting"]), StringMeta)
self.assertEqual(m.defaults, {})
def test_call_method(self):
self.p.create_queue.return_value = queue.Queue()
def f(request):
request.respond_with_return(dict(
greeting="Hello %s" % request.parameters.name))
self.comms.q.put.side_effect = f
ret = self.b.say_hello(name="me")
self.assertEqual(ret.greeting, "Hello me")
def test_put_update_response(self):
response = MagicMock(
id_=self.cc.BLOCK_ID,
changes=[[["substructure"], "change"]])
self.b.update = MagicMock()
self.cc.put(response)
self.b.update.assert_called_once_with([["substructure"], "change"])
def test_put_root_update_response(self):
attr1 = StringMeta("dummy")
attr2 = StringMeta("dummy2")
new_block_structure = {}
new_block_structure["attr1"] = attr1.to_dict()
new_block_structure["attr2"] = attr2.to_dict()
self.b.replace_children = MagicMock()
response = MagicMock(
id_=self.cc.BLOCK_ID,
changes=[[[], new_block_structure]])
self.cc.put(response)
self.assertIs(self.b, self.cc.block)
deserialized_changes = self.b.replace_children.call_args_list[0][0][0]
serialized_changes = [x.to_dict() for x in
deserialized_changes.values()]
expected = [attr1.to_dict(), attr2.to_dict()]
# dicts are not hashable, so cannot use set compare
for x in expected:
self.assertTrue(x in serialized_changes)
for x in serialized_changes:
self.assertTrue(x in expected)
def setUp(self):
self.block = Block("TestBlock")
self.method = MagicMock()
self.method.name = "get_things"
self.response = MagicMock()
self.block.add_method(self.method)
def test_hello_controller_good_input(self):
block = Block("hello")
HelloController(block)
result = block.say_hello(name="me")
self.assertEquals(result["greeting"], "Hello me")
def test_replace_children(self):
b = Block()
b.name = "blockname"
b.methods["m1"] = 2
b.attributes["a1"] = 3
setattr(b, "m1", 2)
setattr(b, "a1", 3)
attr_meta = StringMeta(description="desc")
attr = Attribute(attr_meta)
b.add_attribute('attr', attr)
method = Method(description="desc")
b.add_method('method', method)
b.on_changed = MagicMock(wrap=b.on_changed)
b.replace_children({'attr':attr, 'method':method})
self.assertEqual(b.attributes, dict(attr=attr))
self.assertEqual(b.methods, dict(method=method))
b.on_changed.assert_called_once_with(
[[], b.to_dict()], True)
self.assertFalse(hasattr(b, "m1"))
self.assertFalse(hasattr(b, "a1"))
def test_notify(self):
b = Block()
b.set_parent(MagicMock(), "n")
b.notify_subscribers()
b.parent.notify_subscribers.assert_called_once_with("n")
class Process(Loggable):
"""Hosts a number of Blocks, distributing requests between them"""
def __init__(self, name, sync_factory):
self.set_logger_name(name)
self.name = name
self.sync_factory = sync_factory
self.q = self.create_queue()
self._blocks = OrderedDict() # block_name -> Block
self._controllers = OrderedDict() # block_name -> Controller
self._block_state_cache = Cache()
self._recv_spawned = None
self._other_spawned = []
# lookup of all Subscribe requests, ordered to guarantee subscription
# notification ordering
# {Request.generate_key(): Subscribe}
self._subscriptions = OrderedDict()
self.comms = []
self._client_comms = OrderedDict() # client comms -> list of blocks
self._handle_functions = {
Post: self._forward_block_request,
Put: self._forward_block_request,
Get: self._handle_get,
Subscribe: self._handle_subscribe,
Unsubscribe: self._handle_unsubscribe,
BlockChanges: self._handle_block_changes,
BlockRespond: self._handle_block_respond,
BlockAdd: self._handle_block_add,
BlockList: self._handle_block_list,
AddSpawned: self._add_spawned,
}
self.create_process_block()
def recv_loop(self):
"""Service self.q, distributing the requests to the right block"""
while True:
request = self.q.get()
self.log_debug("Received request %s", request)
if request is PROCESS_STOP:
# Got the sentinel, stop immediately
break
try:
self._handle_functions[type(request)](request)
except Exception as e: # pylint:disable=broad-except
self.log_exception("Exception while handling %s", request)
try:
request.respond_with_error(str(e))
except Exception:
pass
def add_comms(self, comms):
assert not self._recv_spawned, \
"Can't add comms when process has been started"
self.comms.append(comms)
def start(self):
"""Start the process going"""
self._recv_spawned = self.sync_factory.spawn(self.recv_loop)
for comms in self.comms:
comms.start()
def stop(self, timeout=None):
"""Stop the process and wait for it to finish
Args:
timeout (float): Maximum amount of time to wait for each spawned
process. None means forever
"""
assert self._recv_spawned, "Process not started"
self.q.put(PROCESS_STOP)
for comms in self.comms:
comms.stop()
# Wait for recv_loop to complete first
self._recv_spawned.wait(timeout=timeout)
# Now wait for anything it spawned to complete
for s, _ in self._other_spawned:
s.wait(timeout=timeout)
# Garbage collect the syncfactory
del self.sync_factory
def _forward_block_request(self, request):
"""Lookup target Block and spawn block.handle_request(request)
Args:
request (Request): The message that should be passed to the Block
"""
block_name = request.endpoint[0]
block = self._blocks[block_name]
spawned = self.sync_factory.spawn(block.handle_request, request)
self._add_spawned(AddSpawned(spawned, block.handle_request))
def create_queue(self):
"""
Create a queue using sync_factory object
Returns:
Queue: New queue
"""
return self.sync_factory.create_queue()
def create_lock(self):
"""
Create a lock using sync_factory object
Returns:
New lock object
"""
return self.sync_factory.create_lock()
def spawn(self, function, *args, **kwargs):
"""Calls SyncFactory.spawn()"""
def catching_function():
try:
function(*args, **kwargs)
except Exception:
self.log_exception(
"Exception calling %s(*%s, **%s)", function, args, kwargs)
raise
spawned = self.sync_factory.spawn(catching_function)
request = AddSpawned(spawned, function)
self.q.put(request)
return spawned
def _add_spawned(self, request):
spawned = self._other_spawned
self._other_spawned = []
spawned.append((request.spawned, request.function))
# Filter out the spawned that have completed to stop memory leaks
for sp, f in spawned:
if not sp.ready():
self._other_spawned.append((sp, f))
def get_client_comms(self, block_name):
for client_comms, blocks in list(self._client_comms.items()):
if block_name in blocks:
return client_comms
def create_process_block(self):
self.process_block = Block()
# TODO: add a meta here
children = OrderedDict()
children["blocks"] = StringArrayMeta(
description="Blocks hosted by this Process"
).make_attribute([])
children["remoteBlocks"] = StringArrayMeta(
description="Blocks reachable via ClientComms"
).make_attribute([])
self.process_block.replace_endpoints(children)
self.process_block.set_process_path(self, [self.name])
self.add_block(self.process_block, self)
def update_block_list(self, client_comms, blocks):
self.q.put(BlockList(client_comms=client_comms, blocks=blocks))
def _handle_block_list(self, request):
self._client_comms[request.client_comms] = request.blocks
remotes = []
for blocks in self._client_comms.values():
remotes += [b for b in blocks if b not in remotes]
self.process_block["remoteBlocks"].set_value(remotes)
def _handle_block_changes(self, request):
"""Update subscribers with changes and applies stored changes to the
cached structure"""
# update cached dict
subscription_changes = self._block_state_cache.apply_changes(
*request.changes)
# Send out the changes
for subscription, changes in subscription_changes.items():
if subscription.delta:
# respond with the filtered changes
subscription.respond_with_delta(changes)
else:
# respond with the structure of everything
# below the endpoint
d = self._block_state_cache.walk_path(subscription.endpoint)
subscription.respond_with_update(d)
def report_changes(self, *changes):
self.q.put(BlockChanges(changes=list(changes)))
def block_respond(self, response, response_queue):
self.q.put(BlockRespond(response, response_queue))
def _handle_block_respond(self, request):
"""Push the response to the required queue"""
request.response_queue.put(request.response)
def add_block(self, block, controller):
"""Add a block to be hosted by this process
Args:
block (Block): The block to be added
controller (Controller): Its controller
"""
path = block.process_path
assert len(path) == 1, \
"Expected block %r to have %r as parent, got path %r" % \
(block, self, path)
name = path[0]
assert name not in self._blocks, \
"There is already a block called %r" % name
request = BlockAdd(block=block, controller=controller, name=name)
if self._recv_spawned:
# Started, so call in Process thread
self.q.put(request)
else:
# Not started yet so we are safe to add in this thread
self._handle_block_add(request)
def _handle_block_add(self, request):
"""Add a block to be hosted by this process"""
assert request.name not in self._blocks, \
"There is already a block called %r" % request.name
self._blocks[request.name] = request.block
self._controllers[request.name] = request.controller
serialized = request.block.to_dict()
change_request = BlockChanges([[[request.name], serialized]])
self._handle_block_changes(change_request)
# Regenerate list of blocks
self.process_block["blocks"].set_value(list(self._blocks))
def get_block(self, block_name):
try:
return self._blocks[block_name]
except KeyError:
if block_name in self.process_block.remoteBlocks:
return self.make_client_block(block_name)
else:
raise
def make_client_block(self, block_name):
params = ClientController.MethodMeta.prepare_input_map(
mri=block_name)
controller = ClientController(self, {}, params)
return controller.block
def get_controller(self, block_name):
return self._controllers[block_name]
def _handle_subscribe(self, request):
"""Add a new subscriber and respond with the current
sub-structure state"""
key = request.generate_key()
assert key not in self._subscriptions, \
"Subscription on %s already exists" % (key,)
self._subscriptions[key] = request
self._block_state_cache.add_subscriber(request, request.endpoint)
d = self._block_state_cache.walk_path(request.endpoint)
self.log_debug("Initial subscription value %s", d)
if request.delta:
request.respond_with_delta([[[], d]])
else:
request.respond_with_update(d)
def _handle_unsubscribe(self, request):
"""Remove a subscriber and respond with success or error"""
key = request.generate_key()
try:
subscription = self._subscriptions.pop(key)
except KeyError:
request.respond_with_error(
"No subscription found for %s" % (key,))
else:
self._block_state_cache.remove_subscriber(
subscription, subscription.endpoint)
request.respond_with_return()
def _handle_get(self, request):
d = self._block_state_cache.walk_path(request.endpoint)
request.respond_with_return(d)