def __receive_settings(self) -> bool:
"""Receives settings on muscle_settings_in.
Returns:
False iff the port is connnected and ClosePort was received.
"""
default_message = Message(0.0, None, Settings(), Settings())
message = self._communicator.receive_message('muscle_settings_in',
None, default_message)
if isinstance(message.data, ClosePort):
return False
if not isinstance(message.data, Settings):
err_msg = ('"{}" received a message on'
' muscle_settings_in that is not a'
' Settings. It seems that your'
' simulation is miswired or the sending'
' instance is broken.'.format(self._instance_name()))
self.__shutdown(err_msg)
raise RuntimeError(err_msg)
settings = cast(Settings, message.settings)
for key, value in message.data.items():
settings[key] = value
self._settings_manager.overlay = settings
return True
def receive_message(port_name, slot=None, default=None):
if port_name == 'muscle_settings_in':
return Message(0.0, None, Settings(), Settings())
elif port_name == 'in':
data = 'test {}'.format(slot)
return Message(0.0, None, data, Settings())
assert False # pragma: no cover
def from_bytes(message: bytes) -> 'Message':
"""Create an MCP Message from an encoded buffer.
Args:
message: MessagePack encoded message data.
"""
message_dict = msgpack.unpackb(message, raw=False)
sender = Reference(message_dict["sender"])
receiver = Reference(message_dict["receiver"])
port_length = message_dict["port_length"]
timestamp = message_dict["timestamp"]
next_timestamp = message_dict["next_timestamp"]
settings_dict = msgpack.unpackb(message_dict["settings_overlay"].data,
raw=False)
settings_overlay = Settings(settings_dict)
data = message_dict["data"]
if isinstance(data, msgpack.ExtType):
if data.code == ExtTypeId.CLOSE_PORT:
data = ClosePort()
elif data.code == ExtTypeId.SETTINGS:
plain_dict = msgpack.unpackb(data.data, raw=False)
data = Settings(plain_dict)
return Message(sender, receiver, port_length, timestamp,
next_timestamp, settings_overlay, data)
def test_reuse_instance_no_f_init_ports(instance):
instance._communicator.receive_message.return_value = Message(
0.0, None, Settings(), Settings())
instance._communicator.list_ports.return_value = {}
instance._communicator.settings_in_connected.return_value = False
do_reuse = instance.reuse_instance()
assert do_reuse is True
do_reuse = instance.reuse_instance()
assert do_reuse is False
def test_update(settings: Settings) -> None:
settings1 = Settings()
settings1['param1'] = 13
settings1['param2'] = 'testing'
settings.update(settings1)
assert len(settings) == 2
assert settings['param1'] == 13
assert settings['param2'] == 'testing'
settings2 = Settings()
settings2[Reference('param2')] = [[1.0, 2.0], [2.0, 3.0]]
settings2['param3'] = 3.1415
settings.update(settings2)
assert len(settings) == 3
assert settings['param1'] == 13
assert settings['param2'] == [[1, 2], [2, 3]]
assert settings['param3'] == 3.1415
settings3 = Settings()
settings3[Reference('param1')] = True
settings.update(settings3)
assert len(settings) == 3
assert settings['param1'] is True
assert settings['param2'] == [[1, 2], [2, 3]]
assert settings['param3'] == 3.1415
def test_send_settings(communicator, message) -> None:
message.data = Settings()
message.data['test1'] = 'testing'
communicator.send_message('out', message)
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.settings_overlay == Settings()
assert msg.data == Settings({'test1': 'testing'})
def test_as_ordered_dict(settings: Settings) -> None:
settings._store = OrderedDict([(Reference('test1'), 12),
(Reference('test2'), '12'),
(Reference('test3'), 'testing'),
(Reference('test4'), [12.3, 45.6])])
settings_dict = settings.as_ordered_dict()
assert settings_dict['test1'] == 12
assert settings_dict['test2'] == '12'
assert settings_dict['test3'] == 'testing'
assert settings_dict['test4'] == [12.3, 45.6]
for i, (key, _) in enumerate(settings_dict.items()):
assert key == 'test{}'.format(i + 1)
def test_create_settings2() -> None:
setting_values = OrderedDict([('submodel._muscle_grain', [0.01, 0.01]),
('submodel._muscle_extent', [10.0, 3.0]),
('submodel._muscle_timestep', 0.001),
('submodel._muscle_total_time', 0.1),
('bf.velocity', 0.48),
('init.max_depth', 0.11)
]) # type: OrderedDict[str, SettingValue]
settings = Settings(setting_values)
assert list(
settings.ordered_items()[0][0]) == ['submodel', '_muscle_grain']
assert cast(List[float], settings.ordered_items()[1][1])[1] == 3.0
assert settings['submodel._muscle_timestep'] == 0.001
assert list(settings.ordered_items()[4][0]) == ['bf', 'velocity']
assert settings['init.max_depth'] == 0.11
def test_reuse_instance_receive_overlay(instance):
instance._settings_manager.overlay = Settings()
test_base_settings = Settings()
test_base_settings['test1'] = 24
test_base_settings['test2'] = [1.3, 2.0]
test_overlay = Settings()
test_overlay['test2'] = 'abc'
recv = instance._communicator.receive_message
recv.return_value = Message(0.0, None, test_overlay, test_base_settings)
instance.reuse_instance()
assert instance._communicator.receive_message.called_with(
'muscle_settings_in')
assert len(instance._settings_manager.overlay) == 2
assert instance._settings_manager.overlay['test1'] == 24
assert instance._settings_manager.overlay['test2'] == 'abc'
def get_settings(self) -> Settings:
"""Get the central settings from the manager.
Returns:
The requested settings.
"""
LLF = mmp.SETTING_VALUE_TYPE_LIST_LIST_FLOAT
result = self.__client.RequestSettings(mmp.SettingsRequest())
settings = Settings()
for setting in result.setting_values:
if setting.value_type == mmp.SETTING_VALUE_TYPE_STRING:
settings[setting.name] = setting.value_string
elif setting.value_type == mmp.SETTING_VALUE_TYPE_INT:
settings[setting.name] = setting.value_int
elif setting.value_type == mmp.SETTING_VALUE_TYPE_FLOAT:
settings[setting.name] = setting.value_float
elif setting.value_type == mmp.SETTING_VALUE_TYPE_BOOL:
settings[setting.name] = setting.value_bool
elif setting.value_type == mmp.SETTING_VALUE_TYPE_LIST_FLOAT:
settings[setting.name] = list(
setting.value_list_float.values)
elif setting.value_type == LLF:
rows = list() # type: List[List[float]]
for mmp_row in setting.value_list_list_float.values:
rows.append(list(mmp_row.values))
settings[setting.name] = rows
return settings
def test_grid_roundtrip() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float64)
assert (array[0, 0, 0] == 1.0)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
msg_out = Message.from_bytes(wire_data)
assert isinstance(msg_out.data, Grid)
grid_out = msg_out.data
assert grid_out.indexes == ['x', 'y', 'z']
assert isinstance(grid_out.array, np.ndarray)
assert grid_out.array.dtype == np.float64
assert grid_out.array.shape == (2, 2, 3)
assert grid_out.array.size == 12
assert grid_out.array[1, 0, 1] == 8.0
assert grid_out.array[0, 0, 2] == 3.0
def test_copy() -> None:
settings1 = Settings()
settings1['test'] = 12
settings1['test2'] = [23.0, 12.0]
settings1['test3'] = 'test3'
settings2 = settings1.copy()
assert settings1 == settings2
settings2['test'] = 13
assert settings2['test'] == 13
assert settings1['test'] == 12
cast(List[float], settings2['test2'])[0] = 24.0
assert settings2['test2'] == [24.0, 12.0]
assert settings1['test2'] == [24.0, 12.0]
def test_grid_encode() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float32)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
mcp_decoded = msgpack.unpackb(wire_data, raw=False)
grid_decoded = msgpack.unpackb(mcp_decoded['data'].data, raw=False)
assert grid_decoded['type'] == 'float32'
assert grid_decoded['shape'] == [2, 2, 3]
assert grid_decoded['order'] == 'la'
next_value = 1.0
for value in struct.iter_unpack('<f', grid_decoded['data']):
assert value[0] == next_value
next_value = next_value + 1.0
assert grid_decoded['indexes'] == ['x', 'y', 'z']
def __init__(self) -> None:
"""Create a SettingsManager.
Initialises the base and overlay layers to an empty
Settings object.
A SettingsManager has two layers of settings, a base
layer that contains an immutable collection of settings set in
the simulation's yMMSL description, and an overlay layer that
holds settings that have been set at run-time.
Attributes:
base: The base layer.
overlay: The overlay layer.
"""
self.base = Settings()
self.overlay = Settings()
def test_receive_message_default(communicator) -> None:
communicator._peer_manager.is_connected.return_value = False
default_msg = Message(3.0, 4.0, 'test', Settings())
msg = communicator.receive_message('not_connected', default=default_msg)
assert msg.timestamp == 3.0
assert msg.next_timestamp == 4.0
assert msg.data == 'test'
assert len(msg.settings) == 0
def test_iter(settings: Settings) -> None:
assert len(settings) == 0
for setting, value in settings.items():
assert False # pragma: no cover
settings._store = OrderedDict([(Reference('test1'), 13),
(Reference('test2'), 'testing'),
(Reference('test3'), [3.4, 5.6])])
assert len(settings) == 3
for setting in settings:
assert setting in settings
for setting, value in settings.items():
assert (setting == 'test1' and value == 13
or setting == 'test2' and value == 'testing'
or setting == 'test3' and value == [3.4, 5.6])
def test_del_item(settings: Settings) -> None:
settings._store = OrderedDict([(Reference('param1'), 'test'),
(Reference('param2'), 0)])
del settings['param1']
assert len(settings._store) == 1
assert Reference('param1') not in settings._store
with pytest.raises(KeyError):
settings['param1'] # pylint: disable=pointless-statement
assert settings._store[Reference('param2')] == 0
def test_receive_settings(communicator) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(Reference('other.out[13]'),
Reference('kernel[13].in'),
None, 0.0, None,
Settings({'test1': 12}),
Settings({'test':
13})).encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator._Communicator__get_client = get_client_mock
communicator._profiler = MagicMock()
msg = communicator.receive_message('in')
get_client_mock.assert_called_with(Reference('other'))
client_mock.receive.assert_called_with(Reference('kernel[13].in'))
assert isinstance(msg.data, Settings)
assert msg.data['test'] == 13
def test_send_message_with_slot(communicator2, message) -> None:
communicator2.send_message('out', message, 13)
assert 'kernel[13].in' in \
communicator2._post_office._outboxes
msg_bytes = communicator2._post_office._outboxes[
'kernel[13].in']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'other.out[13]'
assert msg.receiver == 'kernel[13].in'
assert msg.settings_overlay == Settings()
assert msg.data == b'test'
def close_port(self, port_name: str, slot: Optional[int] = None) -> None:
"""Closes the given port.
This signals to any connected instance that no more messages
will be sent on this port, which it can use to decide whether
to shut down or continue running.
Args:
port_name: The name of the port to close.
"""
message = Message(float('inf'), None, ClosePort(), Settings())
self.send_message(port_name, message, slot)
def test_close_port(communicator) -> None:
communicator.close_port('out')
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.timestamp == float('inf')
assert msg.next_timestamp is None
assert msg.settings_overlay == Settings()
assert isinstance(msg.data, ClosePort)
def test_receive_close_port(communicator) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(Reference('other.out[13]'),
Reference('kernel[13].in'),
None, 0.0, None, Settings(),
ClosePort()).encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator._Communicator__get_client = get_client_mock
communicator._profiler = MagicMock()
msg = communicator.receive_message('in')
assert isinstance(msg.data, ClosePort)
def test_send_message(communicator, message) -> None:
communicator.send_message('out', message)
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.timestamp == 0.0
assert msg.next_timestamp is None
assert msg.settings_overlay == Settings()
assert msg.data == b'test'
def test_receive_message_resizable(communicator3) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(
Reference('other.out[13]'), Reference('kernel.in[13]'), 20, 0.0, None,
Settings({'test': 'testing'}), b'test').encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator3._Communicator__get_client = get_client_mock
communicator3._profiler = MagicMock()
msg = communicator3.receive_message('in', 13)
get_client_mock.assert_called_with(Reference('other'))
client_mock.receive.assert_called_with(Reference('kernel.in[13]'))
assert msg.data == b'test'
assert communicator3.get_port('in').get_length() == 20
def test_receive_msgpack_with_slot_and_settings(communicator2) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(
Reference('kernel[13].out'), Reference('other.in[13]'), None, 0.0, 1.0,
Settings({'test': 'testing'}), 'test').encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator2._Communicator__get_client = get_client_mock
communicator2._profiler = MagicMock()
msg = communicator2.receive_message('in', 13)
get_client_mock.assert_called_with(Reference('kernel[13]'))
client_mock.receive.assert_called_with(Reference('other.in[13]'))
assert msg.data == 'test'
assert msg.settings['test'] == 'testing'
def qmc():
"""qMC implementation.
"""
instance = Instance({Operator.O_F: ['settings_out[]']})
while instance.reuse_instance():
# o_f
settings0 = Settings({'test2': 14.4})
assert instance.is_connected('settings_out')
assert instance.is_vector_port('settings_out')
assert not instance.is_resizable('settings_out')
length = instance.get_port_length('settings_out')
assert length == 10
for slot in range(length):
instance.send('settings_out',
Message(0.0, None, settings0), slot)
def test_receive_with_settings(communicator) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(Reference('other.out[13]'),
Reference('kernel[13].in'),
None, 0.0, None,
Settings({'test2': 3.1}),
b'test').encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator._Communicator__get_client = get_client_mock
communicator._profiler = MagicMock()
msg = communicator.receive_message('in')
get_client_mock.assert_called_with(Reference('other'))
client_mock.receive.assert_called_with(Reference('kernel[13].in'))
assert msg.data == b'test'
assert msg.settings['test2'] == 3.1
def test_get_setting(settings_manager):
ref = Reference
settings_manager.base[ref('test')] = 13
assert settings_manager.get_setting(ref('instance'), ref('test')) == 13
settings_manager.overlay[ref('test2')] = 14
assert settings_manager.get_setting(ref('instance'), ref('test2')) == 14
settings_manager.base[ref('test2')] = 'test'
assert settings_manager.get_setting(ref('instance'), ref('test2')) == 14
settings_manager.overlay = Settings()
assert settings_manager.get_setting(ref('instance'), ref('test2')) == \
'test'
settings_manager.base[ref('test3')] = 'base_test3'
settings_manager.base[ref('instance.test3')] = 'base_instance_test3'
assert settings_manager.get_setting(ref('instance'), ref('test3')) == \
'base_instance_test3'
assert settings_manager.get_setting(ref('instance2'), ref('test3')) == \
'base_test3'
settings_manager.overlay[ref('test3')] = 'overlay_test3'
settings_manager.overlay[ref('instance.test3')] = 'overlay_instance_test3'
assert settings_manager.get_setting(ref('instance'), ref('test3')) == \
'overlay_instance_test3'
assert settings_manager.get_setting(ref('instance2'), ref('test3')) == \
'overlay_test3'
settings_manager.base[ref('instance.test4')] = 'base_test4'
settings_manager.overlay[ref('test4')] = 'overlay_test4'
assert settings_manager.get_setting(ref('instance'), ref('test4')) == \
'base_test4'
assert settings_manager.get_setting(ref('instance[10]'),
ref('test4')) == 'base_test4'
settings_manager.base[ref('instance[10].test5')] = 'base_test5'
settings_manager.overlay[ref('test5')] = 'overlay_test5'
assert settings_manager.get_setting(ref('instance'), ref('test5')) == \
'overlay_test5'
assert settings_manager.get_setting(ref('instance[10]'),
ref('test5')) == 'base_test5'
assert settings_manager.get_setting(ref('instance[11]'),
ref('test5')) == 'overlay_test5'
def test_cpp_tcp_server(log_file_in_tmpdir):
# create C++ server
# it serves a message for us to receive
# see libmuscle/cpp/src/libmuscle/tests/mmp_server_test.cpp
cpp_build_dir = Path(__file__).parents[1] / 'libmuscle' / 'cpp' / 'build'
lib_paths = [
cpp_build_dir / 'grpc' / 'c-ares' / 'c-ares' / 'lib',
cpp_build_dir / 'grpc' / 'zlib' / 'zlib' / 'lib',
cpp_build_dir / 'grpc' / 'openssl' / 'openssl' / 'lib',
cpp_build_dir / 'protobuf' / 'protobuf' / 'lib',
cpp_build_dir / 'grpc' / 'grpc' / 'lib',
cpp_build_dir / 'msgpack' / 'msgpack' / 'lib'
]
env = {'LD_LIBRARY_PATH': ':'.join(map(str, lib_paths))}
cpp_test_dir = cpp_build_dir / 'libmuscle' / 'tests'
cpp_test_server = cpp_test_dir / 'tcp_server_test'
server = subprocess.Popen([str(cpp_test_server)],
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
universal_newlines=True,
bufsize=1,
close_fds=True)
# get server location
location = server.stdout.readline()
assert TcpClient.can_connect_to(location)
client = TcpClient(Reference('test_receiver'), location)
msg = Message.from_bytes(client.receive(Reference('test_receiver.port')))
client.close()
# assert stuff
assert msg.sender == 'test_sender.port'
assert msg.receiver == 'test_receiver.port'
assert msg.timestamp == 0.0
assert msg.next_timestamp == 1.0
assert msg.settings_overlay == Settings({'par1': 13})
assert msg.data == {'var1': 1, 'var2': 2.0, 'var3': '3'}
server.stdout.close()
server.wait()
assert server.returncode == 0
def instance(sys_argv_instance):
with patch('libmuscle.instance.MMPClient') as mmp_client, \
patch('libmuscle.instance.Communicator') as comm_type:
communicator = MagicMock()
settings = Settings()
settings['test1'] = 12
msg = Message(0.0, 1.0, 'message', settings)
communicator.receive_message.return_value = msg
comm_type.return_value = communicator
mmp_client_object = MagicMock()
mmp_client_object.request_peers.return_value = (None, None, None)
mmp_client.return_value = mmp_client_object
instance = Instance({
Operator.F_INIT: ['in', 'not_connected'],
Operator.O_F: ['out']})
instance._f_init_cache = dict()
instance._f_init_cache[('in', None)] = msg
yield instance
