def from_bytes(message: bytes) -> 'Message':
"""Create an MCP Message from an encoded buffer.
Args:
message: MessagePack encoded message data.
"""
message_dict = msgpack.unpackb(message, raw=False)
sender = Reference(message_dict["sender"])
receiver = Reference(message_dict["receiver"])
port_length = message_dict["port_length"]
timestamp = message_dict["timestamp"]
next_timestamp = message_dict["next_timestamp"]
settings_dict = msgpack.unpackb(message_dict["settings_overlay"].data,
raw=False)
settings_overlay = Settings(settings_dict)
data = message_dict["data"]
if isinstance(data, msgpack.ExtType):
if data.code == ExtTypeId.CLOSE_PORT:
data = ClosePort()
elif data.code == ExtTypeId.SETTINGS:
plain_dict = msgpack.unpackb(data.data, raw=False)
data = Settings(plain_dict)
return Message(sender, receiver, port_length, timestamp,
next_timestamp, settings_overlay, data)
def test_send_receive(receiver, post_office):
message = Message(Reference('test_sender.test_port'), receiver, None, 0.0,
1.0, bytes(), 'message'.encode('utf-8'))
# prepare post office, it's about to get forked
post_office.outboxes[receiver].deposit(message)
# create server in separate process
sender_instance_id = Reference('test_sender')
mux.add_instance(sender_instance_id)
server_proc = mp.Process(target=run_server,
args=(sender_instance_id, receiver, post_office),
name='PipeServer')
server_proc.start()
mux.close_instance_ends(sender_instance_id)
# create client in separate process
server_location = mux.get_address_for(sender_instance_id)
recv_instance_id = Reference('test_receiver')
mux.add_instance(recv_instance_id)
client_proc = mp.Process(target=run_client,
args=(recv_instance_id, server_location, receiver,
message),
name='PipeClient')
client_proc.start()
mux.close_instance_ends(recv_instance_id)
# service connection requests
mux.run()
# shut down
client_proc.join()
server_proc.join()
def test_dump_implementations(dump_configuration: Callable) -> None:
implementations = [
Implementation(
Reference('macro'),
'#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n'),
Implementation(
Reference('meso'),
'#!/bin/bash\n\n/home/test/meso.py'),
Implementation(Reference('micro'), '/home/test/micro')]
configuration = PartialConfiguration(None, None, implementations)
text = dump_configuration(configuration)
assert text == (
'ymmsl_version: v0.1\n'
'implementations:\n'
' macro:\n'
' - \'#!/bin/bash\'\n'
' - \'\'\n'
' - /usr/bin/python3 /home/test/macro.py\n'
' - \'\'\n'
' meso:\n'
' - \'#!/bin/bash\'\n'
' - \'\'\n'
' - /home/test/meso.py\n'
' micro: /home/test/micro\n'
)
def test_create_instance(
sys_argv_instance, log_file_in_tmpdir, sys_argv_manager):
with patch('libmuscle.instance.MMPClient') as mmp_client, \
patch('libmuscle.instance.Communicator') as comm_type:
mmp_client_object = MagicMock()
mmp_client_object.request_peers.return_value = (None, None, None)
mmp_client.return_value = mmp_client_object
ports = {
Operator.F_INIT: ['in'],
Operator.O_F: ['out']}
instance = Instance(ports)
assert instance._name == Reference('test_instance')
assert instance._index == [13, 42]
assert instance._declared_ports == ports
assert isinstance(instance._settings_manager, SettingsManager)
assert len(instance._settings_manager.base) == 0
assert len(instance._settings_manager.overlay) == 0
mmp_client.assert_called_once_with('localhost:9000')
assert mmp_client_object._register.called_with()
assert mmp_client_object._connect.called_with()
comm_type.assert_called_with(Reference('test_instance'), [13, 42],
ports, instance._profiler)
assert instance._communicator == comm_type.return_value
assert isinstance(instance._settings_manager, SettingsManager)
assert len(instance._settings_manager.base) == 0
def test_grid_roundtrip() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float64)
assert (array[0, 0, 0] == 1.0)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
msg_out = Message.from_bytes(wire_data)
assert isinstance(msg_out.data, Grid)
grid_out = msg_out.data
assert grid_out.indexes == ['x', 'y', 'z']
assert isinstance(grid_out.array, np.ndarray)
assert grid_out.array.dtype == np.float64
assert grid_out.array.shape == (2, 2, 3)
assert grid_out.array.size == 12
assert grid_out.array[1, 0, 1] == 8.0
assert grid_out.array[0, 0, 2] == 3.0
def test_grid_encode() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float32)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
mcp_decoded = msgpack.unpackb(wire_data, raw=False)
grid_decoded = msgpack.unpackb(mcp_decoded['data'].data, raw=False)
assert grid_decoded['type'] == 'float32'
assert grid_decoded['shape'] == [2, 2, 3]
assert grid_decoded['order'] == 'la'
next_value = 1.0
for value in struct.iter_unpack('<f', grid_decoded['data']):
assert value[0] == next_value
next_value = next_value + 1.0
assert grid_decoded['indexes'] == ['x', 'y', 'z']
def test_update(settings: Settings) -> None:
settings1 = Settings()
settings1['param1'] = 13
settings1['param2'] = 'testing'
settings.update(settings1)
assert len(settings) == 2
assert settings['param1'] == 13
assert settings['param2'] == 'testing'
settings2 = Settings()
settings2[Reference('param2')] = [[1.0, 2.0], [2.0, 3.0]]
settings2['param3'] = 3.1415
settings.update(settings2)
assert len(settings) == 3
assert settings['param1'] == 13
assert settings['param2'] == [[1, 2], [2, 3]]
assert settings['param3'] == 3.1415
settings3 = Settings()
settings3[Reference('param1')] = True
settings.update(settings3)
assert len(settings) == 3
assert settings['param1'] is True
assert settings['param2'] == [[1, 2], [2, 3]]
assert settings['param3'] == 3.1415
def implementation_process(instance_id: str, implementation: Callable) -> None:
prefix_tag = '--muscle-prefix='
name_prefix = str()
index_prefix = list() # type: List[int]
instance = Reference(instance_id)
for i, arg in enumerate(sys.argv):
if arg.startswith(prefix_tag):
prefix_str = arg[len(prefix_tag):]
name_prefix, index_prefix = _parse_prefix(prefix_str)
name, index = _split_reference(instance)
if len(name_prefix) > 0:
name = Reference(name_prefix) + name
index = index_prefix + index
# replace it with the combined one
sys.argv[i] = '--muscle-instance={}'.format(str(name + index))
break
else:
sys.argv.append('--muscle-instance={}'.format(instance_id))
# chain call
implementation()
def test_config5() -> Configuration:
model = Model(
'test_model',
[
Component('ic', 'isr2d.initial_conditions'),
Component('smc', 'isr2d.smc'),
Component('bf', 'isr2d.blood_flow'),
Component('smc2bf', 'isr2d.smc2bf'),
Component('bf2smc', 'isr2d.bf2smc')],
[
Conduit('ic.out', 'smc.initial_state'),
Conduit('smc.cell_positions', 'smc2bf.in'),
Conduit('smc2bf.out', 'bf.initial_domain'),
Conduit('bf.wss_out', 'bf2smc.in'),
Conduit('bf2smc.out', 'smc.wss_in')])
implementations = [
Implementation(
Reference('isr2d.initial_conditions'), 'isr2d/bin/ic'),
Implementation(
Reference('isr2d.smc'), 'isr2d/bin/smc'),
Implementation(
Reference('isr2d.blood_flow'), 'isr2d/bin/bf'),
Implementation(
Reference('isr2d.smc2bf'), 'isr2d/bin/smc2bf.py'),
Implementation(
Reference('isr2d.bf2smc'), 'isr2d/bin/bf2smc.py')]
resources = [
Resources(Reference('isr2d.initial_conditions'), 4),
Resources(Reference('isr2d.smc'), 4),
Resources(Reference('isr2d.blood_flow'), 4),
Resources(Reference('isr2d.smc2bf'), 1),
Resources(Reference('isr2d.bf2smc'), 1)]
return Configuration(model, None, implementations, resources)
def loaded_instance_registry(instance_registry):
instance_registry.add(Reference('macro'), ['direct:macro'], [])
for j in range(10):
for i in range(10):
name = Reference('micro') + j + i
location = 'direct:{}'.format(name)
instance_registry.add(name, [location], [])
return instance_registry
def gpe(p, s) -> Reference:
endpoint = MagicMock()
endpoint.instance.return_value = Reference('other')
if 'out' in str(p):
endpoint.ref.return_value = Reference('other.in[13]')
else:
endpoint.ref.return_value = Reference('other.out')
return endpoint
def test_del_item(settings: Settings) -> None:
settings._store = OrderedDict([(Reference('param1'), 'test'),
(Reference('param2'), 0)])
del settings['param1']
assert len(settings._store) == 1
assert Reference('param1') not in settings._store
with pytest.raises(KeyError):
settings['param1'] # pylint: disable=pointless-statement
assert settings._store[Reference('param2')] == 0
def test_endpoint_instance() -> None:
endpoint = Endpoint(Reference('test.kernel'), [42], Identifier('port'),
[2])
assert endpoint.instance() == 'test.kernel[42]'
endpoint2 = Endpoint(Reference('test.kernel'), [], Identifier('port'), [])
assert endpoint2.instance() == 'test.kernel'
endpoint3 = Endpoint(Reference('test.kernel'), [], Identifier('port'), [3])
assert endpoint3.instance() == 'test.kernel'
def test_reference_concatenation() -> None:
assert Reference('test') + Reference('test2') == 'test.test2'
assert Reference('test') + Identifier('test2') == 'test.test2'
assert Reference('test') + 5 == 'test[5]'
assert Reference('test') + [Identifier('test2'), 5] == 'test.test2[5]'
assert Reference('test[5]') + Reference('test2[3]') == 'test[5].test2[3]'
assert Reference('test[5]') + Identifier('test2') == 'test[5].test2'
assert Reference('test[5]') + 3 == 'test[5][3]'
assert (Reference('test[5]') +
[3, Identifier('test2')] == 'test[5][3].test2')
def test_configuration_update_resources_add() -> None:
resources1 = Resources(Reference('my.macro'), 10)
base = PartialConfiguration(resources=[resources1])
resources2 = Resources(Reference('my.micro'), 2)
overlay = PartialConfiguration(resources=[resources2])
base.update(overlay)
assert len(base.resources) == 2
assert base.resources[Reference('my.macro')] == resources1
assert base.resources[Reference('my.micro')] == resources2
def test_get_client(mock_servers, communicator) -> None:
mock_servers.__contains__.return_value = True
client = communicator._Communicator__get_client(Reference('other'))
mock_servers.__contains__.assert_called_with('test')
assert isinstance(client, DirectClient)
client2 = communicator._Communicator__get_client(Reference('other'))
assert client == client2
gpl = communicator._peer_manager.get_peer_locations
gpl.return_value = ['non_existent:test']
with pytest.raises(RuntimeError):
communicator._Communicator__get_client(Reference('other2'))
def test_as_ordered_dict(settings: Settings) -> None:
settings._store = OrderedDict([(Reference('test1'), 12),
(Reference('test2'), '12'),
(Reference('test3'), 'testing'),
(Reference('test4'), [12.3, 45.6])])
settings_dict = settings.as_ordered_dict()
assert settings_dict['test1'] == 12
assert settings_dict['test2'] == '12'
assert settings_dict['test3'] == 'testing'
assert settings_dict['test4'] == [12.3, 45.6]
for i, (key, _) in enumerate(settings_dict.items()):
assert key == 'test{}'.format(i + 1)
def test_registration(log_file_in_tmpdir, mmp_server):
client = MMPClient('localhost:9000')
instance_name = Reference('test_instance')
port = Port(Reference('test_in'), Operator.S)
client.register_instance(instance_name, ['tcp://localhost:10000'], [port])
servicer = mmp_server._MMPServer__servicer
registry = servicer._MMPServicer__instance_registry
assert registry.get_locations(instance_name) == ['tcp://localhost:10000']
assert registry.get_ports(instance_name)[0].name == 'test_in'
assert registry.get_ports(instance_name)[0].operator == Operator.S
def test_receive_close_port(communicator) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(Reference('other.out[13]'),
Reference('kernel[13].in'),
None, 0.0, None, Settings(),
ClosePort()).encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator._Communicator__get_client = get_client_mock
communicator._profiler = MagicMock()
msg = communicator.receive_message('in')
assert isinstance(msg.data, ClosePort)
def test_load_string4(test_yaml4: str) -> None:
configuration = load(test_yaml4)
assert isinstance(configuration, PartialConfiguration)
assert not isinstance(configuration, Configuration)
assert len(configuration.implementations) == 5
impls = configuration.implementations
ic = Reference('isr2d.initial_conditions')
bf2smc = Reference('isr2d.bf2smc')
assert impls[ic].name == 'isr2d.initial_conditions'
assert impls[bf2smc].script == 'isr2d/bin/bf2smc.py'
assert len(configuration.resources) == 5
assert configuration.resources[ic].num_cores == 4
assert configuration.resources[bf2smc].num_cores == 1
def test_configuration_update_implementations_add() -> None:
implementation1 = Implementation(
Reference('my.macro'), '/home/test/macro.py')
base = PartialConfiguration(implementations=[implementation1])
implementation2 = Implementation(
Reference('my.micro'), '/home/test/micro.py')
overlay = PartialConfiguration(implementations=[implementation2])
base.update(overlay)
assert len(base.implementations) == 2
assert base.implementations[Reference('my.macro')] == implementation1
assert base.implementations[Reference('my.micro')] == implementation2
def test_dump_resources(dump_configuration: Callable) -> None:
resources = [
Resources(Reference('macro'), 10),
Resources(Reference('micro'), 1)]
configuration = PartialConfiguration(None, None, None, resources)
text = dump_configuration(configuration)
assert text == (
'ymmsl_version: v0.1\n'
'resources:\n'
' macro: 10\n'
' micro: 1\n'
)
def test_receive_message_resizable(communicator3) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(
Reference('other.out[13]'), Reference('kernel.in[13]'), 20, 0.0, None,
Settings({'test': 'testing'}), b'test').encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator3._Communicator__get_client = get_client_mock
communicator3._profiler = MagicMock()
msg = communicator3.receive_message('in', 13)
get_client_mock.assert_called_with(Reference('other'))
client_mock.receive.assert_called_with(Reference('kernel.in[13]'))
assert msg.data == b'test'
assert communicator3.get_port('in').get_length() == 20
def test_receive_msgpack_with_slot_and_settings(communicator2) -> None:
client_mock = MagicMock()
client_mock.receive.return_value = MCPMessage(
Reference('kernel[13].out'), Reference('other.in[13]'), None, 0.0, 1.0,
Settings({'test': 'testing'}), 'test').encoded()
get_client_mock = MagicMock(return_value=client_mock)
communicator2._Communicator__get_client = get_client_mock
communicator2._profiler = MagicMock()
msg = communicator2.receive_message('in', 13)
get_client_mock.assert_called_with(Reference('kernel[13]'))
client_mock.receive.assert_called_with(Reference('other.in[13]'))
assert msg.data == 'test'
assert msg.settings['test'] == 'testing'
def test_reference_equivalence() -> None:
assert Reference('test.test[3]') == Reference('test.test[3]')
assert Reference('test.test[3]') != Reference('test1.test[3]')
assert Reference('test.test[3]') == 'test.test[3]'
assert Reference('test.test[3]') != 'test1.test[3]'
assert 'test.test[3]' == Reference('test.test[3]') # pylint: disable=C0122
assert 'test1.test[3]' != Reference('test.test[3]') # pylint: disable=C0122
def __make_full_name(self) -> Tuple[Reference, List[int]]:
"""Returns instance name and index.
This takes the argument to the --muscle-instance= command-line
option and splits it into a compute element name and an index.
"""
def split_reference(ref: Reference) -> Tuple[Reference, List[int]]:
index = list() # type: List[int]
i = 0
while i < len(ref) and isinstance(ref[i], Identifier):
i += 1
name = cast(Reference, ref[:i])
while i < len(ref) and isinstance(ref[i], int):
index.append(cast(int, ref[i]))
i += 1
return name, index
# Neither getopt, optparse, or argparse will let me pick out
# just one option from the command line and ignore the rest.
# So we do it by hand.
prefix_tag = '--muscle-instance='
for arg in sys.argv[1:]:
if arg.startswith(prefix_tag):
prefix_str = arg[len(prefix_tag):]
prefix_ref = Reference(prefix_str)
name, index = split_reference(prefix_ref)
break
else:
raise RuntimeError(('A --muscle-instance command line argument is'
' required to identify this instance. Please'
' add one.'))
return name, index
def test_request_peers(mocked_mmp_client) -> None:
conduits = [mmp.Conduit(sender='kernel.out', receiver='other.in')]
peer_dimensions = [mmp.PeerResult.PeerDimensions(
peer_name='other', dimensions=[20])]
peer_locations = [mmp.PeerResult.PeerLocations(
instance_name='other', locations=['direct:test', 'tcp:test'])]
mocked_mmp_client[1].RequestPeers.return_value = mmp.PeerResult(
status=mmp.RESULT_STATUS_SUCCESS,
conduits=conduits,
peer_dimensions=peer_dimensions,
peer_locations=peer_locations)
result = mocked_mmp_client[0].request_peers(Reference('kernel[13]'))
assert mocked_mmp_client[1].RequestPeers.called
assert len(result[0]) == 1
assert isinstance(result[0][0], Conduit)
assert result[0][0].sender == 'kernel.out'
assert result[0][0].receiver == 'other.in'
assert isinstance(result[1], dict)
assert result[1]['other'] == [20]
assert isinstance(result[2], dict)
assert result[2]['other'] == ['direct:test', 'tcp:test']
def test_create() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
settings_overlay = (6789).to_bytes(2, 'little', signed=True)
data = (12345).to_bytes(2, 'little', signed=True)
msg = Message(sender, receiver, None, timestamp, next_timestamp,
settings_overlay, data)
assert msg.sender == sender
assert msg.receiver == receiver
assert msg.port_length is None
assert msg.timestamp == 10.0
assert msg.next_timestamp == 11.0
assert msg.settings_overlay == settings_overlay
assert msg.data == data
def communicator2() -> Communicator:
with patch('libmuscle.communicator.server_types', [DirectServer]):
instance_id = Reference('other')
communicator = Communicator(instance_id, [], None, MagicMock())
communicator._peer_manager = MagicMock()
pm = communicator._peer_manager
pm.is_connected.return_value = True
def gpp(x: Reference) -> Reference:
if 'out' in str(x):
return Reference('in')
return Reference('out')
pm.get_peer_port = gpp
pm.get_peer_dims.return_value = []
pm.get_peer_locations.return_value = ['direct:test']
def gpe(p, s) -> Reference:
endpoint = MagicMock()
endpoint.instance.return_value = Reference('kernel[13]')
if 'out' in str(p):
endpoint.ref.return_value = Reference('kernel[13].in')
else:
endpoint.ref.return_value = Reference('kernel[13].out')
return endpoint
pm.get_peer_endpoint = gpe
communicator._ports = {
'out': Port('out', Operator.O_I, True, True, 0, [20]),
'in': Port('in', Operator.S, True, True, 0, [20])
}
yield communicator
communicator.shutdown()
def test_connect() -> None:
ref = Reference
instance_id = Reference('kernel')
conduits = [
Conduit('kernel.out', 'other.in'),
Conduit('other.out', 'kernel.in')
]
peer_dims = {ref('other'): [1]}
peer_locations = {ref('other'): ['direct:test']}
with patch('libmuscle.communicator.PeerManager') as pm_init:
communicator = Communicator(instance_id, [13], None, MagicMock())
communicator.connect(conduits, peer_dims, peer_locations)
pm_init.assert_called_with(instance_id, [13], conduits, peer_dims,
peer_locations)
# check inferred ports
ports = communicator._ports
communicator.shutdown()
assert ports['in'].name == Identifier('in')
assert ports['in'].operator == Operator.F_INIT
assert ports['in']._length is None
assert ports['out'].name == Identifier('out')
assert ports['out'].operator == Operator.O_F
assert ports['out']._length is None