def from_bytes(message: bytes) -> 'Message': """Create an MCP Message from an encoded buffer. Args: message: MessagePack encoded message data. """ message_dict = msgpack.unpackb(message, raw=False) sender = Reference(message_dict["sender"]) receiver = Reference(message_dict["receiver"]) port_length = message_dict["port_length"] timestamp = message_dict["timestamp"] next_timestamp = message_dict["next_timestamp"] settings_dict = msgpack.unpackb(message_dict["settings_overlay"].data, raw=False) settings_overlay = Settings(settings_dict) data = message_dict["data"] if isinstance(data, msgpack.ExtType): if data.code == ExtTypeId.CLOSE_PORT: data = ClosePort() elif data.code == ExtTypeId.SETTINGS: plain_dict = msgpack.unpackb(data.data, raw=False) data = Settings(plain_dict) return Message(sender, receiver, port_length, timestamp, next_timestamp, settings_overlay, data)
def test_send_receive(receiver, post_office): message = Message(Reference('test_sender.test_port'), receiver, None, 0.0, 1.0, bytes(), 'message'.encode('utf-8')) # prepare post office, it's about to get forked post_office.outboxes[receiver].deposit(message) # create server in separate process sender_instance_id = Reference('test_sender') mux.add_instance(sender_instance_id) server_proc = mp.Process(target=run_server, args=(sender_instance_id, receiver, post_office), name='PipeServer') server_proc.start() mux.close_instance_ends(sender_instance_id) # create client in separate process server_location = mux.get_address_for(sender_instance_id) recv_instance_id = Reference('test_receiver') mux.add_instance(recv_instance_id) client_proc = mp.Process(target=run_client, args=(recv_instance_id, server_location, receiver, message), name='PipeClient') client_proc.start() mux.close_instance_ends(recv_instance_id) # service connection requests mux.run() # shut down client_proc.join() server_proc.join()
def test_dump_implementations(dump_configuration: Callable) -> None: implementations = [ Implementation( Reference('macro'), '#!/bin/bash\n\n/usr/bin/python3 /home/test/macro.py\n'), Implementation( Reference('meso'), '#!/bin/bash\n\n/home/test/meso.py'), Implementation(Reference('micro'), '/home/test/micro')] configuration = PartialConfiguration(None, None, implementations) text = dump_configuration(configuration) assert text == ( 'ymmsl_version: v0.1\n' 'implementations:\n' ' macro:\n' ' - \'#!/bin/bash\'\n' ' - \'\'\n' ' - /usr/bin/python3 /home/test/macro.py\n' ' - \'\'\n' ' meso:\n' ' - \'#!/bin/bash\'\n' ' - \'\'\n' ' - /home/test/meso.py\n' ' micro: /home/test/micro\n' )
def test_create_instance( sys_argv_instance, log_file_in_tmpdir, sys_argv_manager): with patch('libmuscle.instance.MMPClient') as mmp_client, \ patch('libmuscle.instance.Communicator') as comm_type: mmp_client_object = MagicMock() mmp_client_object.request_peers.return_value = (None, None, None) mmp_client.return_value = mmp_client_object ports = { Operator.F_INIT: ['in'], Operator.O_F: ['out']} instance = Instance(ports) assert instance._name == Reference('test_instance') assert instance._index == [13, 42] assert instance._declared_ports == ports assert isinstance(instance._settings_manager, SettingsManager) assert len(instance._settings_manager.base) == 0 assert len(instance._settings_manager.overlay) == 0 mmp_client.assert_called_once_with('localhost:9000') assert mmp_client_object._register.called_with() assert mmp_client_object._connect.called_with() comm_type.assert_called_with(Reference('test_instance'), [13, 42], ports, instance._profiler) assert instance._communicator == comm_type.return_value assert isinstance(instance._settings_manager, SettingsManager) assert len(instance._settings_manager.base) == 0
def test_grid_roundtrip() -> None: sender = Reference('sender.port') receiver = Reference('receiver.port') timestamp = 10.0 next_timestamp = 11.0 array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float64) assert (array[0, 0, 0] == 1.0) grid = Grid(array, ['x', 'y', 'z']) msg = Message(sender, receiver, None, timestamp, next_timestamp, Settings(), grid) wire_data = msg.encoded() msg_out = Message.from_bytes(wire_data) assert isinstance(msg_out.data, Grid) grid_out = msg_out.data assert grid_out.indexes == ['x', 'y', 'z'] assert isinstance(grid_out.array, np.ndarray) assert grid_out.array.dtype == np.float64 assert grid_out.array.shape == (2, 2, 3) assert grid_out.array.size == 12 assert grid_out.array[1, 0, 1] == 8.0 assert grid_out.array[0, 0, 2] == 3.0
def test_grid_encode() -> None: sender = Reference('sender.port') receiver = Reference('receiver.port') timestamp = 10.0 next_timestamp = 11.0 array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], [[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float32) grid = Grid(array, ['x', 'y', 'z']) msg = Message(sender, receiver, None, timestamp, next_timestamp, Settings(), grid) wire_data = msg.encoded() mcp_decoded = msgpack.unpackb(wire_data, raw=False) grid_decoded = msgpack.unpackb(mcp_decoded['data'].data, raw=False) assert grid_decoded['type'] == 'float32' assert grid_decoded['shape'] == [2, 2, 3] assert grid_decoded['order'] == 'la' next_value = 1.0 for value in struct.iter_unpack('<f', grid_decoded['data']): assert value[0] == next_value next_value = next_value + 1.0 assert grid_decoded['indexes'] == ['x', 'y', 'z']
def test_update(settings: Settings) -> None: settings1 = Settings() settings1['param1'] = 13 settings1['param2'] = 'testing' settings.update(settings1) assert len(settings) == 2 assert settings['param1'] == 13 assert settings['param2'] == 'testing' settings2 = Settings() settings2[Reference('param2')] = [[1.0, 2.0], [2.0, 3.0]] settings2['param3'] = 3.1415 settings.update(settings2) assert len(settings) == 3 assert settings['param1'] == 13 assert settings['param2'] == [[1, 2], [2, 3]] assert settings['param3'] == 3.1415 settings3 = Settings() settings3[Reference('param1')] = True settings.update(settings3) assert len(settings) == 3 assert settings['param1'] is True assert settings['param2'] == [[1, 2], [2, 3]] assert settings['param3'] == 3.1415
def implementation_process(instance_id: str, implementation: Callable) -> None: prefix_tag = '--muscle-prefix=' name_prefix = str() index_prefix = list() # type: List[int] instance = Reference(instance_id) for i, arg in enumerate(sys.argv): if arg.startswith(prefix_tag): prefix_str = arg[len(prefix_tag):] name_prefix, index_prefix = _parse_prefix(prefix_str) name, index = _split_reference(instance) if len(name_prefix) > 0: name = Reference(name_prefix) + name index = index_prefix + index # replace it with the combined one sys.argv[i] = '--muscle-instance={}'.format(str(name + index)) break else: sys.argv.append('--muscle-instance={}'.format(instance_id)) # chain call implementation()
def test_config5() -> Configuration: model = Model( 'test_model', [ Component('ic', 'isr2d.initial_conditions'), Component('smc', 'isr2d.smc'), Component('bf', 'isr2d.blood_flow'), Component('smc2bf', 'isr2d.smc2bf'), Component('bf2smc', 'isr2d.bf2smc')], [ Conduit('ic.out', 'smc.initial_state'), Conduit('smc.cell_positions', 'smc2bf.in'), Conduit('smc2bf.out', 'bf.initial_domain'), Conduit('bf.wss_out', 'bf2smc.in'), Conduit('bf2smc.out', 'smc.wss_in')]) implementations = [ Implementation( Reference('isr2d.initial_conditions'), 'isr2d/bin/ic'), Implementation( Reference('isr2d.smc'), 'isr2d/bin/smc'), Implementation( Reference('isr2d.blood_flow'), 'isr2d/bin/bf'), Implementation( Reference('isr2d.smc2bf'), 'isr2d/bin/smc2bf.py'), Implementation( Reference('isr2d.bf2smc'), 'isr2d/bin/bf2smc.py')] resources = [ Resources(Reference('isr2d.initial_conditions'), 4), Resources(Reference('isr2d.smc'), 4), Resources(Reference('isr2d.blood_flow'), 4), Resources(Reference('isr2d.smc2bf'), 1), Resources(Reference('isr2d.bf2smc'), 1)] return Configuration(model, None, implementations, resources)
def loaded_instance_registry(instance_registry): instance_registry.add(Reference('macro'), ['direct:macro'], []) for j in range(10): for i in range(10): name = Reference('micro') + j + i location = 'direct:{}'.format(name) instance_registry.add(name, [location], []) return instance_registry
def gpe(p, s) -> Reference: endpoint = MagicMock() endpoint.instance.return_value = Reference('other') if 'out' in str(p): endpoint.ref.return_value = Reference('other.in[13]') else: endpoint.ref.return_value = Reference('other.out') return endpoint
def test_del_item(settings: Settings) -> None: settings._store = OrderedDict([(Reference('param1'), 'test'), (Reference('param2'), 0)]) del settings['param1'] assert len(settings._store) == 1 assert Reference('param1') not in settings._store with pytest.raises(KeyError): settings['param1'] # pylint: disable=pointless-statement assert settings._store[Reference('param2')] == 0
def test_endpoint_instance() -> None: endpoint = Endpoint(Reference('test.kernel'), [42], Identifier('port'), [2]) assert endpoint.instance() == 'test.kernel[42]' endpoint2 = Endpoint(Reference('test.kernel'), [], Identifier('port'), []) assert endpoint2.instance() == 'test.kernel' endpoint3 = Endpoint(Reference('test.kernel'), [], Identifier('port'), [3]) assert endpoint3.instance() == 'test.kernel'
def test_reference_concatenation() -> None: assert Reference('test') + Reference('test2') == 'test.test2' assert Reference('test') + Identifier('test2') == 'test.test2' assert Reference('test') + 5 == 'test[5]' assert Reference('test') + [Identifier('test2'), 5] == 'test.test2[5]' assert Reference('test[5]') + Reference('test2[3]') == 'test[5].test2[3]' assert Reference('test[5]') + Identifier('test2') == 'test[5].test2' assert Reference('test[5]') + 3 == 'test[5][3]' assert (Reference('test[5]') + [3, Identifier('test2')] == 'test[5][3].test2')
def test_configuration_update_resources_add() -> None: resources1 = Resources(Reference('my.macro'), 10) base = PartialConfiguration(resources=[resources1]) resources2 = Resources(Reference('my.micro'), 2) overlay = PartialConfiguration(resources=[resources2]) base.update(overlay) assert len(base.resources) == 2 assert base.resources[Reference('my.macro')] == resources1 assert base.resources[Reference('my.micro')] == resources2
def test_get_client(mock_servers, communicator) -> None: mock_servers.__contains__.return_value = True client = communicator._Communicator__get_client(Reference('other')) mock_servers.__contains__.assert_called_with('test') assert isinstance(client, DirectClient) client2 = communicator._Communicator__get_client(Reference('other')) assert client == client2 gpl = communicator._peer_manager.get_peer_locations gpl.return_value = ['non_existent:test'] with pytest.raises(RuntimeError): communicator._Communicator__get_client(Reference('other2'))
def test_as_ordered_dict(settings: Settings) -> None: settings._store = OrderedDict([(Reference('test1'), 12), (Reference('test2'), '12'), (Reference('test3'), 'testing'), (Reference('test4'), [12.3, 45.6])]) settings_dict = settings.as_ordered_dict() assert settings_dict['test1'] == 12 assert settings_dict['test2'] == '12' assert settings_dict['test3'] == 'testing' assert settings_dict['test4'] == [12.3, 45.6] for i, (key, _) in enumerate(settings_dict.items()): assert key == 'test{}'.format(i + 1)
def test_registration(log_file_in_tmpdir, mmp_server): client = MMPClient('localhost:9000') instance_name = Reference('test_instance') port = Port(Reference('test_in'), Operator.S) client.register_instance(instance_name, ['tcp://localhost:10000'], [port]) servicer = mmp_server._MMPServer__servicer registry = servicer._MMPServicer__instance_registry assert registry.get_locations(instance_name) == ['tcp://localhost:10000'] assert registry.get_ports(instance_name)[0].name == 'test_in' assert registry.get_ports(instance_name)[0].operator == Operator.S
def test_receive_close_port(communicator) -> None: client_mock = MagicMock() client_mock.receive.return_value = MCPMessage(Reference('other.out[13]'), Reference('kernel[13].in'), None, 0.0, None, Settings(), ClosePort()).encoded() get_client_mock = MagicMock(return_value=client_mock) communicator._Communicator__get_client = get_client_mock communicator._profiler = MagicMock() msg = communicator.receive_message('in') assert isinstance(msg.data, ClosePort)
def test_load_string4(test_yaml4: str) -> None: configuration = load(test_yaml4) assert isinstance(configuration, PartialConfiguration) assert not isinstance(configuration, Configuration) assert len(configuration.implementations) == 5 impls = configuration.implementations ic = Reference('isr2d.initial_conditions') bf2smc = Reference('isr2d.bf2smc') assert impls[ic].name == 'isr2d.initial_conditions' assert impls[bf2smc].script == 'isr2d/bin/bf2smc.py' assert len(configuration.resources) == 5 assert configuration.resources[ic].num_cores == 4 assert configuration.resources[bf2smc].num_cores == 1
def test_configuration_update_implementations_add() -> None: implementation1 = Implementation( Reference('my.macro'), '/home/test/macro.py') base = PartialConfiguration(implementations=[implementation1]) implementation2 = Implementation( Reference('my.micro'), '/home/test/micro.py') overlay = PartialConfiguration(implementations=[implementation2]) base.update(overlay) assert len(base.implementations) == 2 assert base.implementations[Reference('my.macro')] == implementation1 assert base.implementations[Reference('my.micro')] == implementation2
def test_dump_resources(dump_configuration: Callable) -> None: resources = [ Resources(Reference('macro'), 10), Resources(Reference('micro'), 1)] configuration = PartialConfiguration(None, None, None, resources) text = dump_configuration(configuration) assert text == ( 'ymmsl_version: v0.1\n' 'resources:\n' ' macro: 10\n' ' micro: 1\n' )
def test_receive_message_resizable(communicator3) -> None: client_mock = MagicMock() client_mock.receive.return_value = MCPMessage( Reference('other.out[13]'), Reference('kernel.in[13]'), 20, 0.0, None, Settings({'test': 'testing'}), b'test').encoded() get_client_mock = MagicMock(return_value=client_mock) communicator3._Communicator__get_client = get_client_mock communicator3._profiler = MagicMock() msg = communicator3.receive_message('in', 13) get_client_mock.assert_called_with(Reference('other')) client_mock.receive.assert_called_with(Reference('kernel.in[13]')) assert msg.data == b'test' assert communicator3.get_port('in').get_length() == 20
def test_receive_msgpack_with_slot_and_settings(communicator2) -> None: client_mock = MagicMock() client_mock.receive.return_value = MCPMessage( Reference('kernel[13].out'), Reference('other.in[13]'), None, 0.0, 1.0, Settings({'test': 'testing'}), 'test').encoded() get_client_mock = MagicMock(return_value=client_mock) communicator2._Communicator__get_client = get_client_mock communicator2._profiler = MagicMock() msg = communicator2.receive_message('in', 13) get_client_mock.assert_called_with(Reference('kernel[13]')) client_mock.receive.assert_called_with(Reference('other.in[13]')) assert msg.data == 'test' assert msg.settings['test'] == 'testing'
def test_reference_equivalence() -> None: assert Reference('test.test[3]') == Reference('test.test[3]') assert Reference('test.test[3]') != Reference('test1.test[3]') assert Reference('test.test[3]') == 'test.test[3]' assert Reference('test.test[3]') != 'test1.test[3]' assert 'test.test[3]' == Reference('test.test[3]') # pylint: disable=C0122 assert 'test1.test[3]' != Reference('test.test[3]') # pylint: disable=C0122
def __make_full_name(self) -> Tuple[Reference, List[int]]: """Returns instance name and index. This takes the argument to the --muscle-instance= command-line option and splits it into a compute element name and an index. """ def split_reference(ref: Reference) -> Tuple[Reference, List[int]]: index = list() # type: List[int] i = 0 while i < len(ref) and isinstance(ref[i], Identifier): i += 1 name = cast(Reference, ref[:i]) while i < len(ref) and isinstance(ref[i], int): index.append(cast(int, ref[i])) i += 1 return name, index # Neither getopt, optparse, or argparse will let me pick out # just one option from the command line and ignore the rest. # So we do it by hand. prefix_tag = '--muscle-instance=' for arg in sys.argv[1:]: if arg.startswith(prefix_tag): prefix_str = arg[len(prefix_tag):] prefix_ref = Reference(prefix_str) name, index = split_reference(prefix_ref) break else: raise RuntimeError(('A --muscle-instance command line argument is' ' required to identify this instance. Please' ' add one.')) return name, index
def test_request_peers(mocked_mmp_client) -> None: conduits = [mmp.Conduit(sender='kernel.out', receiver='other.in')] peer_dimensions = [mmp.PeerResult.PeerDimensions( peer_name='other', dimensions=[20])] peer_locations = [mmp.PeerResult.PeerLocations( instance_name='other', locations=['direct:test', 'tcp:test'])] mocked_mmp_client[1].RequestPeers.return_value = mmp.PeerResult( status=mmp.RESULT_STATUS_SUCCESS, conduits=conduits, peer_dimensions=peer_dimensions, peer_locations=peer_locations) result = mocked_mmp_client[0].request_peers(Reference('kernel[13]')) assert mocked_mmp_client[1].RequestPeers.called assert len(result[0]) == 1 assert isinstance(result[0][0], Conduit) assert result[0][0].sender == 'kernel.out' assert result[0][0].receiver == 'other.in' assert isinstance(result[1], dict) assert result[1]['other'] == [20] assert isinstance(result[2], dict) assert result[2]['other'] == ['direct:test', 'tcp:test']
def test_create() -> None: sender = Reference('sender.port') receiver = Reference('receiver.port') timestamp = 10.0 next_timestamp = 11.0 settings_overlay = (6789).to_bytes(2, 'little', signed=True) data = (12345).to_bytes(2, 'little', signed=True) msg = Message(sender, receiver, None, timestamp, next_timestamp, settings_overlay, data) assert msg.sender == sender assert msg.receiver == receiver assert msg.port_length is None assert msg.timestamp == 10.0 assert msg.next_timestamp == 11.0 assert msg.settings_overlay == settings_overlay assert msg.data == data
def communicator2() -> Communicator: with patch('libmuscle.communicator.server_types', [DirectServer]): instance_id = Reference('other') communicator = Communicator(instance_id, [], None, MagicMock()) communicator._peer_manager = MagicMock() pm = communicator._peer_manager pm.is_connected.return_value = True def gpp(x: Reference) -> Reference: if 'out' in str(x): return Reference('in') return Reference('out') pm.get_peer_port = gpp pm.get_peer_dims.return_value = [] pm.get_peer_locations.return_value = ['direct:test'] def gpe(p, s) -> Reference: endpoint = MagicMock() endpoint.instance.return_value = Reference('kernel[13]') if 'out' in str(p): endpoint.ref.return_value = Reference('kernel[13].in') else: endpoint.ref.return_value = Reference('kernel[13].out') return endpoint pm.get_peer_endpoint = gpe communicator._ports = { 'out': Port('out', Operator.O_I, True, True, 0, [20]), 'in': Port('in', Operator.S, True, True, 0, [20]) } yield communicator communicator.shutdown()
def test_connect() -> None: ref = Reference instance_id = Reference('kernel') conduits = [ Conduit('kernel.out', 'other.in'), Conduit('other.out', 'kernel.in') ] peer_dims = {ref('other'): [1]} peer_locations = {ref('other'): ['direct:test']} with patch('libmuscle.communicator.PeerManager') as pm_init: communicator = Communicator(instance_id, [13], None, MagicMock()) communicator.connect(conduits, peer_dims, peer_locations) pm_init.assert_called_with(instance_id, [13], conduits, peer_dims, peer_locations) # check inferred ports ports = communicator._ports communicator.shutdown() assert ports['in'].name == Identifier('in') assert ports['in'].operator == Operator.F_INIT assert ports['in']._length is None assert ports['out'].name == Identifier('out') assert ports['out'].operator == Operator.O_F assert ports['out']._length is None