def test_grid_roundtrip() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float64)
assert (array[0, 0, 0] == 1.0)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
msg_out = Message.from_bytes(wire_data)
assert isinstance(msg_out.data, Grid)
grid_out = msg_out.data
assert grid_out.indexes == ['x', 'y', 'z']
assert isinstance(grid_out.array, np.ndarray)
assert grid_out.array.dtype == np.float64
assert grid_out.array.shape == (2, 2, 3)
assert grid_out.array.size == 12
assert grid_out.array[1, 0, 1] == 8.0
assert grid_out.array[0, 0, 2] == 3.0
def test_grid_encode() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
array = np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]],
[[7.0, 8.0, 9.0], [10.0, 11.0, 12.0]]], np.float32)
grid = Grid(array, ['x', 'y', 'z'])
msg = Message(sender, receiver, None, timestamp, next_timestamp,
Settings(), grid)
wire_data = msg.encoded()
mcp_decoded = msgpack.unpackb(wire_data, raw=False)
grid_decoded = msgpack.unpackb(mcp_decoded['data'].data, raw=False)
assert grid_decoded['type'] == 'float32'
assert grid_decoded['shape'] == [2, 2, 3]
assert grid_decoded['order'] == 'la'
next_value = 1.0
for value in struct.iter_unpack('<f', grid_decoded['data']):
assert value[0] == next_value
next_value = next_value + 1.0
assert grid_decoded['indexes'] == ['x', 'y', 'z']
def test_grid_decode() -> None:
settings_data = msgpack.packb({}, use_bin_type=True)
grid_buf = bytes([
1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 5, 0, 0, 0, 6, 0
])
grid_dict = {
'type': 'int32',
'shape': [2, 3],
'order': 'la',
'data': grid_buf,
'indexes': []
}
grid_data = msgpack.packb(grid_dict, use_bin_type=True)
msg_dict = {
'sender': 'elem1.port1',
'receiver': 'elem2.port2',
'port_length': 0,
'timestamp': 0.0,
'next_timestamp': None,
'settings_overlay': msgpack.ExtType(1, settings_data),
'data': msgpack.ExtType(2, grid_data)
}
wire_data = msgpack.packb(msg_dict, use_bin_type=True)
msg = Message.from_bytes(wire_data)
assert isinstance(msg.data, Grid)
assert msg.data.array.dtype == np.int32
assert msg.data.array.shape == (2, 3)
assert msg.data.array.flags.c_contiguous
assert msg.data.array[0, 0] == 1
assert msg.data.array[0, 1] == 2
assert msg.data.array[1, 1] == 5 * 65536
assert msg.data.indexes is None
grid_dict['order'] = 'fa'
grid_data = msgpack.packb(grid_dict, use_bin_type=True)
msg_dict['data'] = msgpack.ExtType(2, grid_data)
wire_data = msgpack.packb(msg_dict, use_bin_type=True)
msg = Message.from_bytes(wire_data)
assert isinstance(msg.data, Grid)
assert msg.data.array.dtype == np.int32
assert msg.data.array.shape == (2, 3)
assert msg.data.array.flags.f_contiguous
assert msg.data.array[0, 0] == 1
assert msg.data.array[0, 1] == 3
assert msg.data.array[0, 2] == 5 * 65536
assert msg.data.indexes is None
def send_message(self,
port_name: str,
message: Message,
slot: Optional[int] = None) -> None:
"""Send a message and settings to the outside world.
Sending is non-blocking, a copy of the message will be made
and stored until the receiver is ready to receive it.
Args:
port_name: The port on which this message is to be sent.
message: The message to be sent.
slot: The slot to send the message on, if any.
"""
if slot is None:
slot_list = [] # type: List[int]
else:
slot_list = [slot]
slot_length = self._ports[port_name].get_length()
if slot_length <= slot:
raise RuntimeError(('Slot out of bounds. You are sending on'
' slot {} of port "{}", which is of length'
' {}, so that slot does not exist').format(
slot, port_name, slot_length))
snd_endpoint = self.__get_endpoint(port_name, slot_list)
if not self._peer_manager.is_connected(snd_endpoint.port):
# log sending on disconnected port
return
port = self._ports[port_name]
profile_event = self._profiler.start(ProfileEventType.SEND, port, None,
slot, None)
recv_endpoint = self._peer_manager.get_peer_endpoint(
snd_endpoint.port, slot_list)
port_length = None
if self._ports[port_name].is_resizable():
port_length = self._ports[port_name].get_length()
mcp_message = MCPMessage(snd_endpoint.ref(), recv_endpoint.ref(),
port_length, message.timestamp,
message.next_timestamp,
cast(Settings,
message.settings), message.data)
encoded_message = mcp_message.encoded()
self._post_office.deposit(recv_endpoint.ref(), encoded_message)
profile_event.stop()
if port.is_vector():
profile_event.port_length = port.get_length()
profile_event.message_size = len(encoded_message)
def test_send_receive(receiver, post_office):
message = Message(Reference('test_sender.test_port'), receiver, None, 0.0,
1.0, bytes(), 'message'.encode('utf-8'))
# prepare post office, it's about to get forked
post_office.outboxes[receiver].deposit(message)
# create server in separate process
sender_instance_id = Reference('test_sender')
mux.add_instance(sender_instance_id)
server_proc = mp.Process(target=run_server,
args=(sender_instance_id, receiver, post_office),
name='PipeServer')
server_proc.start()
mux.close_instance_ends(sender_instance_id)
# create client in separate process
server_location = mux.get_address_for(sender_instance_id)
recv_instance_id = Reference('test_receiver')
mux.add_instance(recv_instance_id)
client_proc = mp.Process(target=run_client,
args=(recv_instance_id, server_location, receiver,
message),
name='PipeClient')
client_proc.start()
mux.close_instance_ends(recv_instance_id)
# service connection requests
mux.run()
# shut down
client_proc.join()
server_proc.join()
def test_send_msgpack(communicator, message2) -> None:
communicator.send_message('out', message2)
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.settings_overlay == Settings()
assert msg.data == {'test': 17}
def test_send_message_with_settings(communicator, message) -> None:
message.settings['test2'] = 'testing'
communicator.send_message('out', message)
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.settings_overlay.as_ordered_dict() == {'test2': 'testing'}
assert msg.data == b'test'
def test_send_message_with_slot(communicator2, message) -> None:
communicator2.send_message('out', message, 13)
assert 'kernel[13].in' in \
communicator2._post_office._outboxes
msg_bytes = communicator2._post_office._outboxes[
'kernel[13].in']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'other.out[13]'
assert msg.receiver == 'kernel[13].in'
assert msg.settings_overlay == Settings()
assert msg.data == b'test'
def test_close_port(communicator) -> None:
communicator.close_port('out')
assert 'other.in[13]' in communicator._post_office._outboxes
msg_bytes = communicator._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel[13].out'
assert msg.receiver == 'other.in[13]'
assert msg.timestamp == float('inf')
assert msg.next_timestamp is None
assert msg.settings_overlay == Settings()
assert isinstance(msg.data, ClosePort)
def test_send_message_resizable(communicator3, message) -> None:
with pytest.raises(RuntimeError):
communicator3.send_message('out', message, 13)
communicator3.get_port('out').set_length(20)
communicator3.send_message('out', message, 13)
assert 'other.in[13]' in communicator3._post_office._outboxes
msg_bytes = communicator3._post_office._outboxes[
'other.in[13]']._Outbox__queue.get()
msg = MCPMessage.from_bytes(msg_bytes)
assert msg.sender == 'kernel.out[13]'
assert msg.receiver == 'other.in[13]'
assert msg.port_length == 20
def test_create() -> None:
sender = Reference('sender.port')
receiver = Reference('receiver.port')
timestamp = 10.0
next_timestamp = 11.0
settings_overlay = (6789).to_bytes(2, 'little', signed=True)
data = (12345).to_bytes(2, 'little', signed=True)
msg = Message(sender, receiver, None, timestamp, next_timestamp,
settings_overlay, data)
assert msg.sender == sender
assert msg.receiver == receiver
assert msg.port_length is None
assert msg.timestamp == 10.0
assert msg.next_timestamp == 11.0
assert msg.settings_overlay == settings_overlay
assert msg.data == data
def test_cpp_tcp_server(log_file_in_tmpdir):
# create C++ server
# it serves a message for us to receive
# see libmuscle/cpp/src/libmuscle/tests/mmp_server_test.cpp
cpp_build_dir = Path(__file__).parents[1] / 'libmuscle' / 'cpp' / 'build'
lib_paths = [
cpp_build_dir / 'grpc' / 'c-ares' / 'c-ares' / 'lib',
cpp_build_dir / 'grpc' / 'zlib' / 'zlib' / 'lib',
cpp_build_dir / 'grpc' / 'openssl' / 'openssl' / 'lib',
cpp_build_dir / 'protobuf' / 'protobuf' / 'lib',
cpp_build_dir / 'grpc' / 'grpc' / 'lib',
cpp_build_dir / 'msgpack' / 'msgpack' / 'lib'
]
env = {'LD_LIBRARY_PATH': ':'.join(map(str, lib_paths))}
cpp_test_dir = cpp_build_dir / 'libmuscle' / 'tests'
cpp_test_server = cpp_test_dir / 'tcp_server_test'
server = subprocess.Popen([str(cpp_test_server)],
env=env,
stdin=subprocess.DEVNULL,
stdout=subprocess.PIPE,
universal_newlines=True,
bufsize=1,
close_fds=True)
# get server location
location = server.stdout.readline()
assert TcpClient.can_connect_to(location)
client = TcpClient(Reference('test_receiver'), location)
msg = Message.from_bytes(client.receive(Reference('test_receiver.port')))
client.close()
# assert stuff
assert msg.sender == 'test_sender.port'
assert msg.receiver == 'test_receiver.port'
assert msg.timestamp == 0.0
assert msg.next_timestamp == 1.0
assert msg.settings_overlay == Settings({'par1': 13})
assert msg.data == {'var1': 1, 'var2': 2.0, 'var3': '3'}
server.stdout.close()
server.wait()
assert server.returncode == 0
def tcp_server_process(control_pipe):
control_pipe[0].close()
settings = Settings({'test_setting': 42})
data = {'test1': 10, 'test2': [None, True, 'testing']}
receiver = Reference('test_receiver.test_port2')
message = Message(Reference('test_sender.test_port'), receiver, 10, 1.0,
2.0, settings, data).encoded()
def get_message(receiver):
assert receiver == 'test_receiver.test_port2'
return message
post_office = MagicMock()
post_office.done = False
post_office.get_message = get_message
sender_instance_id = Reference('test_sender')
server = TcpServer(sender_instance_id, post_office)
control_pipe[1].send(server.get_location())
control_pipe[1].recv()
control_pipe[1].close()
server.close()
def test_request(receiver, post_office, direct_server):
message = Message(Reference('test_sender.test_port'), receiver, None, 0.0,
1.0, bytes(), bytes())
post_office.outboxes[receiver].deposit(message)
assert direct_server.request(receiver) == message
def message():
Ref = Reference
return Message(Ref('sender.out'), Ref('receiver.in'), None, 0.0, 1.0,
bytes(), 'testing'.encode('utf-8'))
def receive_message(self,
port_name: str,
slot: Optional[int] = None,
default: Optional[Message] = None) -> Message:
"""Receive a message and attached settings overlay.
Receiving is a blocking operation. This function will contact
the sender, wait for a message to be available, and receive and
return it.
If the port is not connected, then the default value will be
returned if one was given, exactly as it was given. If no
default was given then a RuntimeError will be raised.
Args:
port_name: The endpoint on which a message is to be
received.
slot: The slot to receive the message on, if any.
default: A message to return if this port is not connected.
Returns:
The received message, with message.settings holding
the settings overlay. The settings attribute is
guaranteed to not be None.
Raises:
RuntimeError: If no default was given and the port is not
connected.
"""
if slot is None:
slot_list = [] # type: List[int]
else:
slot_list = [slot]
recv_endpoint = self.__get_endpoint(port_name, slot_list)
if not self._peer_manager.is_connected(recv_endpoint.port):
if default is None:
raise RuntimeError(('Tried to receive on port "{}", which is'
' disconnected, and no default value was'
' given. Either specify a default, or'
' connect a sending component to this'
' port.').format(port_name))
return default
if port_name in self._ports:
port = self._ports[port_name]
else:
# it's muscle_settings_in here, because we check for unknown
# user ports in Instance already, and we don't have any other
# built-in automatic ports.
port = self._muscle_settings_in
profile_event = self._profiler.start(ProfileEventType.RECEIVE, port,
None, slot, None)
snd_endpoint = self._peer_manager.get_peer_endpoint(
recv_endpoint.port, slot_list)
client = self.__get_client(snd_endpoint.instance())
mcp_message_bytes = client.receive(recv_endpoint.ref())
mcp_message = MCPMessage.from_bytes(mcp_message_bytes)
if mcp_message.port_length is not None:
if port.is_resizable():
port.set_length(mcp_message.port_length)
if isinstance(mcp_message.data, ClosePort):
port.set_closed(slot)
message = Message(mcp_message.timestamp, mcp_message.next_timestamp,
mcp_message.data, mcp_message.settings_overlay)
profile_event.stop()
if port.is_vector():
profile_event.port_length = port.get_length()
profile_event.message_size = len(mcp_message_bytes)
return message