def test_rpc_client_recovery_on_broken_pipe(self):
"""Ensure RPC retry on broken pipe error.
When the cluster closes the TCP socket, try reconnecting
and retrying the command before returing error for the operation.
"""
mock_socket = self.mock_object(socket, 'socket')
mock_make_call = self.mock_object(coho.Client, '_make_call')
socket_error = socket.error('[Errno 32] Broken pipe')
socket_error.errno = errno.EPIPE
mock_make_call.side_effect = socket_error
rpc_client = coho.CohoRPCClient(ADDR, RPC_PORT)
with self.assertRaisesRegex(exception.CohoException,
"Failed to establish.*"):
rpc_client.create_snapshot('src', 'dest', 0)
self.assertEqual(coho.COHO_MAX_RETRIES, mock_make_call.call_count)
self.assertEqual(coho.COHO_MAX_RETRIES + 1, mock_socket.call_count)
# assert that on a none EPIPE error it only tries once
socket_error.errno = errno.EINVAL
mock_make_call.side_effect = socket_error
with self.assertRaisesRegex(exception.CohoException,
"Unable to send request.*"):
rpc_client.delete_snapshot('src')
self.assertEqual(coho.COHO_MAX_RETRIES + 1, mock_make_call.call_count)
self.assertEqual(coho.COHO_MAX_RETRIES + 1, mock_socket.call_count)
def test_rpc_client_error_in_reply_header(self):
"""Ensure excpetions in reply header are raised by the RPC client.
Coho cluster's RPC server packs errors into the reply header.
This test ensures that the RPC client parses the reply header
correctly and raises exceptions on various errors that can be
included in the reply header.
"""
mock_socket = self.mock_object(socket, 'socket')
mock_recvrecord = self.mock_object(coho.Client, '_recvrecord')
rpc_client = coho.CohoRPCClient(ADDR, RPC_PORT)
mock_recvrecord.return_value = NO_REPLY_BIN
with self.assertRaisesRegex(exception.CohoException,
"no REPLY.*"):
rpc_client.create_snapshot('src', 'dest', 0)
mock_recvrecord.return_value = MSG_DENIED_BIN
with self.assertRaisesRegex(exception.CohoException,
".*MSG_DENIED.*"):
rpc_client.delete_snapshot('snapshot')
mock_recvrecord.return_value = PROG_UNAVAIL_BIN
with self.assertRaisesRegex(exception.CohoException,
".*PROG_UNAVAIL"):
rpc_client.delete_snapshot('snapshot')
mock_recvrecord.return_value = PROG_MISMATCH_BIN
with self.assertRaisesRegex(exception.CohoException,
".*PROG_MISMATCH.*"):
rpc_client.delete_snapshot('snapshot')
mock_recvrecord.return_value = GARBAGE_ARGS_BIN
with self.assertRaisesRegex(exception.CohoException,
".*GARBAGE_ARGS"):
rpc_client.delete_snapshot('snapshot')
mock_recvrecord.return_value = PROC_UNAVAIL_BIN
with self.assertRaisesRegex(exception.CohoException,
".*PROC_UNAVAIL"):
rpc_client.delete_snapshot('snapshot')
self.assertTrue(mock_recvrecord.called)
mock_socket.assert_has_calls(
[mock.call(socket.AF_INET, socket.SOCK_STREAM),
mock.call().bind(('', 0)),
mock.call().connect((ADDR, RPC_PORT))])
def test_rpc_client_error_in_receive_fragment(self):
"""Ensure exception is raised when malformed packet is received."""
mock_sendrcd = self.mock_object(coho.Client, '_sendrecord')
mock_socket = self.mock_object(socket, 'socket')
mock_socket.return_value.recv.return_value = INVALID_HEADER_BIN
rpc_client = coho.CohoRPCClient(ADDR, RPC_PORT)
with self.assertRaisesRegex(exception.CohoException,
"Invalid response header.*"):
rpc_client.create_snapshot('src', 'dest', 0)
self.assertTrue(mock_sendrcd.called)
mock_socket.assert_has_calls([
mock.call(socket.AF_INET, socket.SOCK_STREAM),
mock.call().connect((ADDR, RPC_PORT)),
mock.call().recv(4)
])
def test_rpc_client_make_call_proper_order(self):
"""This test ensures that the RPC client logic is correct.
When the RPC client's make_call function is called it creates
a packet and sends it to the Coho cluster RPC server. This test
ensures that the functions needed to complete the process are
called in the proper order with valid arguments.
"""
mock_packer = self.mock_object(xdrlib, 'Packer')
mock_unpacker = self.mock_object(xdrlib, 'Unpacker')
mock_unpacker.return_value.unpack_uint.return_value = 0
mock_socket = self.mock_object(socket, 'socket')
mock_init_call = self.mock_object(coho.Client, 'init_call')
mock_init_call.return_value = (1, 2)
mock_sendrecord = self.mock_object(coho.Client, '_sendrecord')
mock_recvrecord = self.mock_object(coho.Client, '_recvrecord')
mock_recvrecord.return_value = 'test_reply'
mock_unpack_replyheader = self.mock_object(coho.Client,
'unpack_replyheader')
mock_unpack_replyheader.return_value = (123, 1)
rpc_client = coho.CohoRPCClient(ADDR, RPC_PORT)
rpc_client.create_volume_from_snapshot('src', 'dest')
self.assertTrue(mock_sendrecord.called)
self.assertTrue(mock_unpack_replyheader.called)
mock_packer.assert_has_calls([mock.call().reset()])
mock_unpacker.assert_has_calls(
[mock.call().reset('test_reply'),
mock.call().unpack_uint()])
mock_socket.assert_has_calls([
mock.call(socket.AF_INET, socket.SOCK_STREAM),
mock.call().bind(('', 0)),
mock.call().connect((ADDR, RPC_PORT))
])
mock_init_call.assert_has_calls([
mock.call(coho.COHO1_CREATE_VOLUME_FROM_SNAPSHOT,
[(six.b('src'), mock_packer().pack_string),
(six.b('dest'), mock_packer().pack_string)])
])
