def remote_fds(self):
"""
A list of sbus file descirptors passed to remote side
"""
storlets_metadata = {}
if self.srequest.has_range:
storlets_metadata.update(
{'start': str(self.srequest.start),
'end': str(self.srequest.end)})
fds = [SBusFileDescriptor(sbus_fd.SBUS_FD_INPUT_OBJECT,
self.input_data_read_fd,
storage_metadata=self.srequest.user_metadata,
storlets_metadata=storlets_metadata),
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT,
self.data_write_fd),
SBusFileDescriptor(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
self.metadata_write_fd),
SBusFileDescriptor(sbus_fd.SBUS_FD_LOGGER,
self.storlet_logger.getfd())]
for source in self.extra_data_sources:
fd = SBusFileDescriptor(
sbus_fd.SBUS_FD_INPUT_OBJECT,
source['read_fd'],
storage_metadata=source['user_metadata'])
fds.append(fd)
return fds
def test_build_datagram_from_raw_message(self):
# SBusServiceDatagram scenario
command = SBUS_CMD_PING
fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT]
fds = [SBusFileDescriptor(sbus_fd.SBUS_FD_SERVICE_OUT, 1)]
params = {'param1': 'paramvalue1'}
task_id = 'id'
cmd_params = {'command': command, 'params': params, 'task_id': task_id}
str_metadata = json.dumps([fd.metadata for fd in fds])
str_cmd_params = json.dumps(cmd_params)
dtg = build_datagram_from_raw_message(fds, str_metadata,
str_cmd_params)
self.assertIsInstance(dtg, SBusServiceDatagram)
self.assertEqual(command, dtg.command)
self.assertEqual(fdtypes, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
# SBusExecuteDatagram scenario
command = SBUS_CMD_EXECUTE
fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT]
fds = [SBusFileDescriptor(fdtype, i + 1,
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i, fdtype in enumerate(fdtypes)]
params = {'param1': 'paramvalue1'}
task_id = 'id'
cmd_params = {'command': command, 'params': params, 'task_id': task_id}
str_metadata = json.dumps([fd.metadata for fd in fds])
str_cmd_params = json.dumps(cmd_params)
dtg = build_datagram_from_raw_message(fds, str_metadata,
str_cmd_params)
self.assertIsInstance(dtg, SBusExecuteDatagram)
self.assertEqual(command, dtg.command)
self.assertEqual(fdtypes, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
def _request(self, command, params=None, task_id=None, extra_fds=None):
read_fd, write_fd = os.pipe()
try:
try:
sfds = \
[SBusFileDescriptor(SBUS_FD_SERVICE_OUT, write_fd)] + \
(extra_fds or [])
datagram = build_datagram(command, sfds, params, task_id)
rc = SBus.send(self.socket_path, datagram)
if rc < 0:
raise SBusClientSendError(
'Faild to send command(%s) to socket %s' %
(datagram.command, self.socket_path))
finally:
# We already sent the write fd to remote, so should close it
# in local side before reading response
os.close(write_fd)
reply = b''
while True:
try:
buf = os.read(read_fd, self.chunk_size)
except IOError:
raise SBusClientIOError(
'Failed to read data from read pipe')
if not buf:
break
reply = reply + buf
finally:
os.close(read_fd)
if not isinstance(reply, str):
reply = reply.decode('utf-8')
return self._parse_response(reply)
def test_metadata(self):
fd = SBusFileDescriptor(
'MYTYPE', 1, {'storlets_key': 'storlets_value'},
{'storage_key': 'storage_value'})
self.assertEqual(
{'storlets': {'type': 'MYTYPE', 'storlets_key': 'storlets_value'},
'storage': {'storage_key': 'storage_value'}},
fd.metadata)
def test_from_fileno_and_metadata_dict(self):
fd = SBusFileDescriptor.from_fileno_and_metadata_dict(
1,
{'storlets': {'type': 'MYTYPE', 'storlets_key': 'storlets_value'},
'storage': {'storage_key': 'storage_value'}})
self.assertEqual(1, fd.fileno)
self.assertEqual('MYTYPE', fd.fdtype)
self.assertEqual({'storlets_key': 'storlets_value'},
fd.storlets_metadata)
self.assertEqual({'storage_key': 'storage_value'},
fd.storage_metadata)
def test_build_datagram(self):
# SBusServiceDatagram scenario
command = SBUS_CMD_PING
fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT]
fds = [SBusFileDescriptor(sbus_fd.SBUS_FD_SERVICE_OUT, 1)]
params = {'param1': 'paramvalue1'}
task_id = 'id'
dtg = build_datagram(command, fds, params, task_id)
self.assertIsInstance(dtg, SBusServiceDatagram)
self.assertEqual(command, dtg.command)
self.assertEqual(fdtypes, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
# SBusExecuteDatagram scenario
command = SBUS_CMD_EXECUTE
fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT]
fds = [SBusFileDescriptor(fdtype, i + 1,
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i, fdtype in enumerate(fdtypes)]
params = {'param1': 'paramvalue1'}
task_id = 'id'
dtg = build_datagram(command, fds, params, task_id)
self.assertIsInstance(dtg, SBusExecuteDatagram)
self.assertEqual(command, dtg.command)
self.assertEqual(fdtypes, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
def setUp(self):
self.command = SBUS_CMD_EXECUTE
self.fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_LOGGER]
self.sfds = [SBusFileDescriptor(
fdtype, i + 1,
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i, fdtype in enumerate(self.fdtypes)]
super(TestSBusExecuteDatagram, self).setUp()
def setUp(self):
self.command = SBUS_CMD_EXECUTE
self.types = [
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER
]
self.sfds = [
SBusFileDescriptor(self.types[i], i + 1,
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i in range(len(self.types))
]
super(TestSBusExecuteDatagram, self).setUp()
def test_init_extra_sources(self):
types = [
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT
]
fds = [
SBusFileDescriptor(types[i], i + 1, {'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i in range(len(types))
]
dtg = self._test_class(self.command, fds, self.params, self.task_id)
self.assertEqual(types, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.task_id, dtg.task_id)
def test_init_extra_sources(self):
fdtypes = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT]
fds = [SBusFileDescriptor(fdtype, i + 1,
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i})
for i, fdtype in enumerate(fdtypes)]
dtg = self._test_class(
self.command, fds, self.params, self.task_id)
self.assertEqual(fdtypes, [sfd.fdtype for sfd in dtg.sfds])
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.task_id, dtg.task_id)
def _test_main_loop_stop(self, stop_command):
sfds = [SBusFileDescriptor(SBUS_FD_SERVICE_OUT, 1)]
scenario = [
('create', 1),
('listen', 1),
('receive',
SBusServiceDatagram(command=stop_command,
sfds=sfds,
params=None,
task_id=None)),
]
fake_sbus_class = create_fake_sbus_class(scenario)
with mock.patch('storlets.agent.common.server.SBus', fake_sbus_class):
with mock.patch('os.fdopen'):
ret = self.server.main_loop()
self.assertEqual(EXIT_SUCCESS, ret)
# sanity for no error and no warning
self.assertEqual([], self.logger.get_log_lines('error'))
self.assertEqual([], self.logger.get_log_lines('warn'))
def setUp(self):
self.command = 'SBUS_CMD_TEST'
self.types = [sbus_fd.SBUS_FD_SERVICE_OUT]
self.sfds = [SBusFileDescriptor(sbus_fd.SBUS_FD_SERVICE_OUT, 1)]
super(TestSBusServiceDatagram, self).setUp()