def test_to_dict(self):
md = FDMetadata('MYTYPE', {'storlets_key': 'storlets_value'},
{'storage_key': 'storage_value'})
self.assertEqual({'storlets': {'type': 'MYTYPE',
'storlets_key': 'storlets_value'},
'storage': {'storage_key': 'storage_value'}},
md.to_dict())
def test_build_datagram_from_raw_message(self):
# SBusServiceDatagram scenario
command = SBUS_CMD_PING
types = [sbus_fd.SBUS_FD_SERVICE_OUT]
fds = [1]
metadata = [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()]
params = {'param1': 'paramvalue1'}
task_id = 'id'
cmd_params = {'command': command, 'params': params, 'task_id': task_id}
str_metadata = json.dumps(metadata)
str_cmd_params = json.dumps(cmd_params)
dtg = build_datagram_from_raw_message(fds, str_metadata,
str_cmd_params)
self.assertEqual(command, dtg.command)
self.assertEqual(fds, dtg.fds)
self.assertEqual(metadata, dtg.metadata)
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
# SBusExecuteDatagram scenario
command = SBUS_CMD_EXECUTE
types = [
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT
]
fds = [i + 1 for i in range(len(types))]
metadata = [
FDMetadata(types[i], {
'key%d' % i: 'value%d' % i
}, {
'skey%d' % i: 'svalue%d' % i
}).to_dict() for i in range(len(types))
]
params = {'param1': 'paramvalue1'}
task_id = 'id'
cmd_params = {'command': command, 'params': params, 'task_id': task_id}
str_metadata = json.dumps(metadata)
str_cmd_params = json.dumps(cmd_params)
dtg = build_datagram_from_raw_message(fds, str_metadata,
str_cmd_params)
self.assertEqual(command, dtg.command)
self.assertEqual(fds, dtg.fds)
self.assertEqual(metadata, dtg.metadata)
self.assertEqual(params, dtg.params)
self.assertEqual(task_id, dtg.task_id)
def test_to_dict(self):
md = FDMetadata('MYTYPE', {'storlets_key': 'storlets_value'},
{'storage_key': 'storage_value'})
self.assertEqual(
{
'storlets': {
'type': 'MYTYPE',
'storlets_key': 'storlets_value'
},
'storage': {
'storage_key': 'storage_value'
}
}, md.to_dict())
def get_storlet_daemon_status(self, storlet_id):
"""
Get the status of SDaemon process in the scope's sandbox
"""
with _open_pipe() as (read_fd, write_fd):
dtg = SBusServiceDatagram(
sbus_cmd.SBUS_CMD_DAEMON_STATUS,
[write_fd],
[FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
{'storlet_name': storlet_id})
pipe_path = self.paths.host_factory_pipe()
rc = SBus.send(pipe_path, dtg)
if (rc < 0):
self.logger.info("Failed to send status command to %s %s" %
(self.scope, storlet_id))
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to get status about storlet daemon: %s' %
error_txt)
return 0
def ping(self):
"""
Ping to daemon factory process inside container
:returns: 1 when the daemon factory is responsive
0 when the daemon factory is not responsive
-1 when it fails to send command to the process
"""
pipe_path = self.paths.host_factory_pipe()
with _open_pipe() as (read_fd, write_fd):
dtg = SBusServiceDatagram(
sbus_cmd.SBUS_CMD_PING,
[write_fd],
[FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()])
rc = SBus.send(pipe_path, dtg)
if (rc < 0):
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to ping to daemon factory: %s' % error_txt)
return 0
def test_from_dict(self):
md = FDMetadata.from_dict(
{'storlets': {'type': 'MYTYPE',
'storlets_key': 'storlets_value'},
'storage': {'storage_key': 'storage_value'}})
self.assertEqual('MYTYPE', md.fdtype)
self.assertEqual({'storlets_key': 'storlets_value'},
md.storlets_metadata)
self.assertEqual({'storage_key': 'storage_value'},
md.storage_metadata)
def remote_fds_metadata(self):
"""
Metadata about file descriptors to be passed to container side
"""
input_fd_metadata = FDMetadata(
sbus_fd.SBUS_FD_INPUT_OBJECT,
storage_metadata=self.srequest.user_metadata)
if self.srequest.user_metadata:
input_fd_metadata.storage_metadata.update(
self.srequest.user_metadata)
if self.srequest.has_range:
input_fd_metadata.storlets_metadata['start'] = \
str(self.srequest.start)
input_fd_metadata.storlets_metadata['end'] = \
str(self.srequest.end)
fds_metadata = [
input_fd_metadata.to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_TASK_ID).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_LOGGER).to_dict()]
for source in self.extra_data_sources:
fdmd = FDMetadata(
sbus_fd.SBUS_FD_INPUT_OBJECT,
storage_metadata=source['user_metadata'])
fds_metadata.append(fdmd.to_dict())
return fds_metadata
def test_from_dict(self):
md = FDMetadata.from_dict({
'storlets': {
'type': 'MYTYPE',
'storlets_key': 'storlets_value'
},
'storage': {
'storage_key': 'storage_value'
}
})
self.assertEqual('MYTYPE', md.fdtype)
self.assertEqual({'storlets_key': 'storlets_value'},
md.storlets_metadata)
self.assertEqual({'storage_key': 'storage_value'}, md.storage_metadata)
def _cancel(self):
"""
Cancel on-going storlet execution
"""
with _open_pipe() as (read_fd, write_fd):
dtg = SBusServiceDatagram(
sbus_cmd.SBUS_CMD_CANCEL,
[write_fd],
[FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
None,
self.task_id)
rc = SBus.send(self.storlet_pipe_path, dtg)
if (rc < 0):
raise StorletRuntimeException('Failed to cancel task')
# TODO(takashi): Check the response here
os.read(read_fd, 10)
def setUp(self):
self.command = SBUS_CMD_EXECUTE
self.types = [
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER
]
self.fds = [i + 1 for i in range(len(self.types))]
self.metadata = [
FDMetadata(self.types[i], {
'key%d' % i: 'value%d' % i
}, {
'skey%d' % i: 'svalue%d' % i
}).to_dict() for i in range(len(self.types))
]
super(TestSBusExecuteDatagram, self).setUp()
def main(argv):
if len(argv) < 2:
print_usage(argv)
return
daemon_factory_pipe_name = argv[1]
try:
fi, fo = os.pipe()
halt_dtg = SBusServiceDatagram(
SBUS_CMD_HALT, [fo], [FDMetadata(SBUS_FD_SERVICE_OUT).to_dict()])
n_status = SBus.send(daemon_factory_pipe_name, halt_dtg)
if n_status < 0:
print('Sending failed')
else:
print('Sending succeeded')
cmd_response = os.read(fi, 256)
print(cmd_response)
finally:
os.close(fi)
os.close(fo)
def _test_main_loop_stop(self, stop_command):
metadata = [FDMetadata(SBUS_FD_SERVICE_OUT).to_dict()]
scenario = [
('create', 1),
('listen', 1),
('receive',
SBusServiceDatagram(command=stop_command,
fds=[1],
metadata=metadata,
params=None,
task_id=None)),
]
fake_sbus_class = create_fake_sbus_class(scenario)
with mock.patch('storlets.agent.common.server.SBus', fake_sbus_class):
with mock.patch('os.fdopen'):
ret = self.server.main_loop()
self.assertEqual(EXIT_SUCCESS, ret)
# sanity for no error and no warning
self.assertEqual([], self.logger.get_log_lines('error'))
self.assertEqual([], self.logger.get_log_lines('warn'))
def test_init_extra_sources(self):
types = [
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT
]
fds = [i + 1 for i in range(len(types))]
metadata = [
FDMetadata(types[i], {
'key%d' % i: 'value%d' % i
}, {
'skey%d' % i: 'svalue%d' % i
}).to_dict() for i in range(len(types))
]
dtg = self._test_class(self.command, fds, metadata, self.params,
self.task_id)
self.assertEqual(dtg.fds, fds)
self.assertEqual(dtg.metadata, metadata)
self.assertEqual(dtg.params, self.params)
self.assertEqual(dtg.task_id, self.task_id)
def start_storlet_daemon(
self, spath, storlet_id, language, language_version=None):
"""
Start SDaemon process in the scope's sandbox
"""
prms = {'daemon_language': language.lower(),
'storlet_path': spath,
'storlet_name': storlet_id,
'uds_path': self.paths.sbox_storlet_pipe(storlet_id),
'log_level': self.storlet_daemon_debug_level,
'pool_size': self.storlet_daemon_thread_pool_size}
if language_version:
prms.update({'daemon_language_version': language_version})
with _open_pipe() as (read_fd, write_fd):
dtg = SBusServiceDatagram(
sbus_cmd.SBUS_CMD_START_DAEMON,
[write_fd],
[FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
prms)
pipe_path = self.paths.host_factory_pipe()
rc = SBus.send(pipe_path, dtg)
# TODO(takashi): Why we should rond rc into -1?
if (rc < 0):
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to start storlet daemon: %s' % error_txt)
return 0
def remote_fds_metadata(self):
"""
Metadata about file descriptors to be passed to container side
"""
input_fd_metadata = FDMetadata(
sbus_fd.SBUS_FD_INPUT_OBJECT,
storage_metadata=self.srequest.user_metadata)
if self.srequest.user_metadata:
input_fd_metadata.storage_metadata.update(
self.srequest.user_metadata)
if self.srequest.has_range:
input_fd_metadata.storlets_metadata['start'] = \
str(self.srequest.start)
input_fd_metadata.storlets_metadata['end'] = \
str(self.srequest.end)
fds_metadata = [
input_fd_metadata.to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_TASK_ID).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA).to_dict(),
FDMetadata(sbus_fd.SBUS_FD_LOGGER).to_dict()]
for source in self.extra_data_sources:
fdmd = FDMetadata(
sbus_fd.SBUS_FD_INPUT_OBJECT,
storage_metadata=source['user_metadata'])
fds_metadata.append(fdmd.to_dict())
return fds_metadata
def setUp(self):
self.command = 'SBUS_CMD_TEST'
self.types = [sbus_fd.SBUS_FD_SERVICE_OUT]
self.fds = [1]
self.metadata = [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()]
super(TestSBusServiceDatagram, self).setUp()
