def start_storlet_daemon(self, spath, storlet_id, language):
"""
Start SDaemon process in the scope's sandbox
"""
prms = {'daemon_language': language.lower(),
'storlet_path': spath,
'storlet_name': storlet_id,
'uds_path': self.paths.sbox_storlet_pipe(storlet_id),
'log_level': self.storlet_daemon_debug_level,
'pool_size': self.storlet_daemon_thread_pool_size}
with _open_pipe() as (read_fd, write_fd):
dtg = SBusDatagram.create_service_datagram(
sbus_cmd.SBUS_CMD_START_DAEMON,
write_fd,
prms)
pipe_path = self.paths.host_factory_pipe()
rc = SBus.send(pipe_path, dtg)
# TODO(takashi): Why we should rond rc into -1?
if (rc < 0):
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to start storlet daemon: %s' % error_txt)
return 0
def test_check_required_fd_types_not_implemented(self):
# SBusDatagram designed not to be called independently
with self.assertRaises(NotImplementedError) as err:
SBusDatagram('', [], [])
self.assertEqual(
'SBusDatagram class should not be initialized as bare',
err.exception.args[0])
def wait_for_daemon_to_initialize(self, storlet_name):
"""
Send a Ping service datagram. Validate that
Daemon response is correct. Give up after the
predefined number of attempts (5)
:param storlet_name: Storlet name we are checking the daemon for
:returns: daemon status (True, False)
"""
storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name]
self.logger.debug('Send PING command to {0} via {1}'.
format(storlet_name, storlet_pipe_name))
read_fd, write_fd = os.pipe()
try:
dtg = SBusDatagram.create_service_datagram(
SBUS_CMD_PING, write_fd)
for i in range(self.NUM_OF_TRIES_PINGING_STARTING_DAEMON):
ret = SBus.send(storlet_pipe_name, dtg)
if ret >= 0:
resp = os.read(read_fd, 128)
if resp.startswith('True'):
return True
time.sleep(1)
else:
return False
finally:
os.close(read_fd)
os.close(write_fd)
def ping(self):
"""
Ping to daemon factory process inside container
:returns: 1 when the daemon factory is responsive
0 when the daemon factory is not responsive
-1 when it fails to send command to the process
"""
pipe_path = self.paths.host_factory_pipe()
with _open_pipe() as (read_fd, write_fd):
dtg = SBusDatagram.create_service_datagram(
sbus_cmd.SBUS_CMD_PING,
write_fd)
rc = SBus.send(pipe_path, dtg)
if (rc < 0):
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to ping to daemon factory: %s' % error_txt)
return 0
def test_create_service_datagram(self):
dtg = SBusDatagram.create_service_datagram(
self.command, 1, self.params, self.task_id)
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.command, dtg.command)
self.assertEqual(self.task_id, dtg.task_id)
self.assertEqual([1], dtg.fds)
self.assertEqual([{'storlets': {'type': sbus_fd.SBUS_FD_SERVICE_OUT},
'storage': {}}], dtg.metadata)
dtg = SBusDatagram.create_service_datagram(
self.command, 1)
self.assertIsNone(dtg.params)
self.assertEqual(self.command, dtg.command)
self.assertIsNone(dtg.task_id)
self.assertEqual([1], dtg.fds)
self.assertEqual([{'storlets': {'type': sbus_fd.SBUS_FD_SERVICE_OUT},
'storage': {}}], dtg.metadata)
def test_build_from_raw_message(self):
str_metadata = json.dumps(self.metadata)
str_cmd_params = json.dumps(self.dtg.cmd_params)
dtg = SBusDatagram.build_from_raw_message(
self.fds, str_metadata, str_cmd_params)
self.assertEqual(self.command, dtg.command)
self.assertEqual(self.fds, dtg.fds)
self.assertEqual(self.metadata, dtg.metadata)
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.task_id, dtg.task_id)
def _cancel(self):
"""
Cancel on-going storlet execution
"""
with _open_pipe() as (read_fd, write_fd):
dtg = SBusDatagram.create_service_datagram(
sbus_cmd.SBUS_CMD_CANCEL,
write_fd,
None,
self.task_id)
rc = SBus.send(self.storlet_pipe_path, dtg)
if (rc < 0):
raise StorletRuntimeException('Failed to cancel task')
# TODO(takashi): Check the reponse here
os.read(read_fd, 10)
def shutdown_process(self, storlet_name):
"""
send HALT command to storlet daemon
:param storlet_name: Storlet name we are checking the daemon for
:raises SDaemonError: when wailed to shutdown the storlet daemon
"""
self.logger.debug(
'Shutdown the storlet daemon {0}'.format(storlet_name))
dmn_pid = self.storlet_name_to_pid.get(storlet_name)
self.logger.debug('Storlet Daemon PID is {0}'.format(dmn_pid))
if dmn_pid is None:
raise SDaemonError('{0} is not found'.format(storlet_name))
storlet_pipe_name = self.storlet_name_to_pipe_name[storlet_name]
self.logger.debug('Send HALT command to {0} via {1}'.
format(storlet_name, storlet_pipe_name))
read_fd, write_fd = os.pipe()
try:
dtg = SBusDatagram.create_service_datagram(
SBUS_CMD_HALT, write_fd)
rc = SBus.send(storlet_pipe_name, dtg)
os.close(write_fd)
if rc < 0:
raise SDaemonError(
'Failed to send halt to {0}'.format(storlet_name))
resp = os.read(read_fd, 128)
if not resp.startswith('True'):
raise SDaemonError(
'Failed to send halt to {0}'.format(storlet_name))
finally:
os.close(read_fd)
try:
os.waitpid(dmn_pid, 0)
self.storlet_name_to_pid.pop(storlet_name)
except OSError:
self.logger.exception(
'Error when waiting the storlet daemon {0}'.format(
storlet_name))
raise SDaemonError('Failed to wait {0}'.format(storlet_name))
def receive(self, sbus_handler):
ph_files = POINTER(c_int)()
pp_metadata = (c_char_p)()
pp_params = (c_char_p)()
pn_files = (c_int)()
pn_metadata = (c_int)()
pn_params = (c_int)()
# Invoke C function
n_status = self.sbus_back_.sbus_recv_msg(sbus_handler,
ph_files,
pn_files,
pp_metadata,
pn_metadata,
pp_params,
pn_params)
if n_status < 0:
return None
# The invocation was successful.
# De-serialize the data
# Aggregate file descriptors
n_files = pn_files.value
h_files = []
for i in xrange(n_files):
h_files.append(ph_files[i])
# Extract Python strings
n_metadata = pn_metadata.value
str_metadata = pp_metadata.value
n_params = pn_params.value
str_params = pp_params.value
# Trim the junk out
if 0 < n_metadata:
str_metadata = str_metadata[0:n_metadata]
str_params = str_params[0:n_params]
# Construct actual result datagram
return SBusDatagram.build_from_raw_message(
h_files, str_metadata, str_params)
def main(argv):
if len(argv) < 2:
print_usage(argv)
return
daemon_factory_pipe_name = argv[1]
try:
fi, fo = os.pipe()
halt_dtg = SBusDatagram.create_service_datagram(
SBUS_CMD_HALT, fo)
n_status = SBus.send(daemon_factory_pipe_name, halt_dtg)
if n_status < 0:
print('Sending failed')
else:
print('Sending succeeded')
cmd_response = os.read(fi, 256)
print(cmd_response)
finally:
os.close(fi)
os.close(fo)
def setUp(self):
self.command = 'COMMAND'
self.types = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT]
self.fds = []
self.metadata = []
for i in xrange(len(self.types)):
self.fds.append(i + 1)
self.metadata.append(
FDMetadata(self.types[i],
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i}).to_dict())
self.params = {'param1': 'paramvalue1'}
self.task_id = 'id'
self.dtg = SBusDatagram(self.command, self.fds, self.metadata,
self.params, self.task_id)
def stop_storlet_daemon(self, storlet_id):
"""
Stop SDaemon process in the scope's sandbox
"""
with _open_pipe() as (read_fd, write_fd):
dtg = SBusDatagram.create_service_datagram(
sbus_cmd.SBUS_CMD_STOP_DAEMON,
write_fd,
{'storlet_name': storlet_id})
pipe_path = self.paths.host_factory_pipe()
rc = SBus.send(pipe_path, dtg)
if (rc < 0):
self.logger.info("Failed to send status command to %s %s" %
(self.scope, storlet_id))
return -1
reply = os.read(read_fd, 10)
res, error_txt = self._parse_sandbox_factory_answer(reply)
if res is True:
return 1
self.logger.error('Failed to stop storlet daemon: %s' % error_txt)
return 0
class TestSBusDatagram(unittest.TestCase):
def setUp(self):
self.command = 'COMMAND'
self.types = [sbus_fd.SBUS_FD_SERVICE_OUT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA,
sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
sbus_fd.SBUS_FD_LOGGER,
sbus_fd.SBUS_FD_INPUT_OBJECT,
sbus_fd.SBUS_FD_INPUT_OBJECT]
self.fds = []
self.metadata = []
for i in xrange(len(self.types)):
self.fds.append(i + 1)
self.metadata.append(
FDMetadata(self.types[i],
{'key%d' % i: 'value%d' % i},
{'skey%d' % i: 'svalue%d' % i}).to_dict())
self.params = {'param1': 'paramvalue1'}
self.task_id = 'id'
self.dtg = SBusDatagram(self.command, self.fds, self.metadata,
self.params, self.task_id)
def test_init(self):
self.assertEqual(self.command, self.dtg.command)
self.assertEqual(self.fds, self.dtg.fds)
self.assertEqual(self.metadata, self.dtg.metadata)
self.assertEqual(self.params, self.dtg.params)
self.assertEqual(self.task_id, self.dtg.task_id)
def test_num_fds(self):
self.assertEqual(len(self.types), self.dtg.num_fds)
def test_cmd_params(self):
self.assertEqual({'command': self.command,
'params': self.params,
'task_id': self.task_id},
self.dtg.cmd_params)
def test_serialized_metadata(self):
self.assertEqual(self.metadata,
json.loads(self.dtg.serialized_metadata))
def test_serialized_cmd_params(self):
res = {'command': self.command,
'params': self.params,
'task_id': self.task_id}
self.assertEqual(res,
json.loads(self.dtg.serialized_cmd_params))
def test_create_service_datagram(self):
dtg = SBusDatagram.create_service_datagram(
self.command, 1, self.params, self.task_id)
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.command, dtg.command)
self.assertEqual(self.task_id, dtg.task_id)
self.assertEqual([1], dtg.fds)
self.assertEqual([{'storlets': {'type': sbus_fd.SBUS_FD_SERVICE_OUT},
'storage': {}}], dtg.metadata)
dtg = SBusDatagram.create_service_datagram(
self.command, 1)
self.assertIsNone(dtg.params)
self.assertEqual(self.command, dtg.command)
self.assertIsNone(dtg.task_id)
self.assertEqual([1], dtg.fds)
self.assertEqual([{'storlets': {'type': sbus_fd.SBUS_FD_SERVICE_OUT},
'storage': {}}], dtg.metadata)
def test_find_fds(self):
self.assertEqual(
[1], self.dtg._find_fds(sbus_fd.SBUS_FD_SERVICE_OUT))
self.assertEqual(
[2, 3], self.dtg._find_fds(sbus_fd.SBUS_FD_OUTPUT_OBJECT))
self.assertEqual(
[], self.dtg._find_fds('DUMMY_TYPE'))
def test_find_fd(self):
self.assertEqual(
1, self.dtg._find_fd(sbus_fd.SBUS_FD_SERVICE_OUT))
self.assertEqual(
2, self.dtg._find_fd(sbus_fd.SBUS_FD_OUTPUT_OBJECT))
self.assertIsNone(
self.dtg._find_fd('DUMMY_TYPE'))
def test_service_out_fd(self):
self.assertEqual(1, self.dtg.service_out_fd)
def test_object_out_fds(self):
self.assertEqual([2, 3], self.dtg.object_out_fds)
def test_object_metadata_out_fds(self):
self.assertEqual([4, 5], self.dtg.object_metadata_out_fds)
def test_task_id_out_fd(self):
self.assertEqual(6, self.dtg.task_id_out_fd)
def test_logger_out_fd(self):
self.assertEqual(7, self.dtg.logger_out_fd)
def test_object_in_fds(self):
self.assertEqual([8, 9], self.dtg.object_in_fds)
def test_build_from_raw_message(self):
str_metadata = json.dumps(self.metadata)
str_cmd_params = json.dumps(self.dtg.cmd_params)
dtg = SBusDatagram.build_from_raw_message(
self.fds, str_metadata, str_cmd_params)
self.assertEqual(self.command, dtg.command)
self.assertEqual(self.fds, dtg.fds)
self.assertEqual(self.metadata, dtg.metadata)
self.assertEqual(self.params, dtg.params)
self.assertEqual(self.task_id, dtg.task_id)
