Exemplo n.º 1
0
 def test_to_dict(self):
     md = FDMetadata('MYTYPE', {'storlets_key': 'storlets_value'},
                     {'storage_key': 'storage_value'})
     self.assertEqual({'storlets': {'type': 'MYTYPE',
                                    'storlets_key': 'storlets_value'},
                       'storage': {'storage_key': 'storage_value'}},
                      md.to_dict())
Exemplo n.º 2
0
    def test_build_datagram_from_raw_message(self):
        # SBusServiceDatagram scenario
        command = SBUS_CMD_PING
        types = [sbus_fd.SBUS_FD_SERVICE_OUT]
        fds = [1]
        metadata = [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()]
        params = {'param1': 'paramvalue1'}
        task_id = 'id'
        cmd_params = {'command': command, 'params': params, 'task_id': task_id}

        str_metadata = json.dumps(metadata)
        str_cmd_params = json.dumps(cmd_params)
        dtg = build_datagram_from_raw_message(fds, str_metadata,
                                              str_cmd_params)

        self.assertEqual(command, dtg.command)
        self.assertEqual(fds, dtg.fds)
        self.assertEqual(metadata, dtg.metadata)
        self.assertEqual(params, dtg.params)
        self.assertEqual(task_id, dtg.task_id)

        # SBusExecuteDatagram scenario
        command = SBUS_CMD_EXECUTE
        types = [
            sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
            sbus_fd.SBUS_FD_OUTPUT_OBJECT,
            sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER,
            sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_INPUT_OBJECT,
            sbus_fd.SBUS_FD_INPUT_OBJECT
        ]
        fds = [i + 1 for i in range(len(types))]
        metadata = [
            FDMetadata(types[i], {
                'key%d' % i: 'value%d' % i
            }, {
                'skey%d' % i: 'svalue%d' % i
            }).to_dict() for i in range(len(types))
        ]
        params = {'param1': 'paramvalue1'}
        task_id = 'id'
        cmd_params = {'command': command, 'params': params, 'task_id': task_id}

        str_metadata = json.dumps(metadata)
        str_cmd_params = json.dumps(cmd_params)
        dtg = build_datagram_from_raw_message(fds, str_metadata,
                                              str_cmd_params)

        self.assertEqual(command, dtg.command)
        self.assertEqual(fds, dtg.fds)
        self.assertEqual(metadata, dtg.metadata)
        self.assertEqual(params, dtg.params)
        self.assertEqual(task_id, dtg.task_id)
Exemplo n.º 3
0
 def test_to_dict(self):
     md = FDMetadata('MYTYPE', {'storlets_key': 'storlets_value'},
                     {'storage_key': 'storage_value'})
     self.assertEqual(
         {
             'storlets': {
                 'type': 'MYTYPE',
                 'storlets_key': 'storlets_value'
             },
             'storage': {
                 'storage_key': 'storage_value'
             }
         }, md.to_dict())
Exemplo n.º 4
0
    def get_storlet_daemon_status(self, storlet_id):
        """
        Get the status of SDaemon process in the scope's sandbox
        """
        with _open_pipe() as (read_fd, write_fd):
            dtg = SBusServiceDatagram(
                sbus_cmd.SBUS_CMD_DAEMON_STATUS,
                [write_fd],
                [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
                {'storlet_name': storlet_id})
            pipe_path = self.paths.host_factory_pipe()
            rc = SBus.send(pipe_path, dtg)
            if (rc < 0):
                self.logger.info("Failed to send status command to %s %s" %
                                 (self.scope, storlet_id))
                return -1

            reply = os.read(read_fd, 10)

        res, error_txt = self._parse_sandbox_factory_answer(reply)
        if res is True:
            return 1
        self.logger.error('Failed to get status about storlet daemon: %s' %
                          error_txt)
        return 0
Exemplo n.º 5
0
    def ping(self):
        """
        Ping to daemon factory process inside container

        :returns:  1 when the daemon factory is responsive
                   0 when the daemon factory is not responsive
                  -1 when it fails to send command to the process
        """
        pipe_path = self.paths.host_factory_pipe()

        with _open_pipe() as (read_fd, write_fd):
            dtg = SBusServiceDatagram(
                sbus_cmd.SBUS_CMD_PING,
                [write_fd],
                [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()])
            rc = SBus.send(pipe_path, dtg)
            if (rc < 0):
                return -1

            reply = os.read(read_fd, 10)

        res, error_txt = self._parse_sandbox_factory_answer(reply)
        if res is True:
            return 1
        self.logger.error('Failed to ping to daemon factory: %s' % error_txt)
        return 0
Exemplo n.º 6
0
 def test_from_dict(self):
     md = FDMetadata.from_dict(
         {'storlets': {'type': 'MYTYPE',
                       'storlets_key': 'storlets_value'},
          'storage': {'storage_key': 'storage_value'}})
     self.assertEqual('MYTYPE', md.fdtype)
     self.assertEqual({'storlets_key': 'storlets_value'},
                      md.storlets_metadata)
     self.assertEqual({'storage_key': 'storage_value'},
                      md.storage_metadata)
Exemplo n.º 7
0
    def remote_fds_metadata(self):
        """
        Metadata about file descriptors to be passed to container side
        """
        input_fd_metadata = FDMetadata(
            sbus_fd.SBUS_FD_INPUT_OBJECT,
            storage_metadata=self.srequest.user_metadata)
        if self.srequest.user_metadata:
            input_fd_metadata.storage_metadata.update(
                self.srequest.user_metadata)
        if self.srequest.has_range:
            input_fd_metadata.storlets_metadata['start'] = \
                str(self.srequest.start)
            input_fd_metadata.storlets_metadata['end'] = \
                str(self.srequest.end)
        fds_metadata = [
            input_fd_metadata.to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_TASK_ID).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_LOGGER).to_dict()]

        for source in self.extra_data_sources:
            fdmd = FDMetadata(
                sbus_fd.SBUS_FD_INPUT_OBJECT,
                storage_metadata=source['user_metadata'])
            fds_metadata.append(fdmd.to_dict())
        return fds_metadata
Exemplo n.º 8
0
 def test_from_dict(self):
     md = FDMetadata.from_dict({
         'storlets': {
             'type': 'MYTYPE',
             'storlets_key': 'storlets_value'
         },
         'storage': {
             'storage_key': 'storage_value'
         }
     })
     self.assertEqual('MYTYPE', md.fdtype)
     self.assertEqual({'storlets_key': 'storlets_value'},
                      md.storlets_metadata)
     self.assertEqual({'storage_key': 'storage_value'}, md.storage_metadata)
Exemplo n.º 9
0
 def _cancel(self):
     """
     Cancel on-going storlet execution
     """
     with _open_pipe() as (read_fd, write_fd):
         dtg = SBusServiceDatagram(
             sbus_cmd.SBUS_CMD_CANCEL,
             [write_fd],
             [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
             None,
             self.task_id)
         rc = SBus.send(self.storlet_pipe_path, dtg)
         if (rc < 0):
             raise StorletRuntimeException('Failed to cancel task')
         # TODO(takashi): Check the response here
         os.read(read_fd, 10)
Exemplo n.º 10
0
 def setUp(self):
     self.command = SBUS_CMD_EXECUTE
     self.types = [
         sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
         sbus_fd.SBUS_FD_OUTPUT_OBJECT,
         sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER
     ]
     self.fds = [i + 1 for i in range(len(self.types))]
     self.metadata = [
         FDMetadata(self.types[i], {
             'key%d' % i: 'value%d' % i
         }, {
             'skey%d' % i: 'svalue%d' % i
         }).to_dict() for i in range(len(self.types))
     ]
     super(TestSBusExecuteDatagram, self).setUp()
Exemplo n.º 11
0
def main(argv):
    if len(argv) < 2:
        print_usage(argv)
        return

    daemon_factory_pipe_name = argv[1]
    try:
        fi, fo = os.pipe()
        halt_dtg = SBusServiceDatagram(
            SBUS_CMD_HALT, [fo], [FDMetadata(SBUS_FD_SERVICE_OUT).to_dict()])
        n_status = SBus.send(daemon_factory_pipe_name, halt_dtg)
        if n_status < 0:
            print('Sending failed')
        else:
            print('Sending succeeded')
            cmd_response = os.read(fi, 256)
            print(cmd_response)
    finally:
        os.close(fi)
        os.close(fo)
Exemplo n.º 12
0
    def _test_main_loop_stop(self, stop_command):
        metadata = [FDMetadata(SBUS_FD_SERVICE_OUT).to_dict()]
        scenario = [
            ('create', 1),
            ('listen', 1),
            ('receive',
             SBusServiceDatagram(command=stop_command,
                                 fds=[1],
                                 metadata=metadata,
                                 params=None,
                                 task_id=None)),
        ]

        fake_sbus_class = create_fake_sbus_class(scenario)
        with mock.patch('storlets.agent.common.server.SBus', fake_sbus_class):
            with mock.patch('os.fdopen'):
                ret = self.server.main_loop()

        self.assertEqual(EXIT_SUCCESS, ret)
        # sanity for no error and no warning
        self.assertEqual([], self.logger.get_log_lines('error'))
        self.assertEqual([], self.logger.get_log_lines('warn'))
Exemplo n.º 13
0
 def test_init_extra_sources(self):
     types = [
         sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_OUTPUT_TASK_ID,
         sbus_fd.SBUS_FD_OUTPUT_OBJECT,
         sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA, sbus_fd.SBUS_FD_LOGGER,
         sbus_fd.SBUS_FD_INPUT_OBJECT, sbus_fd.SBUS_FD_INPUT_OBJECT,
         sbus_fd.SBUS_FD_INPUT_OBJECT
     ]
     fds = [i + 1 for i in range(len(types))]
     metadata = [
         FDMetadata(types[i], {
             'key%d' % i: 'value%d' % i
         }, {
             'skey%d' % i: 'svalue%d' % i
         }).to_dict() for i in range(len(types))
     ]
     dtg = self._test_class(self.command, fds, metadata, self.params,
                            self.task_id)
     self.assertEqual(dtg.fds, fds)
     self.assertEqual(dtg.metadata, metadata)
     self.assertEqual(dtg.params, self.params)
     self.assertEqual(dtg.task_id, self.task_id)
Exemplo n.º 14
0
    def start_storlet_daemon(
            self, spath, storlet_id, language, language_version=None):
        """
        Start SDaemon process in the scope's sandbox

        """
        prms = {'daemon_language': language.lower(),
                'storlet_path': spath,
                'storlet_name': storlet_id,
                'uds_path': self.paths.sbox_storlet_pipe(storlet_id),
                'log_level': self.storlet_daemon_debug_level,
                'pool_size': self.storlet_daemon_thread_pool_size}

        if language_version:
            prms.update({'daemon_language_version': language_version})

        with _open_pipe() as (read_fd, write_fd):
            dtg = SBusServiceDatagram(
                sbus_cmd.SBUS_CMD_START_DAEMON,
                [write_fd],
                [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()],
                prms)

            pipe_path = self.paths.host_factory_pipe()
            rc = SBus.send(pipe_path, dtg)
            # TODO(takashi): Why we should rond rc into -1?
            if (rc < 0):
                return -1

            reply = os.read(read_fd, 10)

        res, error_txt = self._parse_sandbox_factory_answer(reply)
        if res is True:
            return 1
        self.logger.error('Failed to start storlet daemon: %s' % error_txt)
        return 0
Exemplo n.º 15
0
    def remote_fds_metadata(self):
        """
        Metadata about file descriptors to be passed to container side
        """
        input_fd_metadata = FDMetadata(
            sbus_fd.SBUS_FD_INPUT_OBJECT,
            storage_metadata=self.srequest.user_metadata)
        if self.srequest.user_metadata:
            input_fd_metadata.storage_metadata.update(
                self.srequest.user_metadata)
        if self.srequest.has_range:
            input_fd_metadata.storlets_metadata['start'] = \
                str(self.srequest.start)
            input_fd_metadata.storlets_metadata['end'] = \
                str(self.srequest.end)
        fds_metadata = [
            input_fd_metadata.to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_TASK_ID).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_OUTPUT_OBJECT_METADATA).to_dict(),
            FDMetadata(sbus_fd.SBUS_FD_LOGGER).to_dict()]

        for source in self.extra_data_sources:
            fdmd = FDMetadata(
                sbus_fd.SBUS_FD_INPUT_OBJECT,
                storage_metadata=source['user_metadata'])
            fds_metadata.append(fdmd.to_dict())
        return fds_metadata
Exemplo n.º 16
0
 def setUp(self):
     self.command = 'SBUS_CMD_TEST'
     self.types = [sbus_fd.SBUS_FD_SERVICE_OUT]
     self.fds = [1]
     self.metadata = [FDMetadata(sbus_fd.SBUS_FD_SERVICE_OUT).to_dict()]
     super(TestSBusServiceDatagram, self).setUp()