Ejemplo n.º 1
0
    def test_validate_dependency_registration(self):
        # w/o dependency parameter
        obj = 'dep_file'
        params = {'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ correct dependency parameter
        params = {
            'Dependency-Permissions': '755',
            'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ wrong dependency parameter
        params = {
            'Dependency-Permissions': '400',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ invalid dependency parameter
        params = {
            'Dependency-Permissions': 'foo',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
        params = {
            'Dependency-Permissions': '888',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
Ejemplo n.º 2
0
    def test_validate_dependency_registration(self):
        # w/o dependency parameter
        obj = 'dep_file'
        params = {'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ correct dependency parameter
        params = {
            'Dependency-Permissions': '755',
            'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ wrong dependency parameter
        params = {
            'Dependency-Permissions': '400',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ invalid dependency parameter
        params = {
            'Dependency-Permissions': 'foo',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
        params = {
            'Dependency-Permissions': '888',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
Ejemplo n.º 3
0
    def test_check_mandatory_params(self):
        params = {'keyA': 'valueA', 'keyB': 'valueB', 'keyC': 'valueC'}

        # all mandatory headers are included
        StorletGatewayDocker._check_mandatory_params(params, ['keyA', 'keyB'])

        # some of mandatory headers are missing
        with self.assertRaises(ValueError):
            StorletGatewayDocker._check_mandatory_params(
                params, ['keyA', 'KeyD'])
Ejemplo n.º 4
0
    def test_check_mandatory_params(self):
        params = {'keyA': 'valueA',
                  'keyB': 'valueB',
                  'keyC': 'valueC'}

        # all mandatory headers are included
        StorletGatewayDocker._check_mandatory_params(
            params, ['keyA', 'keyB'])

        # some of mandatory headers are missing
        with self.assertRaises(ValueError):
            StorletGatewayDocker._check_mandatory_params(
                params, ['keyA', 'KeyD'])
Ejemplo n.º 5
0
    def test_validate_storlet_registration_java(self):
        # correct name and headers w/ dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # correct name and headers w/o dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # some header keys are missing
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 6
0
    def test_validate_storlet_registration_java(self):
        # correct name and headers w/ dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # correct name and headers w/o dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # some header keys are missing
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 7
0
    def setUp(self):
        if six.PY3:
            self.skipTest(
                "swift.common.internal_client.InternalClient for testing"
                "doesn't suppoert py3 yet")
        # TODO(takashi): take these values from config file
        self.tempdir = mkdtemp()
        self.sconf = {
            'host_root': self.tempdir,
            'swift_dir': self.tempdir,
            'storlet_timeout': '9',
            'storlet_container': 'storlet',
            'storlet_dependency': 'dependency',
            'reseller_prefix': 'AUTH'
        }
        self.logger = FakeLogger()

        self.storlet_container = self.sconf['storlet_container']
        self.storlet_dependency = self.sconf['storlet_dependency']

        self.version = 'v1'
        self.account = 'AUTH_account'
        self.container = 'container'
        self.obj = 'object'
        self.sobj = 'storlet-1.0.jar'

        # TODO(kota_): should be 'storlet-internal-client.conf' actually
        ic_conf_path = os.path.join(self.tempdir, 'storlet-proxy-server.conf')
        with open(ic_conf_path, 'w') as f:
            f.write("""
[DEFAULT]
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server

[app:proxy-server]
use = egg:swift#proxy

[filter:cache]
use = egg:swift#memcache

[filter:proxy-logging]
use = egg:swift#proxy_logging

[filter:catch_errors]
use = egg:swift#catch_errors
""")

        self.gateway = StorletGatewayDocker(self.sconf, self.logger,
                                            self.account)
Ejemplo n.º 8
0
    def test_validate_storlet_registration_python(self):
        # correct name and headers w/ dependency
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.pyfoo'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong main class
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'another_storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.foo.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 9
0
    def test_validate_storlet_registration_not_suppoeted(self):
        # unsupported language
        obj = 'storlet.foo'
        params = {'Language': 'bar',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # same name for storlet and dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'storlet-1.0.jar',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # duplicated name in dependencies
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file,dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 10
0
    def test_validate_storlet_registration_not_suppoeted(self):
        # unsupported language
        obj = 'storlet.foo'
        params = {'Language': 'bar',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # same name for storlet and dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'storlet-1.0.jar',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # duplicated name in dependencies
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file,dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 11
0
    def setUp(self):
        if six.PY3:
            self.skipTest(
                "swift.common.internal_client.InternalClient for testing"
                "doesn't suppoert py3 yet")
        # TODO(takashi): take these values from config file
        self.tempdir = mkdtemp()
        self.sconf = {
            'host_root': self.tempdir,
            'swift_dir': self.tempdir,
            'storlet_timeout': '9',
            'storlet_container': 'storlet',
            'storlet_dependency': 'dependency',
            'reseller_prefix': 'AUTH'
        }
        self.logger = FakeLogger()

        self.storlet_container = self.sconf['storlet_container']
        self.storlet_dependency = self.sconf['storlet_dependency']

        self.version = 'v1'
        self.account = 'AUTH_account'
        self.container = 'container'
        self.obj = 'object'
        self.sobj = 'storlet-1.0.jar'

        # TODO(kota_): shoudl be 'storlet-internal-client.conf' actually
        ic_conf_path = os.path.join(self.tempdir,
                                    'storlet-proxy-server.conf')
        with open(ic_conf_path, 'w') as f:
            f.write("""
[DEFAULT]
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server

[app:proxy-server]
use = egg:swift#proxy

[filter:cache]
use = egg:swift#memcache

[filter:proxy-logging]
use = egg:swift#proxy_logging

[filter:catch_errors]
use = egg:swift#catch_errors
""")

        self.gateway = StorletGatewayDocker(
            self.sconf, self.logger, self.account)
Ejemplo n.º 12
0
    def test_validate_storlet_registration_python(self):
        # correct name and headers w/ dependency
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Language-Version': '2.7',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong version
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Language-Version': '1.7',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.pyfoo'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong main class
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'another_storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.foo.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)
Ejemplo n.º 13
0
class TestStorletDockerGateway(unittest.TestCase):

    def setUp(self):
        # TODO(takashi): take these values from config file
        self.tempdir = mkdtemp()
        self.sconf = {
            'host_root': self.tempdir,
            'swift_dir': self.tempdir,
            'storlet_timeout': '9',
            'storlet_container': 'storlet',
            'storlet_dependency': 'dependency',
            'reseller_prefix': 'AUTH'
        }
        self.logger = FakeLogger()

        self.storlet_container = self.sconf['storlet_container']
        self.storlet_dependency = self.sconf['storlet_dependency']

        self.version = 'v1'
        self.account = 'AUTH_account'
        self.container = 'container'
        self.obj = 'object'
        self.sobj = 'storlet-1.0.jar'

        # TODO(kota_): should be 'storlet-internal-client.conf' actually
        ic_conf_path = os.path.join(self.tempdir,
                                    'storlet-proxy-server.conf')
        with open(ic_conf_path, 'wb') as f:
            f.write(b"""
[DEFAULT]
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server

[app:proxy-server]
use = egg:swift#proxy

[filter:cache]
use = egg:swift#memcache

[filter:proxy-logging]
use = egg:swift#proxy_logging

[filter:catch_errors]
use = egg:swift#catch_errors
""")

        self.gateway = StorletGatewayDocker(
            self.sconf, self.logger, self.account)

    def tearDown(self):
        rmtree(self.tempdir)

    @property
    def req_path(self):
        return self._create_proxy_path(
            self.version, self.account, self.container,
            self.obj)

    @property
    def storlet_path(self):
        return self._create_proxy_path(
            self.version, self.account, self.storlet_container,
            self.sobj)

    def _create_proxy_path(self, version, account, container, obj):
        return '/'.join(['', version, account, container, obj])

    def test_check_mandatory_params(self):
        params = {'keyA': 'valueA',
                  'keyB': 'valueB',
                  'keyC': 'valueC'}

        # all mandatory headers are included
        StorletGatewayDocker._check_mandatory_params(
            params, ['keyA', 'keyB'])

        # some of mandatory headers are missing
        with self.assertRaises(ValueError):
            StorletGatewayDocker._check_mandatory_params(
                params, ['keyA', 'KeyD'])

    def test_validate_storlet_registration_java(self):
        # correct name and headers w/ dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # correct name and headers w/o dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # some header keys are missing
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_storlet_registration_python(self):
        # correct name and headers w/ dependency
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Language-Version': '2.7',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong version
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Language-Version': '1.7',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.pyfoo'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong main class
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'another_storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.foo.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_storlet_registration_not_suppoeted(self):
        # unsupported language
        obj = 'storlet.foo'
        params = {'Language': 'bar',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # same name for storlet and dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'storlet-1.0.jar',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # duplicated name in dependencies
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file,dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_dependency_registration(self):
        # w/o dependency parameter
        obj = 'dep_file'
        params = {'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ correct dependency parameter
        params = {
            'Dependency-Permissions': '755',
            'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ wrong dependency parameter
        params = {
            'Dependency-Permissions': '400',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ invalid dependency parameter
        params = {
            'Dependency-Permissions': 'foo',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
        params = {
            'Dependency-Permissions': '888',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

    def _test_docker_gateway_communicate(self, extra_sources=None):
        extra_sources = extra_sources or []
        options = {'generate_log': False,
                   'scope': 'AUTH_account',
                   'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}

        st_req = DockerStorletRequest(
            storlet_id=self.sobj,
            params={},
            user_metadata={},
            data_iter=iter('body'), options=options)

        # TODO(kota_): need more efficient way for emuration of return value
        # from SDaemon
        value_generator = iter([
            # first, we get metadata json
            json.dumps({'metadata': 'return'}),
            # then we get object data
            'something', '',
        ])

        def mock_read(fd, size):
            try:
                value = next(value_generator)
            except StopIteration:
                raise Exception('called more then expected')
            # NOTE(takashi): Make sure that we return bytes in PY3
            return value.encode('utf-8')

        def mock_close(fd):
            pass

        called_fd_and_bodies = []
        invocation_protocol = \
            'storlets.gateway.gateways.docker.runtime.' \
            'StorletInvocationProtocol._write_input_data'

        def mock_writer(self, fd, app_iter):
            body = ''
            for chunk in app_iter:
                body += chunk
            called_fd_and_bodies.append((fd, body))

        # prepare nested mock patch
        # SBus -> mock SBus.send() for container communication
        # os.read -> mock reading the file descriptor from container
        # select.slect -> mock fd communication which can be readable
        @mock.patch('storlets.gateway.gateways.docker.runtime.SBusClient')
        @mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
                    mock_read)
        @mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
                    mock_close)
        @mock.patch('storlets.gateway.gateways.docker.runtime.select.select',
                    lambda r, w, x, timeout=None: (r, w, x))
        @mock.patch('storlets.gateway.common.stob.os.read', mock_read)
        @mock.patch(invocation_protocol, mock_writer)
        def test_invocation_flow(client):
            client.ping.return_value = SBusResponse(True, 'OK')
            client.stop_daemon.return_value = SBusResponse(True, 'OK')
            client.start_daemon.return_value = SBusResponse(True, 'OK')
            client.execute.return_value = SBusResponse(True, 'OK', 'someid')

            sresp = self.gateway.invocation_flow(st_req, extra_sources)
            eventlet.sleep(0.1)
            file_like = FileLikeIter(sresp.data_iter)
            self.assertEqual(b'something', file_like.read())

        # I hate the decorator to return an instance but to track current
        # implementation, we have to make a mock class for this. Need to fix.

        class MockFileManager(object):
            def get_storlet(self, req):
                return BytesIO(b'mock'), None

            def get_dependency(self, req):
                return BytesIO(b'mock'), None

        st_req.file_manager = MockFileManager()

        test_invocation_flow()

        # ensure st_req.app_iter is drawn
        self.assertRaises(StopIteration, next, st_req.data_iter)
        expected_mock_writer_calls = len(extra_sources) + 1
        self.assertEqual(expected_mock_writer_calls,
                         len(called_fd_and_bodies))
        self.assertEqual('body', called_fd_and_bodies[0][1])
        return called_fd_and_bodies

    def test_docker_gateway_communicate(self):
        self._test_docker_gateway_communicate()

    def test_docker_gateway_communicate_with_extra_sources(self):
        options = {'generate_log': False,
                   'scope': 'AUTH_account',
                   'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}

        data_sources = []

        def generate_extra_st_request():
            # This works similarly with build_storlet_request
            # TODO(kota_): think of more generarl way w/o
            # build_storlet_request
            sw_req = Request.blank(
                self.req_path, environ={'REQUEST_METHOD': 'GET'},
                headers={'X-Run-Storlet': self.sobj})

            sw_resp = Response(
                app_iter=iter(['This is a response body']), status=200)

            st_req = DockerStorletRequest(
                storlet_id=sw_req.headers['X-Run-Storlet'],
                params=sw_req.params,
                user_metadata={},
                data_iter=sw_resp.app_iter, options=options)
            data_sources.append(sw_resp.app_iter)
            return st_req

        extra_request = generate_extra_st_request()
        mock_calls = self._test_docker_gateway_communicate(
            extra_sources=[extra_request])
        self.assertEqual('This is a response body', mock_calls[1][1])

        # run all existing eventlet threads
        for app_iter in data_sources:
            # ensure all app_iters are drawn
            self.assertRaises(StopIteration, next, app_iter)
Ejemplo n.º 14
0
class TestStorletDockerGateway(unittest.TestCase):

    def setUp(self):
        if six.PY3:
            self.skipTest(
                "swift.common.internal_client.InternalClient for testing"
                "doesn't suppoert py3 yet")
        # TODO(takashi): take these values from config file
        self.tempdir = mkdtemp()
        self.sconf = {
            'host_root': self.tempdir,
            'swift_dir': self.tempdir,
            'storlet_timeout': '9',
            'storlet_container': 'storlet',
            'storlet_dependency': 'dependency',
            'reseller_prefix': 'AUTH'
        }
        self.logger = FakeLogger()

        self.storlet_container = self.sconf['storlet_container']
        self.storlet_dependency = self.sconf['storlet_dependency']

        self.version = 'v1'
        self.account = 'AUTH_account'
        self.container = 'container'
        self.obj = 'object'
        self.sobj = 'storlet-1.0.jar'

        # TODO(kota_): shoudl be 'storlet-internal-client.conf' actually
        ic_conf_path = os.path.join(self.tempdir,
                                    'storlet-proxy-server.conf')
        with open(ic_conf_path, 'w') as f:
            f.write("""
[DEFAULT]
[pipeline:main]
pipeline = catch_errors proxy-logging cache proxy-server

[app:proxy-server]
use = egg:swift#proxy

[filter:cache]
use = egg:swift#memcache

[filter:proxy-logging]
use = egg:swift#proxy_logging

[filter:catch_errors]
use = egg:swift#catch_errors
""")

        self.gateway = StorletGatewayDocker(
            self.sconf, self.logger, self.account)

    def tearDown(self):
        rmtree(self.tempdir)

    @property
    def req_path(self):
        return self._create_proxy_path(
            self.version, self.account, self.container,
            self.obj)

    @property
    def storlet_path(self):
        return self._create_proxy_path(
            self.version, self.account, self.storlet_container,
            self.sobj)

    def _create_proxy_path(self, version, account, container, obj):
        return '/'.join(['', version, account, container, obj])

    def test_check_mandatory_params(self):
        params = {'keyA': 'valueA',
                  'keyB': 'valueB',
                  'keyC': 'valueC'}

        # all mandatory headers are included
        StorletGatewayDocker._check_mandatory_params(
            params, ['keyA', 'keyB'])

        # some of mandatory headers are missing
        with self.assertRaises(ValueError):
            StorletGatewayDocker._check_mandatory_params(
                params, ['keyA', 'KeyD'])

    def test_validate_storlet_registration_java(self):
        # correct name and headers w/ dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # correct name and headers w/o dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # some header keys are missing
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_storlet_registration_python(self):
        # correct name and headers w/ dependency
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong name
        obj = 'storlet.pyfoo'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # wrong main class
        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'another_storlet.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        obj = 'storlet.py'
        params = {'Language': 'python',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'storlet.foo.Storlet'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_storlet_registration_not_suppoeted(self):
        # unsupported language
        obj = 'storlet.foo'
        params = {'Language': 'bar',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # same name for storlet and dependency
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'storlet-1.0.jar',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

        # duplicated name in dependencies
        obj = 'storlet-1.0.jar'
        params = {'Language': 'java',
                  'Interface-Version': '1.0',
                  'Dependency': 'dep_file,dep_file',
                  'Object-Metadata': 'no',
                  'Main': 'path.to.storlet.class'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_storlet_registration(params, obj)

    def test_validate_dependency_registration(self):
        # w/o dependency parameter
        obj = 'dep_file'
        params = {'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ correct dependency parameter
        params = {
            'Dependency-Permissions': '755',
            'Dependency-Version': '1.0'}
        StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ wrong dependency parameter
        params = {
            'Dependency-Permissions': '400',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

        # w/ invalid dependency parameter
        params = {
            'Dependency-Permissions': 'foo',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)
        params = {
            'Dependency-Permissions': '888',
            'Dependency-Version': '1.0'}
        with self.assertRaises(ValueError):
            StorletGatewayDocker.validate_dependency_registration(params, obj)

    def _test_docker_gateway_communicate(self, extra_sources=None):
        extra_sources = extra_sources or []
        options = {'generate_log': False,
                   'scope': 'AUTH_account',
                   'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}

        st_req = DockerStorletRequest(
            storlet_id=self.sobj,
            params={},
            user_metadata={},
            data_iter=iter('body'), options=options)

        # TODO(kota_): need more efficient way for emuration of return value
        # from SDaemon
        value_generator = iter([
            # Firt is for confirmation for SDaemon running
            'True: daemon running confirmation',
            # Second is stop SDaemon in activation
            'True: stop daemon',
            # Third is start SDaemon again in activation
            'True: start daemon',
            # Forth is return value for invoking as task_id
            'This is task id',
            # Fifth is for getting meta
            json.dumps({'metadata': 'return'}),
            # At last return body and EOF
            'something', '',
        ])

        def mock_read(fd, size):
            try:
                value = next(value_generator)
            except StopIteration:
                raise Exception('called more then expected')
            return value

        def mock_close(fd):
            pass

        called_fd_and_bodies = []
        invocation_protocol = \
            'storlets.gateway.gateways.docker.runtime.' \
            'StorletInvocationProtocol._write_input_data'

        def mock_writer(self, fd, app_iter):
            body = ''
            for chunk in app_iter:
                body += chunk
            called_fd_and_bodies.append((fd, body))

        # prepare nested mock patch
        # SBus -> mock SBus.send() for container communication
        # os.read -> mock reading the file descriptor from container
        # select.slect -> mock fd communication wich can be readable
        @mock.patch('storlets.gateway.gateways.docker.runtime.SBus', MockSBus)
        @mock.patch('storlets.gateway.gateways.docker.runtime.os.read',
                    mock_read)
        @mock.patch('storlets.gateway.gateways.docker.runtime.os.close',
                    mock_close)
        @mock.patch('storlets.gateway.gateways.docker.runtime.select.select',
                    lambda r, w, x, timeout=None: (r, w, x))
        @mock.patch('storlets.gateway.common.stob.os.read',
                    mock_read)
        @mock.patch(invocation_protocol, mock_writer)
        def test_invocation_flow():
            sresp = self.gateway.invocation_flow(st_req, extra_sources)
            eventlet.sleep(0.1)
            file_like = FileLikeIter(sresp.data_iter)
            self.assertEqual('something', file_like.read())

        # I hate the decorator to return an instance but to track current
        # implementation, we have to make a mock class for this. Need to fix.

        class MockFileManager(object):
            def get_storlet(self, req):
                return StringIO('mock'), None

            def get_dependency(self, req):
                return StringIO('mock'), None

        st_req.file_manager = MockFileManager()

        test_invocation_flow()

        # ensure st_req.app_iter is drawn
        self.assertRaises(StopIteration, next, st_req.data_iter)
        expected_mock_writer_calls = len(extra_sources) + 1
        self.assertEqual(expected_mock_writer_calls,
                         len(called_fd_and_bodies))
        self.assertEqual(called_fd_and_bodies[0][1], 'body')
        return called_fd_and_bodies

    def test_docker_gateway_communicate(self):
        self._test_docker_gateway_communicate()

    def test_docker_gateway_communicate_with_extra_sources(self):
        options = {'generate_log': False,
                   'scope': 'AUTH_account',
                   'storlet_main': 'org.openstack.storlet.Storlet',
                   'storlet_dependency': 'dep1,dep2',
                   'storlet_language': 'java',
                   'file_manager': FakeFileManager('storlet', 'dep')}

        data_sources = []

        def generate_extra_st_request():
            # This works similarly with build_storlet_request
            # TODO(kota_): think of more generarl way w/o
            # build_storlet_request
            sw_req = Request.blank(
                self.req_path, environ={'REQUEST_METHOD': 'GET'},
                headers={'X-Run-Storlet': self.sobj})

            sw_resp = Response(
                app_iter=iter(['This is a response body']), status=200)

            st_req = DockerStorletRequest(
                storlet_id=sw_req.headers['X-Run-Storlet'],
                params=sw_req.params,
                user_metadata={},
                data_iter=sw_resp.app_iter, options=options)
            data_sources.append(sw_resp.app_iter)
            return st_req

        extra_request = generate_extra_st_request()
        mock_calls = self._test_docker_gateway_communicate(
            extra_sources=[extra_request])
        self.assertEqual(mock_calls[1][1], 'This is a response body')

        # run all existing eventlet threads
        for app_iter in data_sources:
            # ensure all app_iters are drawn
            self.assertRaises(StopIteration, next, app_iter)