Example #1
0
    def test_pymode(self):
        args = set_frontend_parser().parse_args([])

        p_args = set_preprocessor_parser().parse_args([
            '--port_in', str(args.port_out),
            '--port_out', '5531',
            '--socket_in', str(SocketType.PULL_CONNECT),
            '--socket_out', str(SocketType.PUSH_BIND),
            '--yaml_path', '!UnaryPreprocessor {parameters: {doc_type: 1}}'
        ])

        e_args = set_encoder_parser().parse_args([
            '--port_in', str(p_args.port_out),
            '--port_out', str(args.port_in),
            '--socket_in', str(SocketType.PULL_CONNECT),
            '--socket_out', str(SocketType.PUSH_CONNECT),
            '--yaml_path', 'flair.yml',
        ])

        with ServiceManager(EncoderService, e_args), \
             ServiceManager(PreprocessorService, p_args), \
             FrontendService(args), \
             grpc.insecure_channel('%s:%s' % (args.grpc_host, args.grpc_port),
                                   options=[('grpc.max_send_message_length', 70 * 1024 * 1024),
                                            ('grpc.max_receive_message_length', 70 * 1024 * 1024)]) as channel:
            stub = gnes_pb2_grpc.GnesRPCStub(channel)
            resp = stub.Call(list(RequestGenerator.index([b'hello world', b'goodbye!'], 1))[0])
            self.assertEqual(resp.request_id, '0')
Example #2
0
    def test_singleton_preprocessor_service_realdata(self):
        args = set_preprocessor_service_parser().parse_args(
            ['--yaml_path', self.singleton_img_pre_yaml])
        c_args = _set_client_parser().parse_args(
            ['--port_in',
             str(args.port_out), '--port_out',
             str(args.port_in)])
        all_zips = zipfile.ZipFile(os.path.join(self.dirname, 'imgs/test.zip'))
        all_bytes = [all_zips.open(v).read() for v in all_zips.namelist()]

        with PreprocessorService(args), ZmqClient(c_args) as client:
            for req in RequestGenerator.index(all_bytes):
                msg = gnes_pb2.Message()
                msg.request.index.CopyFrom(req.index)
                client.send_message(msg)
                r = client.recv_message()
                self.assertEqual(
                    r.envelope.routes[0].service,
                    'PreprocessorService:BaseSingletonPreprocessor')
                for d in r.request.index.docs:
                    self.assertEqual(len(d.chunks), 1)
                    self.assertEqual(len(blob2array(d.chunks[0].blob).shape),
                                     3)
                    self.assertEqual(blob2array(d.chunks[0].blob).shape[-1], 3)
                    self.assertEqual(
                        blob2array(d.chunks[0].blob).shape[0], 224)
                    self.assertEqual(
                        blob2array(d.chunks[0].blob).shape[1], 224)
                    print(blob2array(d.chunks[0].blob).dtype)
Example #3
0
    def test_pymode(self):
        args = set_frontend_parser().parse_args([
            '--socket_in',
            str(SocketType.PULL_BIND),
            '--socket_out',
            str(SocketType.PUSH_BIND),
        ])

        p_args = set_preprocessor_parser().parse_args([
            '--port_in',
            str(args.port_out), '--port_out',
            str(args.port_in), '--socket_in',
            str(SocketType.PULL_CONNECT), '--socket_out',
            str(SocketType.PUSH_CONNECT), '--yaml_path', 'pipline.yml',
            '--py_path', 'mypreprocessor1.py', 'mypreprocessor2.py'
        ])

        with ServiceManager(PreprocessorService, p_args), \
             FrontendService(args), \
             grpc.insecure_channel('%s:%s' % (args.grpc_host, args.grpc_port),
                                   options=[('grpc.max_send_message_length', 70 * 1024 * 1024),
                                            ('grpc.max_receive_message_length', 70 * 1024 * 1024)]) as channel:
            stub = gnes_pb2_grpc.GnesRPCStub(channel)
            resp = stub.Call(
                list(RequestGenerator.index([b'doc1:', b'doc2:'], 1))[0])
            self.assertEqual(resp.request_id, '0')
Example #4
0
    def test_pymode(self):
        args = set_frontend_parser().parse_args([])

        p_args = set_preprocessor_parser().parse_args([
            '--port_in',
            str(args.port_out), '--port_out', '5531', '--socket_in',
            str(SocketType.PULL_CONNECT), '--socket_out',
            str(SocketType.PUSH_BIND), '--yaml_path', 'SentSplitPreprocessor'
        ])

        e_args = set_encoder_parser().parse_args([
            '--port_in',
            str(p_args.port_out), '--port_out',
            str(args.port_in), '--socket_in',
            str(SocketType.PULL_CONNECT), '--socket_out',
            str(SocketType.PUSH_CONNECT), '--yaml_path', 'transformer.yml',
            '--py_path', 'distilbert.py'
        ])

        with ServiceManager(EncoderService, e_args), \
             ServiceManager(PreprocessorService, p_args), \
             FrontendService(args), \
             grpc.insecure_channel('%s:%s' % (args.grpc_host, args.grpc_port),
                                   options=[('grpc.max_send_message_length', 70 * 1024 * 1024),
                                            ('grpc.max_receive_message_length', 70 * 1024 * 1024)]) as channel:
            stub = gnes_pb2_grpc.GnesRPCStub(channel)
            resp = stub.Call(
                list(
                    RequestGenerator.index([
                        b'hello world, good to see you.',
                        b'goodbye! wish to see you again!'
                    ], 1))[0])
            self.assertEqual(resp.request_id, 0)
Example #5
0
    def test_fasterrcnn_preprocessor(self):
        args = set_preprocessor_parser().parse_args(
            ['--yaml_path', self.fasterrcnn_yaml])
        c_args = _set_client_parser().parse_args(
            ['--port_in',
             str(args.port_out), '--port_out',
             str(args.port_in)])
        all_zips = zipfile.ZipFile(self.data_path)
        all_bytes = [all_zips.open(v).read() for v in all_zips.namelist()]

        with ServiceManager(PreprocessorService,
                            args), ZmqClient(c_args) as client:
            for req in RequestGenerator.index(all_bytes):
                msg = gnes_pb2.Message()
                msg.request.index.CopyFrom(req.index)
                client.send_message(msg)
                r = client.recv_message()
                for d in r.request.index.docs:
                    self.assertGreater(len(d.chunks), 0)
                    for _ in range(len(d.chunks)):
                        self.assertEqual(
                            len(blob2array(d.chunks[_].blob).shape), 3)
                        self.assertEqual(
                            blob2array(d.chunks[_].blob).shape[-1], 3)
                        self.assertEqual(
                            blob2array(d.chunks[_].blob).shape[0], 224)
                        self.assertEqual(
                            blob2array(d.chunks[_].blob).shape[1], 224)
                        print(blob2array(d.chunks[0].blob).dtype)
    def test_video_cut_by_clustering(self):
        args = set_preprocessor_parser().parse_args(
            ['--yaml_path', self.yml_path_4])
        c_args = _set_client_parser().parse_args(
            ['--port_in',
             str(args.port_out), '--port_out',
             str(args.port_in)])

        with PreprocessorService(args), ZmqClient(c_args) as client:
            for req in RequestGenerator.index(self.video_bytes):
                msg = gnes_pb2.Message()
                msg.request.index.CopyFrom(req.index)
                client.send_message(msg)
                r = client.recv_message()
                for d in r.request.index.docs:
                    self.assertEqual(len(d.chunks), 6)
Example #7
0
    def test_pymode(self):
        os.unsetenv('http_proxy')
        os.unsetenv('https_proxy')
        args = set_frontend_parser().parse_args([])

        p_args = set_preprocessor_parser().parse_args([
            '--port_in',
            str(args.port_out), '--port_out', '5531', '--socket_in',
            str(SocketType.PULL_CONNECT), '--socket_out',
            str(SocketType.PUSH_BIND), '--yaml_path', 'SentSplitPreprocessor'
        ])

        e_args = set_indexer_parser().parse_args([
            '--port_in',
            str(p_args.port_out),
            '--port_out',
            str(args.port_in),
            '--socket_in',
            str(SocketType.PULL_CONNECT),
            '--socket_out',
            str(SocketType.PUSH_CONNECT),
            '--yaml_path',
            '!DictIndexer {gnes_config: {name: dummy_dict_indexer}}',
        ])

        with ServiceManager(IndexerService, e_args), \
             ServiceManager(PreprocessorService, p_args), \
             FrontendService(args), \
             grpc.insecure_channel('%s:%s' % (args.grpc_host, args.grpc_port),
                                   options=[('grpc.max_send_message_length', 70 * 1024 * 1024),
                                            ('grpc.max_receive_message_length', 70 * 1024 * 1024)]) as channel:
            stub = gnes_pb2_grpc.GnesRPCStub(channel)
            all_bytes = []
            with open(os.path.join(self.dirname, '26-doc-chinese.txt'),
                      'r',
                      encoding='utf8') as fp:
                for v in fp:
                    if v.strip():
                        all_bytes.append(v.encode())
            for r in stub.StreamCall(RequestGenerator.index(all_bytes)):
                print(r)

        bi = BaseIndexer.load('dummy_dict_indexer.bin')
        self.assertEqual(bi.size, 26)
        print(bi.query([0]))
    def test_video_preprocessor_service_realdata(self):
        args = set_preprocessor_parser().parse_args(
            ['--yaml_path', self.yml_path])

        c_args = _set_client_parser().parse_args(
            ['--port_in',
             str(args.port_out), '--port_out',
             str(args.port_in)])

        with PreprocessorService(args), ZmqClient(c_args) as client:
            for req in RequestGenerator.index(self.video_bytes):
                msg = gnes_pb2.Message()
                msg.request.index.CopyFrom(req.index)
                client.send_message(msg)
                r = client.recv_message()
                for d in r.request.index.docs:
                    self.assertGreater(len(d.chunks), 0)
                    for _ in range(len(d.chunks)):
                        shape = blob2array(d.chunks[_].blob).shape
                        self.assertEqual(shape, (168, 192, 3))
Example #9
0
    def test_video_decode_preprocessor(self):
        args = set_preprocessor_parser().parse_args(['--yaml_path', self.yml_path])
        c_args = _set_client_parser().parse_args([
            '--port_in', str(args.port_out),
            '--port_out', str(args.port_in)])
        video_bytes = [
            open(os.path.join(self.video_path, _), 'rb').read()
            for _ in os.listdir(self.video_path)
        ]

        with ServiceManager(PreprocessorService, args), ZmqClient(c_args) as client:
            for req in RequestGenerator.index(video_bytes):
                msg = gnes_pb2.Message()
                msg.request.index.CopyFrom(req.index)
                client.send_message(msg)
                r = client.recv_message()
                for d in r.request.index.docs:
                    self.assertGreater(len(d.chunks), 0)
                    for _ in range(len(d.chunks)):
                        shape = blob2array(d.chunks[_].blob).shape
                        self.assertEqual(shape[1:], (299, 299, 3))