def test_download_proxy(self): import urllib.request # first test no proxy args = set_hw_parser().parse_args([]) opener = urllib.request.build_opener() if args.download_proxy: proxy = urllib.request.ProxyHandler({ 'http': args.download_proxy, 'https': args.download_proxy }) opener.add_handler(proxy) urllib.request.install_opener(opener) # head check req = urllib.request.Request(args.index_data_url, method="HEAD") response = urllib.request.urlopen(req, timeout=5) self.assertEqual(response.status, 200) # test with proxy args = set_hw_parser().parse_args( ["--download-proxy", os.getenv("HTTP_PROXY")]) opener = urllib.request.build_opener() if args.download_proxy: proxy = urllib.request.ProxyHandler({ 'http': args.download_proxy, 'https': args.download_proxy }) opener.add_handler(proxy) urllib.request.install_opener(opener) # head check req = urllib.request.Request(args.index_data_url, method="HEAD") response = urllib.request.urlopen(req, timeout=5) self.assertEqual(response.status, 200)
def test_helloworld_flow(self): args = set_hw_parser().parse_args([]) os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources') os.environ['SHARDS'] = str(args.shards) os.environ['REPLICAS'] = str(args.replicas) os.environ['HW_WORKDIR'] = args.workdir os.environ['WITH_LOGSERVER'] = str(args.logserver) f = Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml')))) targets = { 'index': { 'url': args.index_data_url, 'filename': os.path.join(args.workdir, 'index-original') }, 'query': { 'url': args.query_data_url, 'filename': os.path.join(args.workdir, 'query-original') } } # download the data Path(args.workdir).mkdir(parents=True, exist_ok=True) download_data(targets) # run it! with f: py_client(host=f.host, port_grpc=f.port_grpc, ).index(input_fn(targets['index']['filename']), batch_size=args.index_batch_size)
def _update_autocomplete(): from jina.main.parser import get_main_parser, set_pea_parser, \ set_hw_parser, set_flow_parser, set_pod_parser, \ set_check_parser, set_gateway_parser, set_ping_parser, set_client_cli_parser, set_logger_parser def _gaa(parser): _compl = [] for v in parser._actions: if v.option_strings: _compl.extend(v.option_strings) elif v.choices: _compl.extend(v.choices) # filer out single dash, as they serve as abbrev _compl = [k for k in _compl if (not k.startswith('-') or k.startswith('--'))] return _compl compl = { 'commands': _gaa(get_main_parser()), 'completions': { 'pea': _gaa(set_pea_parser()), 'hello-world': _gaa(set_hw_parser()), 'flow': _gaa(set_flow_parser()), 'pod': _gaa(set_pod_parser()), 'check': _gaa(set_check_parser()), 'gateway': _gaa(set_gateway_parser()), 'ping': _gaa(set_ping_parser()), 'client': _gaa(set_client_cli_parser()), 'log': _gaa(set_logger_parser()) } } with open(__file__, 'a') as fp: fp.write(f'\nac_table = {compl}\n')
def test_helloworld_flow(self): args = set_hw_parser().parse_args([]) os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources') os.environ['SHARDS'] = str(args.shards) os.environ['REPLICAS'] = str(args.replicas) os.environ['HW_WORKDIR'] = args.workdir os.environ['WITH_LOGSERVER'] = str(args.logserver) a = Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml')))) a.build() for p in a._pod_nodes.values(): print(f'{p.name}, {p.needs}')
def test_helloworld_flow_dry_run(self): args = set_hw_parser().parse_args([]) os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources') os.environ['SHARDS'] = str(args.shards) os.environ['PARALLEL'] = str(args.parallel) os.environ['HW_WORKDIR'] = args.workdir os.environ['WITH_LOGSERVER'] = str(args.logserver) # run it! with Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml')))): pass # run it! with Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.query.yml')))): pass
'index': { 'url': args.index_data_url, 'filename': os.path.join(args.workdir, 'index-original') }, 'query': { 'url': args.query_data_url, 'filename': os.path.join(args.workdir, 'query-original') } } # download the data download_data(targets, args.download_proxy) # run it! py_client(port_expose=args.port_expose, host=args.host).index(input_numpy(targets['index']['data']), batch_size=args.index_batch_size) if __name__ == '__main__': p = set_hw_parser() p.add_argument('--port-expose', required=True, type=int, help='the grpc port of the hello-world server') p.add_argument('--host', type=str, default='localhost', help='the address hello-world server') hello_world(p.parse_args())
def test_helloworld_py(self): from jina.main.parser import set_hw_parser from jina.helloworld import hello_world hello_world(set_hw_parser().parse_args([]))
from pkg_resources import resource_filename from jina.helloworld.components import * def hello_world(args): # this envs are referred in index and query flow YAMLs os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources') os.environ['SHARDS'] = str(args.shards) os.environ['REPLICAS'] = str(args.replicas) os.environ['HW_WORKDIR'] = args.workdir os.environ['WITH_LOGSERVER'] = str(args.logserver) # reduce the network load by using `fp16`, or even `uint8` os.environ['JINA_ARRAY_QUANT'] = 'fp16' # now comes the real work # load index flow from a YAML file f = Flow.load_config(args.index_yaml_path) # run it! with f: default_logger.success( f'hello-world server is started at {f.host}:{f.port_expose}, ' f'you can now use "python client.py --port-expose {f.port_expose} --host {f.host}" to send request!' ) f.block() if __name__ == '__main__': hello_world(set_hw_parser().parse_args())