def test_download_proxy(self):
import urllib.request
# first test no proxy
args = set_hw_parser().parse_args([])
opener = urllib.request.build_opener()
if args.download_proxy:
proxy = urllib.request.ProxyHandler({
'http': args.download_proxy,
'https': args.download_proxy
})
opener.add_handler(proxy)
urllib.request.install_opener(opener)
# head check
req = urllib.request.Request(args.index_data_url, method="HEAD")
response = urllib.request.urlopen(req, timeout=5)
self.assertEqual(response.status, 200)
# test with proxy
args = set_hw_parser().parse_args(
["--download-proxy", os.getenv("HTTP_PROXY")])
opener = urllib.request.build_opener()
if args.download_proxy:
proxy = urllib.request.ProxyHandler({
'http': args.download_proxy,
'https': args.download_proxy
})
opener.add_handler(proxy)
urllib.request.install_opener(opener)
# head check
req = urllib.request.Request(args.index_data_url, method="HEAD")
response = urllib.request.urlopen(req, timeout=5)
self.assertEqual(response.status, 200)
def test_helloworld_flow(self):
args = set_hw_parser().parse_args([])
os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources')
os.environ['SHARDS'] = str(args.shards)
os.environ['REPLICAS'] = str(args.replicas)
os.environ['HW_WORKDIR'] = args.workdir
os.environ['WITH_LOGSERVER'] = str(args.logserver)
f = Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml'))))
targets = {
'index': {
'url': args.index_data_url,
'filename': os.path.join(args.workdir, 'index-original')
},
'query': {
'url': args.query_data_url,
'filename': os.path.join(args.workdir, 'query-original')
}
}
# download the data
Path(args.workdir).mkdir(parents=True, exist_ok=True)
download_data(targets)
# run it!
with f:
py_client(host=f.host,
port_grpc=f.port_grpc,
).index(input_fn(targets['index']['filename']), batch_size=args.index_batch_size)
def _update_autocomplete():
from jina.main.parser import get_main_parser, set_pea_parser, \
set_hw_parser, set_flow_parser, set_pod_parser, \
set_check_parser, set_gateway_parser, set_ping_parser, set_client_cli_parser, set_logger_parser
def _gaa(parser):
_compl = []
for v in parser._actions:
if v.option_strings:
_compl.extend(v.option_strings)
elif v.choices:
_compl.extend(v.choices)
# filer out single dash, as they serve as abbrev
_compl = [k for k in _compl if (not k.startswith('-') or k.startswith('--'))]
return _compl
compl = {
'commands': _gaa(get_main_parser()),
'completions': {
'pea': _gaa(set_pea_parser()),
'hello-world': _gaa(set_hw_parser()),
'flow': _gaa(set_flow_parser()),
'pod': _gaa(set_pod_parser()),
'check': _gaa(set_check_parser()),
'gateway': _gaa(set_gateway_parser()),
'ping': _gaa(set_ping_parser()),
'client': _gaa(set_client_cli_parser()),
'log': _gaa(set_logger_parser())
}
}
with open(__file__, 'a') as fp:
fp.write(f'\nac_table = {compl}\n')
def test_helloworld_flow(self):
args = set_hw_parser().parse_args([])
os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources')
os.environ['SHARDS'] = str(args.shards)
os.environ['REPLICAS'] = str(args.replicas)
os.environ['HW_WORKDIR'] = args.workdir
os.environ['WITH_LOGSERVER'] = str(args.logserver)
a = Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml'))))
a.build()
for p in a._pod_nodes.values():
print(f'{p.name}, {p.needs}')
def test_helloworld_flow_dry_run(self):
args = set_hw_parser().parse_args([])
os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources')
os.environ['SHARDS'] = str(args.shards)
os.environ['PARALLEL'] = str(args.parallel)
os.environ['HW_WORKDIR'] = args.workdir
os.environ['WITH_LOGSERVER'] = str(args.logserver)
# run it!
with Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.index.yml')))):
pass
# run it!
with Flow.load_config(resource_filename('jina', '/'.join(('resources', 'helloworld.flow.query.yml')))):
pass
'index': {
'url': args.index_data_url,
'filename': os.path.join(args.workdir, 'index-original')
},
'query': {
'url': args.query_data_url,
'filename': os.path.join(args.workdir, 'query-original')
}
}
# download the data
download_data(targets, args.download_proxy)
# run it!
py_client(port_expose=args.port_expose,
host=args.host).index(input_numpy(targets['index']['data']),
batch_size=args.index_batch_size)
if __name__ == '__main__':
p = set_hw_parser()
p.add_argument('--port-expose',
required=True,
type=int,
help='the grpc port of the hello-world server')
p.add_argument('--host',
type=str,
default='localhost',
help='the address hello-world server')
hello_world(p.parse_args())
def test_helloworld_py(self):
from jina.main.parser import set_hw_parser
from jina.helloworld import hello_world
hello_world(set_hw_parser().parse_args([]))
from pkg_resources import resource_filename
from jina.helloworld.components import *
def hello_world(args):
# this envs are referred in index and query flow YAMLs
os.environ['RESOURCE_DIR'] = resource_filename('jina', 'resources')
os.environ['SHARDS'] = str(args.shards)
os.environ['REPLICAS'] = str(args.replicas)
os.environ['HW_WORKDIR'] = args.workdir
os.environ['WITH_LOGSERVER'] = str(args.logserver)
# reduce the network load by using `fp16`, or even `uint8`
os.environ['JINA_ARRAY_QUANT'] = 'fp16'
# now comes the real work
# load index flow from a YAML file
f = Flow.load_config(args.index_yaml_path)
# run it!
with f:
default_logger.success(
f'hello-world server is started at {f.host}:{f.port_expose}, '
f'you can now use "python client.py --port-expose {f.port_expose} --host {f.host}" to send request!'
)
f.block()
if __name__ == '__main__':
hello_world(set_hw_parser().parse_args())
