def test_cont_gateway(self):
f1_args = set_gateway_parser().parse_args(['--allow-spawn'])
f2_args = set_gateway_parser().parse_args([])
with GatewayPod(f1_args):
pass
with GatewayPod(f2_args):
pass
def test_index_remote(self):
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
def start_gateway():
with GatewayPod(f_args):
time.sleep(20)
t = mp.Process(target=start_gateway)
t.daemon = True
t.start()
f = Flow().add(uses=os.path.join(cur_dir,
'yaml/test-index-remote.yml'),
parallel=3,
separated_workspace=True,
host='localhost',
port_expose=f_args.port_expose)
with f:
f.index(input_fn=random_docs(1000))
time.sleep(3)
for j in range(3):
self.assertTrue(os.path.exists(f'test2-{j + 1}/test2.bin'))
self.assertTrue(os.path.exists(f'test2-{j + 1}/tmp2'))
self.add_tmpfile(f'test2-{j + 1}/test2.bin', f'test2-{j + 1}/tmp2',
f'test2-{j + 1}')
def test_index_remote(test_workspace):
f_args = set_gateway_parser().parse_args(['--host', '0.0.0.0'])
def start_gateway():
with GatewayPod(f_args):
time.sleep(20)
t = mp.Process(target=start_gateway)
t.daemon = True
t.start()
f = Flow().add(uses=os.path.join(cur_dir, 'yaml/test-index-remote.yml'),
parallel=3,
separated_workspace=True,
host='0.0.0.0',
port_expose=f_args.port_expose)
with f:
f.index(input_fn=random_docs_new_api(1000))
time.sleep(3)
for j in range(3):
bin_path = os.path.join(test_workspace, f'test2-{j + 1}/test2.bin')
index_filename_path = os.path.join(test_workspace,
f'test2-{j + 1}/tmp2')
assert os.path.exists(bin_path)
assert os.path.exists(index_filename_path)
def test_remote_not_allowed():
f_args = set_gateway_parser().parse_args([])
p_args = set_pea_parser().parse_args(
['--host', 'localhost', '--port-expose',
str(f_args.port_expose)])
with GatewayPod(f_args):
PeaSpawnHelper(p_args).start()
def test_remote_two_pea(self):
# NOTE: right now there is no way to spawn two peas with one gateway!!!
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
def start_gateway():
with GatewayPod(f_args):
time.sleep(5)
def start_client(d):
print('im running %d' % d)
p_args = set_pea_parser().parse_args(
['--host', 'localhost', '--name', 'testpea%d' % d, '--port-expose', str(f_args.port_expose)])
PeaSpawnHelper(p_args).start()
t = Process(target=start_gateway)
t.daemon = True
t.start()
time.sleep(1)
c1 = Process(target=start_client, args=(1,))
c2 = Process(target=start_client, args=(2,))
c1.daemon = True
c2.daemon = True
c1.start()
c2.start()
time.sleep(5)
c1.join()
c2.join()
def test_gateway_ready():
p = set_gateway_parser().parse_args([])
with RESTGatewayPea(p):
a = requests.get(f'http://0.0.0.0:{p.port_expose}/ready')
assert a.status_code == 200
with RESTGatewayPea(p):
a = requests.post(f'http://0.0.0.0:{p.port_expose}/api/ass')
assert a.status_code == 405
def test_customized_pod(self):
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
p_args = set_pod_parser().parse_args(
['--host', 'localhost', '--parallel', '3', '--port-expose', str(f_args.port_expose)])
p = BasePod(p_args)
def start_gateway():
with GatewayPod(f_args):
time.sleep(5)
t = Process(target=start_gateway)
t.daemon = True
t.start()
MutablePodSpawnHelper(p.peas_args).start()
def test_remote_pod_process(self):
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
p_args = set_pod_parser().parse_args(
['--host', 'localhost', '--parallel', '3',
'--port-expose', str(f_args.port_expose), '--runtime', 'process'])
def start_spawn():
PodSpawnHelper(p_args).start()
with GatewayPod(f_args):
t = Process(target=start_spawn)
t.daemon = True
t.start()
time.sleep(5)
def test_remote_pod2(self):
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
p_args = set_pea_parser().parse_args(['--host', 'localhost', '--port-expose', str(f_args.port_expose)])
def start_gateway():
with GatewayPod(f_args):
time.sleep(5)
t = Process(target=start_gateway)
t.daemon = True
t.start()
with RemotePod(p_args):
pass
t.join()
def test_index_remote_rpi(self):
f_args = set_gateway_parser().parse_args(['--allow-spawn'])
def start_gateway():
with GatewayPod(f_args):
time.sleep(50)
t = mp.Process(target=start_gateway)
t.daemon = True
t.start()
f = (Flow(optimize_level=FlowOptimizeLevel.IGNORE_GATEWAY)
.add(uses=os.path.join(cur_dir, 'yaml/test-index.yml'),
parallel=3, separated_workspace=True,
host='192.168.31.76', port_expose=44444))
with f:
f.index(input_fn=random_docs(1000))
def test_index_remote_rpi(test_workspace):
f_args = set_gateway_parser().parse_args(['--host', '0.0.0.0'])
def start_gateway():
with GatewayPod(f_args):
time.sleep(3)
t = mp.Process(target=start_gateway)
t.daemon = True
t.start()
f = Flow(optimize_level=FlowOptimizeLevel.IGNORE_GATEWAY).add(
uses=os.path.join(cur_dir, 'yaml/test-index-remote.yml'),
parallel=3,
separated_workspace=True,
host='0.0.0.0',
port_expose=random_port())
with f:
f.index(input_fn=random_docs_new_api(1000))
def test_gateway_runtime(runtime, pea_cls):
args = set_gateway_parser().parse_args(['--runtime', runtime])
with LocalRuntime(args, pea_cls=pea_cls):
pass
LocalRuntime(args, pea_cls=pea_cls).start().close()
def test_gateway_ready(port_expose, route, status_code):
p = set_gateway_parser().parse_args(['--port-expose', str(port_expose)])
with RESTGatewayPea(p):
a = requests.get(f'http://0.0.0.0:{p.port_expose}{route}')
assert a.status_code == status_code
def test_gateway_pea(runtime):
args = set_gateway_parser().parse_args(['--runtime', runtime])
with GatewayPea(args):
pass
GatewayPea(args).start().close()
def test_cont_gateway(args):
parsed_args = set_gateway_parser().parse_args(args)
with GatewayPod(parsed_args):
pass
def test_gateway_ready(port_expose, route, status_code):
p = set_gateway_parser().parse_args(['--port-expose', str(port_expose)])
with LocalRuntime(p, pea_cls=RESTGatewayPea):
time.sleep(0.5)
a = requests.get(f'http://0.0.0.0:{p.port_expose}{route}')
assert a.status_code == status_code