def test_flow_to_k8s_yaml_sandbox(tmpdir):
flow = Flow(name='test-flow',
port=8080).add(uses=f'jinahub+sandbox://DummyHubExecutor')
dump_path = os.path.join(str(tmpdir), 'test_flow_k8s')
namespace = 'test-flow-ns'
flow.to_kubernetes_yaml(
output_base_path=dump_path,
k8s_namespace=namespace,
)
yaml_dicts_per_deployment = {
'gateway': [],
}
for pod_name in set(os.listdir(dump_path)):
file_set = set(os.listdir(os.path.join(dump_path, pod_name)))
for file in file_set:
with open(os.path.join(dump_path, pod_name, file)) as f:
yml_document_all = list(yaml.safe_load_all(f))
yaml_dicts_per_deployment[file[:-4]] = yml_document_all
gateway_objects = yaml_dicts_per_deployment['gateway']
gateway_args = gateway_objects[2]['spec']['template']['spec'][
'containers'][0]['args']
assert (gateway_args[gateway_args.index('--graph-description') + 1] ==
'{"executor0": ["end-gateway"], "start-gateway": ["executor0"]}')
assert '--deployments-addresses' in gateway_args
deployment_addresses = json.loads(
gateway_args[gateway_args.index('--deployments-addresses') + 1])
assert deployment_addresses['executor0'][0].startswith('grpcs://')
def test_flow_to_k8s_yaml_external_pod(tmpdir, has_external):
flow = Flow(name='test-flow', port=8080).add(name='executor0', )
if has_external:
flow = flow.add(name='external_executor',
external=True,
host='1.2.3.4',
port=9090)
else:
flow = flow.add(name='external_executor')
dump_path = os.path.join(str(tmpdir), 'test_flow_k8s')
namespace = 'test-flow-ns'
flow.to_kubernetes_yaml(
output_base_path=dump_path,
k8s_namespace=namespace,
)
yaml_dicts_per_deployment = {
'gateway': [],
'executor0': [],
}
assert len(set(os.listdir(dump_path))) == 2 if has_external else 3
for pod_name in set(os.listdir(dump_path)):
file_set = set(os.listdir(os.path.join(dump_path, pod_name)))
for file in file_set:
with open(os.path.join(dump_path, pod_name, file)) as f:
yml_document_all = list(yaml.safe_load_all(f))
yaml_dicts_per_deployment[file[:-4]] = yml_document_all
gateway_objects = yaml_dicts_per_deployment['gateway']
gateway_args = gateway_objects[2]['spec']['template']['spec'][
'containers'][0]['args']
assert (
gateway_args[gateway_args.index('--graph-description') + 1] ==
'{"executor0": ["external_executor"], "start-gateway": ["executor0"], "external_executor": ["end-gateway"]}'
)
if has_external:
assert '--deployments-addresses' in gateway_args
assert (
gateway_args[gateway_args.index('--deployments-addresses') + 1] ==
'{"executor0": ["grpc://executor0.test-flow-ns.svc:8080"], "external_executor": ["grpc://1.2.3.4:9090"]}'
)
else:
assert '--deployments-addresses' in gateway_args
assert (
gateway_args[gateway_args.index('--deployments-addresses') + 1] ==
'{"executor0": ["grpc://executor0.test-flow-ns.svc:8080"], "external_executor": ["grpc://external-executor.test-flow-ns.svc:8080"]}'
)
async def test_flow_with_monitoring(logger, tmpdir, docker_images,
port_generator):
dump_path = os.path.join(str(tmpdir), 'test-flow-with-monitoring')
namespace = f'test-flow-monitoring'.lower()
port1 = port_generator()
port2 = port_generator()
flow = Flow(name='test-flow-monitoring',
monitoring=True,
port_monitoring=port1).add(
name='segmenter',
uses=f'docker://{docker_images[0]}',
monitoring=True,
port_monitoring=port2,
)
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
deployment_replicas_expected={
'gateway': 1,
'segmenter': 1,
},
logger=logger,
)
import portforward
config_path = os.environ['KUBECONFIG']
gateway_pod_name = (core_client.list_namespaced_pod(
namespace=namespace,
label_selector='app=gateway').items[0].metadata.name)
pod_port_ref = [(gateway_pod_name, port1)]
for (pod_name, port) in pod_port_ref:
with portforward.forward(namespace, pod_name, port, port, config_path):
resp = req.get(f'http://localhost:{port}/')
assert resp.status_code == 200
core_client.delete_namespace(namespace)
async def test_flow_with_external_native_deployment(logger, docker_images,
tmpdir):
class DocReplaceExecutor(Executor):
@requests
def add(self, **kwargs):
return DocumentArray(
[Document(text='executor was here') for _ in range(100)])
args = set_deployment_parser().parse_args(['--uses', 'DocReplaceExecutor'])
with Deployment(args) as external_deployment:
ports = [args.port for args in external_deployment.pod_args['pods'][0]]
flow = Flow(name='k8s_flow-with_external_deployment', port=9090).add(
name='external_executor',
external=True,
host=f'172.17.0.1',
port=ports[0],
)
namespace = 'test-flow-with-external-deployment'
dump_path = os.path.join(str(tmpdir), namespace)
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
deployment_replicas_expected={
'gateway': 1,
},
logger=logger,
)
resp = await run_test(
flow=flow,
namespace=namespace,
core_client=core_client,
endpoint='/',
)
docs = resp[0].docs
assert len(docs) == 100
for doc in docs:
assert doc.text == 'executor was here'
core_client.delete_namespace(namespace)
async def test_flow_with_external_k8s_deployment(logger, docker_images,
tmpdir):
namespace = 'test-flow-with-external-k8s-deployment'
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await _create_external_deployment(api_client, app_client, docker_images,
tmpdir)
flow = Flow(name='k8s_flow-with_external_deployment', port=9090).add(
name='external_executor',
external=True,
host='external-deployment.external-deployment-ns.svc',
port=GrpcConnectionPool.K8S_PORT,
)
dump_path = os.path.join(str(tmpdir), namespace)
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
deployment_replicas_expected={
'gateway': 1,
},
logger=logger,
)
resp = await run_test(
flow=flow,
namespace=namespace,
core_client=core_client,
endpoint='/workspace',
)
docs = resp[0].docs
for doc in docs:
assert 'workspace' in doc.tags
async def test_flow_with_workspace(logger, docker_images, tmpdir):
flow = Flow(name='k8s_flow-with_workspace', port=9090,
protocol='http').add(
name='test_executor',
uses=f'docker://{docker_images[0]}',
workspace='/shared',
)
dump_path = os.path.join(str(tmpdir), 'test-flow-with-workspace')
namespace = f'test-flow-with-workspace'.lower()
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
deployment_replicas_expected={
'gateway': 1,
'test-executor': 1,
},
logger=logger,
)
resp = await run_test(
flow=flow,
namespace=namespace,
core_client=core_client,
endpoint='/workspace',
)
docs = resp[0].docs
assert len(docs) == 10
for doc in docs:
assert doc.tags['workspace'] == '/shared/TestExecutor/0'
core_client.delete_namespace(namespace)
def test_raise_exception_invalid_executor(tmpdir):
from jina.excepts import NoContainerizedError
with pytest.raises(NoContainerizedError):
f = Flow().add(uses='A')
f.to_kubernetes_yaml(str(tmpdir))
async def test_failure_scenarios(logger, docker_images, tmpdir, k8s_cluster):
namespace = 'test-failure-scenarios'
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
flow = Flow(prefetch=100).add(replicas=3,
uses=f'docker://{docker_images[0]}')
dump_path = os.path.join(str(tmpdir), namespace)
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
deployment_replicas_expected={
'gateway': 1,
'executor0': 3,
},
logger=logger,
)
stop_event = asyncio.Event()
send_task = asyncio.create_task(
run_test_until_event(
flow=flow,
namespace=namespace,
core_client=core_client,
endpoint='/',
stop_event=stop_event,
logger=logger,
sleep_time=None,
))
await asyncio.sleep(5.0)
# Scale down the Executor to 2 replicas
await scale(
deployment_name='executor0',
desired_replicas=2,
core_client=core_client,
app_client=app_client,
k8s_namespace=namespace,
logger=logger,
)
# Scale back up to 3 replicas
await scale(
deployment_name='executor0',
desired_replicas=3,
core_client=core_client,
app_client=app_client,
k8s_namespace=namespace,
logger=logger,
)
await asyncio.sleep(5.0)
# restart all pods in the deployment
await restart_deployment(
deployment='executor0',
app_client=app_client,
core_client=core_client,
k8s_namespace=namespace,
logger=logger,
)
await asyncio.sleep(5.0)
await delete_pod(
deployment='executor0',
core_client=core_client,
k8s_namespace=namespace,
logger=logger,
)
await asyncio.sleep(5.0)
stop_event.set()
responses, sent_ids = await send_task
assert len(sent_ids) == len(responses)
doc_ids = set()
pod_ids = set()
for response in responses:
doc_id, pod_id = response.docs.texts[0].split('_')
doc_ids.add(doc_id)
pod_ids.add(pod_id)
assert len(sent_ids) == len(doc_ids)
assert len(
pod_ids) == 8 # 3 original + 3 restarted + 1 scaled up + 1 deleted
# do the random failure test
# start sending again
logger.info('Start sending for random failure test')
stop_event.clear()
send_task = asyncio.create_task(
run_test_until_event(
flow=flow,
namespace=namespace,
core_client=core_client,
endpoint='/',
stop_event=stop_event,
logger=logger,
))
# inject failures
inject_failures(k8s_cluster, logger)
# wait a bit
await asyncio.sleep(3.0)
# check that no message was lost
stop_event.set()
responses, sent_ids = await send_task
assert len(sent_ids) == len(responses)
async def test_linear_processing_time_scaling(docker_images, logger, tmpdir):
flow = Flow(name='test-flow-slow-process-executor',).add(
name='slow_process_executor',
uses=f'docker://{docker_images[0]}',
replicas=3,
)
dump_path = os.path.join(str(tmpdir), 'test_flow_k8s')
namespace = 'test-flow-slow-process-executor-ns-3'
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
)
# start port forwarding
logger.debug(f' Start port forwarding')
gateway_pod_name = (
core_client.list_namespaced_pod(
namespace=namespace, label_selector='app=gateway'
)
.items[0]
.metadata.name
)
config_path = os.environ['KUBECONFIG']
import portforward
with portforward.forward(
namespace, gateway_pod_name, flow.port, flow.port, config_path
):
time.sleep(0.1)
client_kwargs = dict(
host='localhost',
port=flow.port,
)
client_kwargs.update(flow._common_kwargs)
stop_event = multiprocessing.Event()
scale_event = multiprocessing.Event()
received_responses = multiprocessing.Queue()
response_arrival_times = multiprocessing.Queue()
process = multiprocessing.Process(
target=send_requests,
kwargs={
'client_kwargs': client_kwargs,
'stop_event': stop_event,
'scale_event': scale_event,
'received_responses': received_responses,
'response_arrival_times': response_arrival_times,
'logger': logger,
},
)
process.start()
process.join()
import numpy as np
response_times = []
while not response_arrival_times.empty():
response_times.append(response_arrival_times.get())
mean_response_time = np.mean(response_times)
logger.debug(
f'Mean time between responses is {mean_response_time}, expected is 1/3 second'
)
assert mean_response_time < 0.4
responses_list = []
while not received_responses.empty():
responses_list.append(int(received_responses.get()))
logger.debug(f'Got the following responses {sorted(responses_list)}')
assert sorted(responses_list) == list(
range(min(responses_list), max(responses_list) + 1)
)
async def test_no_message_lost_during_kill(logger, docker_images, tmpdir):
flow = Flow(name='test-flow-slow-process-executor',).add(
name='slow_process_executor',
uses=f'docker://{docker_images[0]}',
replicas=3,
)
dump_path = os.path.join(str(tmpdir), 'test_flow_k8s')
namespace = 'test-flow-slow-process-executor-ns-2'
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
)
# start port forwarding
logger.debug(f' Start port forwarding')
gateway_pod_name = (
core_client.list_namespaced_pod(
namespace=namespace, label_selector='app=gateway'
)
.items[0]
.metadata.name
)
config_path = os.environ['KUBECONFIG']
import portforward
with portforward.forward(
namespace, gateway_pod_name, flow.port, flow.port, config_path
):
# send requests and validate
time.sleep(0.1)
client_kwargs = dict(
host='localhost',
port=flow.port,
)
client_kwargs.update(flow._common_kwargs)
stop_event = multiprocessing.Event()
scale_event = multiprocessing.Event()
received_responses = multiprocessing.Queue()
response_arrival_times = multiprocessing.Queue()
process = multiprocessing.Process(
target=send_requests,
kwargs={
'client_kwargs': client_kwargs,
'stop_event': stop_event,
'scale_event': scale_event,
'received_responses': received_responses,
'response_arrival_times': response_arrival_times,
'logger': logger,
},
daemon=True,
)
process.start()
time.sleep(1.0)
logger.debug('Kill 2 replicas')
pods = core_client.list_namespaced_pod(
namespace=namespace,
label_selector=f'app=slow-process-executor',
)
names = [item.metadata.name for item in pods.items]
core_client.delete_namespaced_pod(names[0], namespace=namespace)
core_client.delete_namespaced_pod(names[1], namespace=namespace)
scale_event.set()
# wait for replicas to be dead
while True:
pods = core_client.list_namespaced_pod(
namespace=namespace,
label_selector=f'app=slow-process-executor',
)
current_pod_names = [item.metadata.name for item in pods.items]
if names[0] not in current_pod_names and names[1] not in current_pod_names:
logger.debug('Killing pods complete')
time.sleep(1.0)
stop_event.set()
break
else:
logger.debug(
f'not dead yet {current_pod_names} waiting for {names[0]} and {names[1]}'
)
time.sleep(1.0)
process.join()
responses_list = []
while not received_responses.empty():
responses_list.append(int(received_responses.get()))
logger.debug(f'Got the following responses {sorted(responses_list)}')
assert sorted(responses_list) == list(
range(min(responses_list), max(responses_list) + 1)
)
async def test_no_message_lost_during_scaling(logger, docker_images, tmpdir):
flow = Flow(name='test-flow-slow-process-executor',).add(
name='slow_process_executor',
uses=f'docker://{docker_images[0]}',
replicas=3,
)
dump_path = os.path.join(str(tmpdir), 'test_flow_k8s')
namespace = 'test-flow-slow-process-executor-ns'
flow.to_kubernetes_yaml(dump_path, k8s_namespace=namespace)
from kubernetes import client
api_client = client.ApiClient()
core_client = client.CoreV1Api(api_client=api_client)
app_client = client.AppsV1Api(api_client=api_client)
await create_all_flow_deployments_and_wait_ready(
dump_path,
namespace=namespace,
api_client=api_client,
app_client=app_client,
core_client=core_client,
)
# start port forwarding
gateway_pod_name = (
core_client.list_namespaced_pod(
namespace=namespace, label_selector='app=gateway'
)
.items[0]
.metadata.name
)
config_path = os.environ['KUBECONFIG']
import portforward
with portforward.forward(
namespace, gateway_pod_name, flow.port, flow.port, config_path
):
# send requests and validate
time.sleep(0.1)
client_kwargs = dict(
return_responses=True,
host='localhost',
port=flow.port,
)
client_kwargs.update(flow._common_kwargs)
stop_event = multiprocessing.Event()
scale_event = multiprocessing.Event()
received_responses = multiprocessing.Queue()
response_arrival_times = multiprocessing.Queue()
process = multiprocessing.Process(
target=send_requests,
kwargs={
'client_kwargs': client_kwargs,
'stop_event': stop_event,
'scale_event': scale_event,
'received_responses': received_responses,
'response_arrival_times': response_arrival_times,
'logger': logger,
},
daemon=True,
)
process.start()
time.sleep(1.0)
logger.debug('Scale down executor to 1 replica')
app_client.patch_namespaced_deployment_scale(
'slow-process-executor',
namespace=namespace,
body={'spec': {'replicas': 1}},
)
scale_event.set()
# wait for replicas to be dead
while True:
pods = core_client.list_namespaced_pod(
namespace=namespace,
label_selector=f'app=slow-process-executor',
)
if len(pods.items) == 1:
# still continue for a bit to hit the new replica only
logger.debug('Scale down complete')
time.sleep(1.0)
stop_event.set()
break
await asyncio.sleep(1.0)
await asyncio.sleep(10.0)
# kill the process as the client can hang due to lost responsed
if process.is_alive():
process.kill()
process.join()
responses_list = []
while not received_responses.empty():
responses_list.append(int(received_responses.get()))
logger.debug(f'Got the following responses {sorted(responses_list)}')
assert sorted(responses_list) == list(
range(min(responses_list), max(responses_list) + 1)
)
