def test_flow_to_k8s_yaml_sandbox(tmpdir): flow = Flow(name='test-flow', port=8080).add(uses=f'jinahub+sandbox://DummyHubExecutor') dump_path = os.path.join(str(tmpdir), 'test_flow_k8s') namespace = 'test-flow-ns' flow.to_k8s_yaml( output_base_path=dump_path, k8s_namespace=namespace, ) yaml_dicts_per_deployment = { 'gateway': [], } for pod_name in set(os.listdir(dump_path)): file_set = set(os.listdir(os.path.join(dump_path, pod_name))) for file in file_set: with open(os.path.join(dump_path, pod_name, file)) as f: yml_document_all = list(yaml.safe_load_all(f)) yaml_dicts_per_deployment[file[:-4]] = yml_document_all gateway_objects = yaml_dicts_per_deployment['gateway'] gateway_args = gateway_objects[2]['spec']['template']['spec'][ 'containers'][0]['args'] assert (gateway_args[gateway_args.index('--graph-description') + 1] == '{"executor0": ["end-gateway"], "start-gateway": ["executor0"]}') assert '--deployments-addresses' in gateway_args deployment_addresses = json.loads( gateway_args[gateway_args.index('--deployments-addresses') + 1]) assert deployment_addresses['executor0'][0].startswith('grpcs://')
async def test_flow_connection_pool(logger, k8s_connection_pool, docker_images, tmpdir): flow = Flow(name='k8s_flow_connection_pool', port_expose=9090, protocol='http').add( name='test_executor', replicas=2, uses=f'docker://{docker_images[0]}', ) dump_path = os.path.join(str(tmpdir), 'test-flow-connection-pool') namespace = f'test-flow-connection-pool-{k8s_connection_pool}'.lower() flow.to_k8s_yaml(dump_path, k8s_namespace=namespace, k8s_connection_pool=k8s_connection_pool) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, 'test-executor-head': 1, 'test-executor': 2, }, logger=logger, ) resp = await run_test( flow=flow, namespace=namespace, core_client=core_client, endpoint='/debug', n_docs=100, request_size=5, ) core_client.delete_namespace(namespace) visited = set() for r in resp: for doc in r.docs: visited.add(doc.tags['hostname']) if k8s_connection_pool: assert len(visited) == 2 else: assert len(visited) == 1
def test_flow_to_k8s_yaml_external_pod(tmpdir, has_external): flow = Flow(name='test-flow', port=8080).add(name='executor0', ) if has_external: flow = flow.add(name='external_executor', external=True, host='1.2.3.4', port=9090) else: flow = flow.add(name='external_executor') dump_path = os.path.join(str(tmpdir), 'test_flow_k8s') namespace = 'test-flow-ns' flow.to_k8s_yaml( output_base_path=dump_path, k8s_namespace=namespace, ) yaml_dicts_per_deployment = { 'gateway': [], 'executor0': [], } assert len(set(os.listdir(dump_path))) == 2 if has_external else 3 for pod_name in set(os.listdir(dump_path)): file_set = set(os.listdir(os.path.join(dump_path, pod_name))) for file in file_set: with open(os.path.join(dump_path, pod_name, file)) as f: yml_document_all = list(yaml.safe_load_all(f)) yaml_dicts_per_deployment[file[:-4]] = yml_document_all gateway_objects = yaml_dicts_per_deployment['gateway'] gateway_args = gateway_objects[2]['spec']['template']['spec'][ 'containers'][0]['args'] assert ( gateway_args[gateway_args.index('--graph-description') + 1] == '{"executor0": ["external_executor"], "start-gateway": ["executor0"], "external_executor": ["end-gateway"]}' ) if has_external: assert '--deployments-addresses' in gateway_args assert ( gateway_args[gateway_args.index('--deployments-addresses') + 1] == '{"executor0": ["grpc://executor0.test-flow-ns.svc:8080"], "external_executor": ["grpc://1.2.3.4:9090"]}' ) else: assert '--deployments-addresses' in gateway_args assert ( gateway_args[gateway_args.index('--deployments-addresses') + 1] == '{"executor0": ["grpc://executor0.test-flow-ns.svc:8080"], "external_executor": ["grpc://external-executor.test-flow-ns.svc:8080"]}' )
async def test_flow_with_monitoring(logger, tmpdir, docker_images, port_generator): dump_path = os.path.join(str(tmpdir), 'test-flow-with-monitoring') namespace = f'test-flow-monitoring'.lower() port1 = port_generator() port2 = port_generator() flow = Flow(name='test-flow-monitoring', monitoring=True, port_monitoring=port1).add( name='segmenter', uses=f'docker://{docker_images[0]}', monitoring=True, port_monitoring=port2, ) flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, 'segmenter': 1, }, logger=logger, ) import portforward config_path = os.environ['KUBECONFIG'] gateway_pod_name = (core_client.list_namespaced_pod( namespace=namespace, label_selector='app=gateway').items[0].metadata.name) pod_port_ref = [(gateway_pod_name, port1)] for (pod_name, port) in pod_port_ref: with portforward.forward(namespace, pod_name, port, port, config_path): resp = req.get(f'http://localhost:{port}/') assert resp.status_code == 200 core_client.delete_namespace(namespace)
async def test_flow_with_external_native_deployment(logger, docker_images, tmpdir): class DocReplaceExecutor(Executor): @requests def add(self, **kwargs): return DocumentArray( [Document(text='executor was here') for _ in range(100)]) args = set_deployment_parser().parse_args(['--uses', 'DocReplaceExecutor']) with Deployment(args) as external_deployment: ports = [args.port for args in external_deployment.pod_args['pods'][0]] flow = Flow(name='k8s_flow-with_external_deployment', port=9090).add( name='external_executor', external=True, host=f'172.17.0.1', port=ports[0], ) namespace = 'test-flow-with-external-deployment' dump_path = os.path.join(str(tmpdir), namespace) flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, }, logger=logger, ) resp = await run_test( flow=flow, namespace=namespace, core_client=core_client, endpoint='/', ) docs = resp[0].docs assert len(docs) == 100 for doc in docs: assert doc.text == 'executor was here' core_client.delete_namespace(namespace)
async def test_flow_with_external_k8s_deployment(logger, docker_images, tmpdir): namespace = 'test-flow-with-external-k8s-deployment' from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await _create_external_deployment(api_client, app_client, docker_images, tmpdir) flow = Flow(name='k8s_flow-with_external_deployment', port_expose=9090).add( name='external_executor', external=True, host='external-deployment-head.external-deployment-ns.svc', port_in=K8sGrpcConnectionPool.K8S_PORT_IN, ) dump_path = os.path.join(str(tmpdir), namespace) flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, }, logger=logger, ) resp = await run_test( flow=flow, namespace=namespace, core_client=core_client, endpoint='/workspace', ) docs = resp[0].docs for doc in docs: assert 'workspace' in doc.tags
async def test_flow_with_workspace(logger, k8s_connection_pool, docker_images, tmpdir): flow = Flow(name='k8s_flow-with_workspace', port_expose=9090, protocol='http').add( name='test_executor', uses=f'docker://{docker_images[0]}', workspace='/shared', ) dump_path = os.path.join(str(tmpdir), 'test-flow-with-workspace') namespace = f'test-flow-with-workspace-{k8s_connection_pool}'.lower() flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, 'test-executor-head': 1, 'test-executor': 1, }, logger=logger, ) resp = await run_test( flow=flow, namespace=namespace, core_client=core_client, endpoint='/workspace', ) docs = resp[0].docs assert len(docs) == 10 for doc in docs: assert doc.tags['workspace'] == '/shared/TestExecutor/0' core_client.delete_namespace(namespace)
def to_k8s_yaml( uses: str, output_base_path: str, k8s_namespace: Optional[str] = None, executor_type: Optional[ StandaloneExecutorType] = StandaloneExecutorType.EXTERNAL, uses_with: Optional[Dict] = None, uses_metas: Optional[Dict] = None, uses_requests: Optional[Dict] = None, **kwargs, ): """ Converts the Executor into a set of yaml deployments to deploy in Kubernetes. If you don't want to rebuild image on Jina Hub, you can set `JINA_HUB_NO_IMAGE_REBUILD` environment variable. :param uses: the Executor to use. Has to be containerized and accessible from K8s :param output_base_path: The base path where to dump all the yaml files :param k8s_namespace: The name of the k8s namespace to set for the configurations. If None, the name of the Flow will be used. :param executor_type: The type of Executor. Can be external or shared. External Executors include the Gateway. Shared Executors don't. Defaults to External :param uses_with: dictionary of parameters to overwrite from the default config's with field :param uses_metas: dictionary of parameters to overwrite from the default config's metas field :param uses_requests: dictionary of parameters to overwrite from the default config's requests field :param kwargs: other kwargs accepted by the Flow, full list can be found `here <https://docs.jina.ai/api/jina.orchestrate.flow.base/>` """ from jina import Flow f = Flow(**kwargs).add( uses=uses, uses_with=uses_with, uses_metas=uses_metas, uses_requests=uses_requests, ) f.to_k8s_yaml( output_base_path=output_base_path, k8s_namespace=k8s_namespace, include_gateway=executor_type == BaseExecutor.StandaloneExecutorType.EXTERNAL, )
async def test_failure_scenarios(logger, docker_images, tmpdir, k8s_cluster): namespace = 'test-failure-scenarios' from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) flow = Flow(prefetch=100).add(replicas=3, uses=f'docker://{docker_images[0]}') dump_path = os.path.join(str(tmpdir), namespace) flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, deployment_replicas_expected={ 'gateway': 1, 'executor0': 3, }, logger=logger, ) stop_event = asyncio.Event() send_task = asyncio.create_task( run_test_until_event( flow=flow, namespace=namespace, core_client=core_client, endpoint='/', stop_event=stop_event, logger=logger, sleep_time=None, )) await asyncio.sleep(5.0) # Scale down the Executor to 2 replicas await scale( deployment_name='executor0', desired_replicas=2, core_client=core_client, app_client=app_client, k8s_namespace=namespace, logger=logger, ) # Scale back up to 3 replicas await scale( deployment_name='executor0', desired_replicas=3, core_client=core_client, app_client=app_client, k8s_namespace=namespace, logger=logger, ) await asyncio.sleep(5.0) # restart all pods in the deployment await restart_deployment( deployment='executor0', app_client=app_client, core_client=core_client, k8s_namespace=namespace, logger=logger, ) await asyncio.sleep(5.0) await delete_pod( deployment='executor0', core_client=core_client, k8s_namespace=namespace, logger=logger, ) await asyncio.sleep(5.0) stop_event.set() responses, sent_ids = await send_task assert len(sent_ids) == len(responses) doc_ids = set() pod_ids = set() for response in responses: doc_id, pod_id = response.docs.texts[0].split('_') doc_ids.add(doc_id) pod_ids.add(pod_id) assert len(sent_ids) == len(doc_ids) assert len( pod_ids) == 8 # 3 original + 3 restarted + 1 scaled up + 1 deleted # do the random failure test # start sending again logger.info('Start sending for random failure test') stop_event.clear() send_task = asyncio.create_task( run_test_until_event( flow=flow, namespace=namespace, core_client=core_client, endpoint='/', stop_event=stop_event, logger=logger, )) # inject failures inject_failures(k8s_cluster, logger) # wait a bit await asyncio.sleep(3.0) # check that no message was lost stop_event.set() responses, sent_ids = await send_task assert len(sent_ids) == len(responses)
def test_raise_exception_invalid_executor(tmpdir): from jina.excepts import NoContainerizedError with pytest.raises(NoContainerizedError): f = Flow().add(uses='A') f.to_k8s_yaml(str(tmpdir))
async def test_linear_processing_time_scaling(docker_images, logger, tmpdir): flow = Flow(name='test-flow-slow-process-executor', ).add( name='slow_process_executor', uses=f'docker://{docker_images[0]}', replicas=3, ) dump_path = os.path.join(str(tmpdir), 'test_flow_k8s') namespace = 'test-flow-slow-process-executor-ns-3' flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, ) # start port forwarding logger.debug(f' Start port forwarding') gateway_pod_name = (core_client.list_namespaced_pod( namespace=namespace, label_selector='app=gateway').items[0].metadata.name) config_path = os.environ['KUBECONFIG'] import portforward with portforward.forward(namespace, gateway_pod_name, flow.port, flow.port, config_path): time.sleep(0.1) client_kwargs = dict( host='localhost', port=flow.port, ) client_kwargs.update(flow._common_kwargs) stop_event = multiprocessing.Event() scale_event = multiprocessing.Event() received_responses = multiprocessing.Queue() response_arrival_times = multiprocessing.Queue() process = multiprocessing.Process( target=send_requests, kwargs={ 'client_kwargs': client_kwargs, 'stop_event': stop_event, 'scale_event': scale_event, 'received_responses': received_responses, 'response_arrival_times': response_arrival_times, 'logger': logger, }, ) process.start() process.join() import numpy as np response_times = [] while not response_arrival_times.empty(): response_times.append(response_arrival_times.get()) mean_response_time = np.mean(response_times) logger.debug( f'Mean time between responses is {mean_response_time}, expected is 1/3 second' ) assert mean_response_time < 0.4 responses_list = [] while not received_responses.empty(): responses_list.append(int(received_responses.get())) logger.debug(f'Got the following responses {sorted(responses_list)}') assert sorted(responses_list) == list( range(min(responses_list), max(responses_list) + 1))
async def test_no_message_lost_during_kill(logger, docker_images, tmpdir): flow = Flow(name='test-flow-slow-process-executor', ).add( name='slow_process_executor', uses=f'docker://{docker_images[0]}', replicas=3, ) dump_path = os.path.join(str(tmpdir), 'test_flow_k8s') namespace = 'test-flow-slow-process-executor-ns-2' flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, ) # start port forwarding logger.debug(f' Start port forwarding') gateway_pod_name = (core_client.list_namespaced_pod( namespace=namespace, label_selector='app=gateway').items[0].metadata.name) config_path = os.environ['KUBECONFIG'] import portforward with portforward.forward(namespace, gateway_pod_name, flow.port, flow.port, config_path): # send requests and validate time.sleep(0.1) client_kwargs = dict( host='localhost', port=flow.port, ) client_kwargs.update(flow._common_kwargs) stop_event = multiprocessing.Event() scale_event = multiprocessing.Event() received_responses = multiprocessing.Queue() response_arrival_times = multiprocessing.Queue() process = multiprocessing.Process( target=send_requests, kwargs={ 'client_kwargs': client_kwargs, 'stop_event': stop_event, 'scale_event': scale_event, 'received_responses': received_responses, 'response_arrival_times': response_arrival_times, 'logger': logger, }, daemon=True, ) process.start() time.sleep(1.0) logger.debug('Kill 2 replicas') pods = core_client.list_namespaced_pod( namespace=namespace, label_selector=f'app=slow-process-executor', ) names = [item.metadata.name for item in pods.items] core_client.delete_namespaced_pod(names[0], namespace=namespace) core_client.delete_namespaced_pod(names[1], namespace=namespace) scale_event.set() # wait for replicas to be dead while True: pods = core_client.list_namespaced_pod( namespace=namespace, label_selector=f'app=slow-process-executor', ) current_pod_names = [item.metadata.name for item in pods.items] if names[0] not in current_pod_names and names[ 1] not in current_pod_names: logger.debug('Killing pods complete') time.sleep(1.0) stop_event.set() break else: logger.debug( f'not dead yet {current_pod_names} waiting for {names[0]} and {names[1]}' ) time.sleep(1.0) process.join() responses_list = [] while not received_responses.empty(): responses_list.append(int(received_responses.get())) logger.debug(f'Got the following responses {sorted(responses_list)}') assert sorted(responses_list) == list( range(min(responses_list), max(responses_list) + 1))
async def test_no_message_lost_during_scaling(logger, docker_images, tmpdir): flow = Flow(name='test-flow-slow-process-executor', ).add( name='slow_process_executor', uses=f'docker://{docker_images[0]}', replicas=3, ) dump_path = os.path.join(str(tmpdir), 'test_flow_k8s') namespace = 'test-flow-slow-process-executor-ns' flow.to_k8s_yaml(dump_path, k8s_namespace=namespace) from kubernetes import client api_client = client.ApiClient() core_client = client.CoreV1Api(api_client=api_client) app_client = client.AppsV1Api(api_client=api_client) await create_all_flow_deployments_and_wait_ready( dump_path, namespace=namespace, api_client=api_client, app_client=app_client, core_client=core_client, ) # start port forwarding gateway_pod_name = (core_client.list_namespaced_pod( namespace=namespace, label_selector='app=gateway').items[0].metadata.name) config_path = os.environ['KUBECONFIG'] import portforward with portforward.forward(namespace, gateway_pod_name, flow.port, flow.port, config_path): # send requests and validate time.sleep(0.1) client_kwargs = dict( return_responses=True, host='localhost', port=flow.port, ) client_kwargs.update(flow._common_kwargs) stop_event = multiprocessing.Event() scale_event = multiprocessing.Event() received_responses = multiprocessing.Queue() response_arrival_times = multiprocessing.Queue() process = multiprocessing.Process( target=send_requests, kwargs={ 'client_kwargs': client_kwargs, 'stop_event': stop_event, 'scale_event': scale_event, 'received_responses': received_responses, 'response_arrival_times': response_arrival_times, 'logger': logger, }, daemon=True, ) process.start() time.sleep(1.0) logger.debug('Scale down executor to 1 replica') app_client.patch_namespaced_deployment_scale( 'slow-process-executor', namespace=namespace, body={'spec': { 'replicas': 1 }}, ) scale_event.set() # wait for replicas to be dead while True: pods = core_client.list_namespaced_pod( namespace=namespace, label_selector=f'app=slow-process-executor', ) if len(pods.items) == 1: # still continue for a bit to hit the new replica only logger.debug('Scale down complete') time.sleep(1.0) stop_event.set() break await asyncio.sleep(1.0) await asyncio.sleep(10.0) # kill the process as the client can hang due to lost responsed if process.is_alive(): process.kill() process.join() responses_list = [] while not received_responses.empty(): responses_list.append(int(received_responses.get())) logger.debug(f'Got the following responses {sorted(responses_list)}') assert sorted(responses_list) == list( range(min(responses_list), max(responses_list) + 1))