def test_two_client_route_parallel(): fa1 = set_flow_parser().parse_args( ['--optimize-level', str(FlowOptimizeLevel.NONE)]) f1 = Flow(fa1).add(parallel=3) f2 = Flow(optimize_level=FlowOptimizeLevel.IGNORE_GATEWAY).add(parallel=3) def start_client(fl): fl.index(input_fn=random_docs(10)) with f1: assert f1.num_peas == 6 t1 = mp.Process(target=start_client, args=(f1, )) t1.daemon = True t2 = mp.Process(target=start_client, args=(f1, )) t2.daemon = True t1.start() t2.start() with f2: # no optimization can be made because we ignored the gateway assert f2.num_peas == 6 t1 = mp.Process(target=start_client, args=(f2, )) t1.daemon = True t2 = mp.Process(target=start_client, args=(f2, )) t2.daemon = True t1.start() t2.start()
async def _create(flow: 'FlowModel', ports: Optional[PortMappings] = None): """ .. #noqa: DAR101 .. #noqa: DAR201""" try: args = ArgNamespace.kwargs2namespace(flow.dict(), set_flow_parser()) return store.add(args, ports) except Exception as ex: raise PartialDaemon400Exception from ex
def test_flowstore_add(monkeypatch, partial_flow_store): flow_model = FlowModel() flow_model.uses = f'{cur_dir}/flow.yml' args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser()) partial_store_item = partial_flow_store.add(args) assert partial_store_item assert isinstance(partial_flow_store.object, Flow) assert 'executor1' in partial_store_item.yaml_source assert partial_flow_store.object.port_expose == 12345
def test_flowstore_scale(partial_flow_store, mocker): flow_model = FlowModel() flow_model.uses = f'{cur_dir}/flow.yml' args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser()) partial_flow_store.add(args) scale_mock = mocker.Mock() partial_flow_store.object.scale = scale_mock partial_flow_store.scale(pod_name='executor1', replicas=2) scale_mock.assert_called()
def test_flowstore_rolling_update(partial_flow_store, mocker): flow_model = FlowModel() flow_model.uses = f'{cur_dir}/flow.yml' args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser()) partial_flow_store.add(args) rolling_update_mock = mocker.Mock() partial_flow_store.object.rolling_update = rolling_update_mock partial_flow_store.rolling_update(pod_name='executor1', uses_with={}) rolling_update_mock.assert_called()
async def test_flowstore_scale(partial_flow_store, mocker): flow_model = FlowModel() flow_model.uses = f'{cur_dir}/flow.yml' args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser()) partial_flow_store.add(args) future = asyncio.Future() future.set_result(PartialStoreItem()) mocker.patch('daemon.stores.partial.PartialFlowStore.scale', return_value=future) resp = await partial_flow_store.scale(deployment_name='executor1', replicas=2) assert resp
def test_flowstore_update(partial_flow_store, mocker): flow_model = FlowModel() flow_model.uses = f'{cur_dir}/flow.yml' port_expose = helper.random_port() args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser()) partial_flow_store.add(args, port_expose) update_mock = mocker.Mock() partial_flow_store.object.rolling_update = update_mock partial_flow_store.update(kind=UpdateOperation.ROLLING_UPDATE, dump_path='', pod_name='pod1', shards=1) update_mock.assert_called()
def build_pydantic_model(kind: str = 'local', model_name: str = 'CustomModel', module: str = 'pod'): if kind == 'api': all_cli_args = get_latest_api() module_args = get_module_args(all_args=all_cli_args, module=module) all_fields, field_validators = get_pydantic_fields(config=module_args) elif kind == 'local': from jina.parsers import set_pea_parser, set_pod_parser from jina.parsers.flow import set_flow_parser if module == 'pod': parser = set_pod_parser() elif module == 'pea': parser = set_pea_parser() elif module == 'flow': parser = set_flow_parser() all_fields, field_validators = get_pydantic_fields(config=parser) return create_model(model_name, **all_fields, __config__=PydanticConfig, __validators__=field_validators)
def test_load_flow_from_cli(): a = set_flow_parser().parse_args(['--uses', 'yaml/test-flow-port.yml']) f = Flow.load_config(a.uses) with f: assert f.port_expose == 12345
def get_main_parser(): """The main parser for Jina :return: the parser """ from jina.parsers.base import set_base_parser from jina.parsers.create import set_new_project_parser from jina.parsers.export_api import set_export_api_parser from jina.parsers.flow import set_flow_parser from jina.parsers.helloworld import set_hello_parser from jina.parsers.helper import _SHOW_ALL_ARGS, _chf from jina.parsers.hubble import set_hub_parser from jina.parsers.ping import set_ping_parser # create the top-level parser parser = set_base_parser() sp = parser.add_subparsers( dest='cli', required=True, ) set_hello_parser( sp.add_parser( 'hello', help='Hello Jina!', description='Start hello world demos.', formatter_class=_chf, )) set_pod_parser( sp.add_parser( 'executor', help='Start an Executor', description= 'Start an Executor. Executor is how Jina processes Document.', formatter_class=_chf, )) set_flow_parser( sp.add_parser( 'flow', description= 'Start a Flow. Flow is how Jina streamlines and distributes Executors.', help='Start a Flow', formatter_class=_chf, )) set_ping_parser( sp.add_parser( 'ping', help='Ping an Executor', description='Ping a Deployment and check its network connectivity.', formatter_class=_chf, )) set_new_project_parser( sp.add_parser( 'new', help='Create a new Jina project', description= 'Create a new Jina toy project with the predefined template.', formatter_class=_chf, )) set_gateway_parser( sp.add_parser( 'gateway', description= 'Start a Gateway that receives client Requests via gRPC/REST interface', **(dict(help='Start a Gateway')) if _SHOW_ALL_ARGS else {}, formatter_class=_chf, )) set_hub_parser( sp.add_parser( 'hub', help='Push/pull an Executor to/from Jina Hub', description='Push/Pull an Executor to/from Jina Hub', formatter_class=_chf, )) set_help_parser( sp.add_parser( 'help', help='Show help text of a CLI argument', description='Show help text of a CLI argument', formatter_class=_chf, )) # Below are low-level / internal / experimental CLIs, hidden from users by default set_pod_parser( sp.add_parser( 'pod', description='Start a Pod. ' 'You should rarely use this directly unless you ' 'are doing low-level orchestration', formatter_class=_chf, **(dict(help='Start a Pod')) if _SHOW_ALL_ARGS else {}, )) set_deployment_parser( sp.add_parser( 'deployment', description='Start a Deployment. ' 'You should rarely use this directly unless you ' 'are doing low-level orchestration', formatter_class=_chf, **(dict(help='Start a Deployment')) if _SHOW_ALL_ARGS else {}, )) set_client_cli_parser( sp.add_parser( 'client', description= 'Start a Python client that connects to a remote Jina gateway', formatter_class=_chf, **(dict(help='Start a Client')) if _SHOW_ALL_ARGS else {}, )) set_export_api_parser( sp.add_parser( 'export-api', description= 'Export Jina API to JSON/YAML file for 3rd party applications', formatter_class=_chf, **(dict(help='Export Jina API to file')) if _SHOW_ALL_ARGS else {}, )) return parser