def test_two_client_route_parallel():
fa1 = set_flow_parser().parse_args(
['--optimize-level', str(FlowOptimizeLevel.NONE)])
f1 = Flow(fa1).add(parallel=3)
f2 = Flow(optimize_level=FlowOptimizeLevel.IGNORE_GATEWAY).add(parallel=3)
def start_client(fl):
fl.index(input_fn=random_docs(10))
with f1:
assert f1.num_peas == 6
t1 = mp.Process(target=start_client, args=(f1, ))
t1.daemon = True
t2 = mp.Process(target=start_client, args=(f1, ))
t2.daemon = True
t1.start()
t2.start()
with f2:
# no optimization can be made because we ignored the gateway
assert f2.num_peas == 6
t1 = mp.Process(target=start_client, args=(f2, ))
t1.daemon = True
t2 = mp.Process(target=start_client, args=(f2, ))
t2.daemon = True
t1.start()
t2.start()
async def _create(flow: 'FlowModel', ports: Optional[PortMappings] = None):
"""
.. #noqa: DAR101
.. #noqa: DAR201"""
try:
args = ArgNamespace.kwargs2namespace(flow.dict(), set_flow_parser())
return store.add(args, ports)
except Exception as ex:
raise PartialDaemon400Exception from ex
def test_flowstore_add(monkeypatch, partial_flow_store):
flow_model = FlowModel()
flow_model.uses = f'{cur_dir}/flow.yml'
args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser())
partial_store_item = partial_flow_store.add(args)
assert partial_store_item
assert isinstance(partial_flow_store.object, Flow)
assert 'executor1' in partial_store_item.yaml_source
assert partial_flow_store.object.port_expose == 12345
def test_flowstore_scale(partial_flow_store, mocker):
flow_model = FlowModel()
flow_model.uses = f'{cur_dir}/flow.yml'
args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser())
partial_flow_store.add(args)
scale_mock = mocker.Mock()
partial_flow_store.object.scale = scale_mock
partial_flow_store.scale(pod_name='executor1', replicas=2)
scale_mock.assert_called()
def test_flowstore_rolling_update(partial_flow_store, mocker):
flow_model = FlowModel()
flow_model.uses = f'{cur_dir}/flow.yml'
args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser())
partial_flow_store.add(args)
rolling_update_mock = mocker.Mock()
partial_flow_store.object.rolling_update = rolling_update_mock
partial_flow_store.rolling_update(pod_name='executor1', uses_with={})
rolling_update_mock.assert_called()
async def test_flowstore_scale(partial_flow_store, mocker):
flow_model = FlowModel()
flow_model.uses = f'{cur_dir}/flow.yml'
args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser())
partial_flow_store.add(args)
future = asyncio.Future()
future.set_result(PartialStoreItem())
mocker.patch('daemon.stores.partial.PartialFlowStore.scale',
return_value=future)
resp = await partial_flow_store.scale(deployment_name='executor1',
replicas=2)
assert resp
def test_flowstore_update(partial_flow_store, mocker):
flow_model = FlowModel()
flow_model.uses = f'{cur_dir}/flow.yml'
port_expose = helper.random_port()
args = ArgNamespace.kwargs2namespace(flow_model.dict(), set_flow_parser())
partial_flow_store.add(args, port_expose)
update_mock = mocker.Mock()
partial_flow_store.object.rolling_update = update_mock
partial_flow_store.update(kind=UpdateOperation.ROLLING_UPDATE,
dump_path='',
pod_name='pod1',
shards=1)
update_mock.assert_called()
def build_pydantic_model(kind: str = 'local',
model_name: str = 'CustomModel',
module: str = 'pod'):
if kind == 'api':
all_cli_args = get_latest_api()
module_args = get_module_args(all_args=all_cli_args, module=module)
all_fields, field_validators = get_pydantic_fields(config=module_args)
elif kind == 'local':
from jina.parsers import set_pea_parser, set_pod_parser
from jina.parsers.flow import set_flow_parser
if module == 'pod':
parser = set_pod_parser()
elif module == 'pea':
parser = set_pea_parser()
elif module == 'flow':
parser = set_flow_parser()
all_fields, field_validators = get_pydantic_fields(config=parser)
return create_model(model_name,
**all_fields,
__config__=PydanticConfig,
__validators__=field_validators)
def test_load_flow_from_cli():
a = set_flow_parser().parse_args(['--uses', 'yaml/test-flow-port.yml'])
f = Flow.load_config(a.uses)
with f:
assert f.port_expose == 12345
def get_main_parser():
"""The main parser for Jina
:return: the parser
"""
from jina.parsers.base import set_base_parser
from jina.parsers.create import set_new_project_parser
from jina.parsers.export_api import set_export_api_parser
from jina.parsers.flow import set_flow_parser
from jina.parsers.helloworld import set_hello_parser
from jina.parsers.helper import _SHOW_ALL_ARGS, _chf
from jina.parsers.hubble import set_hub_parser
from jina.parsers.ping import set_ping_parser
# create the top-level parser
parser = set_base_parser()
sp = parser.add_subparsers(
dest='cli',
required=True,
)
set_hello_parser(
sp.add_parser(
'hello',
help='Hello Jina!',
description='Start hello world demos.',
formatter_class=_chf,
))
set_pod_parser(
sp.add_parser(
'executor',
help='Start an Executor',
description=
'Start an Executor. Executor is how Jina processes Document.',
formatter_class=_chf,
))
set_flow_parser(
sp.add_parser(
'flow',
description=
'Start a Flow. Flow is how Jina streamlines and distributes Executors.',
help='Start a Flow',
formatter_class=_chf,
))
set_ping_parser(
sp.add_parser(
'ping',
help='Ping an Executor',
description='Ping a Deployment and check its network connectivity.',
formatter_class=_chf,
))
set_new_project_parser(
sp.add_parser(
'new',
help='Create a new Jina project',
description=
'Create a new Jina toy project with the predefined template.',
formatter_class=_chf,
))
set_gateway_parser(
sp.add_parser(
'gateway',
description=
'Start a Gateway that receives client Requests via gRPC/REST interface',
**(dict(help='Start a Gateway')) if _SHOW_ALL_ARGS else {},
formatter_class=_chf,
))
set_hub_parser(
sp.add_parser(
'hub',
help='Push/pull an Executor to/from Jina Hub',
description='Push/Pull an Executor to/from Jina Hub',
formatter_class=_chf,
))
set_help_parser(
sp.add_parser(
'help',
help='Show help text of a CLI argument',
description='Show help text of a CLI argument',
formatter_class=_chf,
))
# Below are low-level / internal / experimental CLIs, hidden from users by default
set_pod_parser(
sp.add_parser(
'pod',
description='Start a Pod. '
'You should rarely use this directly unless you '
'are doing low-level orchestration',
formatter_class=_chf,
**(dict(help='Start a Pod')) if _SHOW_ALL_ARGS else {},
))
set_deployment_parser(
sp.add_parser(
'deployment',
description='Start a Deployment. '
'You should rarely use this directly unless you '
'are doing low-level orchestration',
formatter_class=_chf,
**(dict(help='Start a Deployment')) if _SHOW_ALL_ARGS else {},
))
set_client_cli_parser(
sp.add_parser(
'client',
description=
'Start a Python client that connects to a remote Jina gateway',
formatter_class=_chf,
**(dict(help='Start a Client')) if _SHOW_ALL_ARGS else {},
))
set_export_api_parser(
sp.add_parser(
'export-api',
description=
'Export Jina API to JSON/YAML file for 3rd party applications',
formatter_class=_chf,
**(dict(help='Export Jina API to file')) if _SHOW_ALL_ARGS else {},
))
return parser
