def export_api(args: 'Namespace'):
import json
from .export import api_to_dict
from jina.jaml import JAML
from jina import __version__
from jina.logging import default_logger
from jina.schemas import get_full_schema
if args.yaml_path:
dump_api = api_to_dict()
for yp in args.yaml_path:
f_name = (yp % __version__) if '%s' in yp else yp
with open(f_name, 'w', encoding='utf8') as fp:
JAML.dump(dump_api, fp)
default_logger.info(f'API is exported to {f_name}')
if args.json_path:
dump_api = api_to_dict()
for jp in args.json_path:
f_name = (jp % __version__) if '%s' in jp else jp
with open(f_name, 'w', encoding='utf8') as fp:
json.dump(dump_api, fp, sort_keys=True)
default_logger.info(f'API is exported to {f_name}')
if args.schema_path:
dump_api = get_full_schema()
for jp in args.schema_path:
f_name = (jp % __version__) if '%s' in jp else jp
with open(f_name, 'w', encoding='utf8') as fp:
json.dump(dump_api, fp, sort_keys=True)
default_logger.info(f'API is exported to {f_name}')
def test_dump_load_build(monkeypatch):
f: Flow = Flow.load_config('''
jtype: Flow
with:
name: abc
port: 12345
protocol: http
executors:
- name: executor1
port: 45678
shards: 2
- name: executor2
uses: docker://exec
host: 1.2.3.4
- name: executor3
uses: docker://exec
shards: 2
''').build()
f1: Flow = Flow.load_config(JAML.dump(f)).build()
# these were passed by the user
assert f.port == f1.port
assert f.protocol == f1.protocol
assert f['executor1'].args.port == f1['executor1'].args.port
assert f['executor2'].args.host == f1['executor2'].args.host
# this was set during `load_config`
assert f['executor2'].args.port == f1['executor2'].args.port
monkeypatch.setenv('JINA_FULL_CLI', 'true')
f2: Flow = Flow.load_config(JAML.dump(f)).build()
# these were passed by the user
assert f.port == f2.port
# validate gateway args (set during build)
assert f['gateway'].args.port == f2['gateway'].args.port
def export_schema(args):
"""Export to JSON Schemas
:param args: args from CLI
"""
if args.yaml_path:
dump_api = api_to_dict()
for yp in args.yaml_path:
f_name = (yp % __version__) if '%s' in yp else yp
with open(f_name, 'w', encoding='utf8') as fp:
JAML.dump(dump_api, fp)
default_logger.info(f'API is exported to {f_name}')
if args.json_path:
dump_api = api_to_dict()
for jp in args.json_path:
f_name = (jp % __version__) if '%s' in jp else jp
with open(f_name, 'w', encoding='utf8') as fp:
json.dump(dump_api, fp, sort_keys=True)
default_logger.info(f'API is exported to {f_name}')
if args.schema_path:
dump_api = get_full_schema()
for jp in args.schema_path:
f_name = (jp % __version__) if '%s' in jp else jp
with open(f_name, 'w', encoding='utf8') as fp:
json.dump(dump_api, fp, sort_keys=True)
default_logger.info(f'API is exported to {f_name}')
def _write_optimization_parameter(executor_configurations, target_file,
overwrite_parameter_file):
output = [
parameter for config in executor_configurations.values()
for parameter in config
]
if os.path.exists(target_file) and not overwrite_parameter_file:
logger.warning(
f"{target_file} already exists. Skip writing. Please remove it before parameter discovery."
)
else:
with open(target_file, "w") as outfile:
JAML.dump(output, outfile)
def test_dump_load_build(monkeypatch):
f: Flow = Flow.load_config(
'''
jtype: Flow
with:
name: abc
port_expose: 12345
protocol: http
executors:
- name: executor1
port_in: 45678
shards: 2
- name: executor2
uses: docker://exec
host: 1.2.3.4
- name: executor3
uses: docker://exec
shards: 2
'''
).build()
f['gateway'].args.runs_in_docker = True
f['executor1'].args.runs_in_docker = True
f1: Flow = Flow.load_config(JAML.dump(f)).build()
assert not f1[
'gateway'
].args.runs_in_docker # gateway doesn't have custom args set, as env was not set
assert f1['executor1'].args.runs_in_docker
# these were passed by the user
assert f.port_expose == f1.port_expose
assert f.protocol == f1.protocol
assert f['executor1'].args.port_in == f1['executor1'].args.port_in
assert f['executor2'].args.host == f1['executor2'].args.host
# this was set during `load_config`
assert f['executor2'].args.port_in == f1['executor2'].args.port_in
assert f['executor3'].args.port_out == f1['executor3'].args.port_out
# gateway args are not set, if `JINA_FULL_CLI` is not set
assert f['gateway'].args.port_in != f1['gateway'].args.port_in
assert f['gateway'].args.port_out != f1['gateway'].args.port_out
monkeypatch.setenv('JINA_FULL_CLI', 'true')
f2: Flow = Flow.load_config(JAML.dump(f)).build()
assert f2['gateway'].args.runs_in_docker
assert f2['executor1'].args.runs_in_docker
# these were passed by the user
assert f.port_expose == f2.port_expose
# validate gateway args (set during build)
assert f['gateway'].args.port_in == f2['gateway'].args.port_in
assert f['gateway'].args.port_out == f2['gateway'].args.port_out
assert f['gateway'].args.port_ctrl == f2['gateway'].args.port_ctrl
def test_load_yaml1(tmpdir):
with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp:
a = JAML.load(fp)
assert isinstance(a[0], KVSearchDriver)
assert isinstance(a[1], ControlReqDriver)
assert isinstance(a[2], BaseDriver)
with open(os.path.join(tmpdir, 'test_driver.yml'), 'w', encoding='utf8') as fp:
JAML.dump(a[0], fp)
with open(os.path.join(tmpdir, 'test_driver.yml'), encoding='utf8') as fp:
b = JAML.load(fp)
assert isinstance(b, KVSearchDriver)
assert b._executor_name == a[0]._executor_name
def export_api(args: 'Namespace'):
from .export import api_to_dict
from jina import __version__
from jina.logging import default_logger
if args.yaml_path:
for yp in args.yaml_path:
f_name = (yp % __version__) if '%s' in yp else yp
from jina.jaml import JAML
with open(f_name, 'w', encoding='utf8') as fp:
JAML.dump(api_to_dict(), fp)
default_logger.info(f'API is exported to {f_name}')
if args.json_path:
for jp in args.json_path:
f_name = (jp % __version__) if '%s' in jp else jp
import json
with open(f_name, 'w', encoding='utf8') as fp:
json.dump(api_to_dict(), fp, sort_keys=True)
default_logger.info(f'API is exported to {f_name}')
def test_enum_dump():
assert JAML.dump(SocketType.PUSH_CONNECT).strip() == '"PUSH_CONNECT"'
def test_export_api(tmpdir):
with open(tmpdir / 'test.yml', 'w', encoding='utf8') as fp:
JAML.dump(api_to_dict(), fp)
with open(tmpdir / 'test.json', 'w', encoding='utf8') as fp:
json.dump(api_to_dict(), fp)
def test_enum_yaml():
assert JAML.load(JAML.dump(SocketType.PAIR_BIND)) == SocketType.PAIR_BIND
def parse_config_source(
path: Union[str, TextIO, Dict],
allow_stream: bool = True,
allow_yaml_file: bool = True,
allow_raw_yaml_content: bool = True,
allow_class_type: bool = True,
allow_dict: bool = True,
allow_json: bool = True,
extra_search_paths: Optional[List[str]] = None,
*args,
**kwargs,
) -> Tuple[TextIO, Optional[str]]:
"""
Check if the text or text stream is valid.
.. # noqa: DAR401
:param path: the multi-kind source of the configs.
:param allow_stream: flag
:param allow_yaml_file: flag
:param allow_raw_yaml_content: flag
:param allow_class_type: flag
:param allow_dict: flag
:param allow_json: flag
:param extra_search_paths: extra paths to search for
:param args: unused
:param kwargs: unused
:return: a tuple, the first element is the text stream, the second element is the file path associate to it
if available.
"""
import io
if not path:
raise BadConfigSource
elif allow_dict and isinstance(path, dict):
from jina.jaml import JAML
tmp = JAML.dump(path)
return io.StringIO(tmp), None
elif allow_stream and hasattr(path, 'read'):
# already a readable stream
return path, None
elif allow_yaml_file and is_yaml_filepath(path):
comp_path = complete_path(path, extra_search_paths)
return open(comp_path, encoding='utf8'), comp_path
elif allow_raw_yaml_content and path.lstrip().startswith(('!', 'jtype')):
# possible YAML content
path = path.replace('|', '\n with: ')
return io.StringIO(path), None
elif allow_class_type and path.isidentifier():
# possible class name
return io.StringIO(f'!{path}'), None
elif allow_json and isinstance(path, str):
try:
from jina.jaml import JAML
tmp = json.loads(path)
tmp = JAML.dump(tmp)
return io.StringIO(tmp), None
except json.JSONDecodeError:
raise BadConfigSource(path)
else:
raise BadConfigSource(
f'{path} can not be resolved, it should be a readable stream,'
' or a valid file path, or a supported class name.')
