def export_api(args: 'Namespace'): import json from .export import api_to_dict from jina.jaml import JAML from jina import __version__ from jina.logging import default_logger from jina.schemas import get_full_schema if args.yaml_path: dump_api = api_to_dict() for yp in args.yaml_path: f_name = (yp % __version__) if '%s' in yp else yp with open(f_name, 'w', encoding='utf8') as fp: JAML.dump(dump_api, fp) default_logger.info(f'API is exported to {f_name}') if args.json_path: dump_api = api_to_dict() for jp in args.json_path: f_name = (jp % __version__) if '%s' in jp else jp with open(f_name, 'w', encoding='utf8') as fp: json.dump(dump_api, fp, sort_keys=True) default_logger.info(f'API is exported to {f_name}') if args.schema_path: dump_api = get_full_schema() for jp in args.schema_path: f_name = (jp % __version__) if '%s' in jp else jp with open(f_name, 'w', encoding='utf8') as fp: json.dump(dump_api, fp, sort_keys=True) default_logger.info(f'API is exported to {f_name}')
def test_dump_load_build(monkeypatch): f: Flow = Flow.load_config(''' jtype: Flow with: name: abc port: 12345 protocol: http executors: - name: executor1 port: 45678 shards: 2 - name: executor2 uses: docker://exec host: 1.2.3.4 - name: executor3 uses: docker://exec shards: 2 ''').build() f1: Flow = Flow.load_config(JAML.dump(f)).build() # these were passed by the user assert f.port == f1.port assert f.protocol == f1.protocol assert f['executor1'].args.port == f1['executor1'].args.port assert f['executor2'].args.host == f1['executor2'].args.host # this was set during `load_config` assert f['executor2'].args.port == f1['executor2'].args.port monkeypatch.setenv('JINA_FULL_CLI', 'true') f2: Flow = Flow.load_config(JAML.dump(f)).build() # these were passed by the user assert f.port == f2.port # validate gateway args (set during build) assert f['gateway'].args.port == f2['gateway'].args.port
def export_schema(args): """Export to JSON Schemas :param args: args from CLI """ if args.yaml_path: dump_api = api_to_dict() for yp in args.yaml_path: f_name = (yp % __version__) if '%s' in yp else yp with open(f_name, 'w', encoding='utf8') as fp: JAML.dump(dump_api, fp) default_logger.info(f'API is exported to {f_name}') if args.json_path: dump_api = api_to_dict() for jp in args.json_path: f_name = (jp % __version__) if '%s' in jp else jp with open(f_name, 'w', encoding='utf8') as fp: json.dump(dump_api, fp, sort_keys=True) default_logger.info(f'API is exported to {f_name}') if args.schema_path: dump_api = get_full_schema() for jp in args.schema_path: f_name = (jp % __version__) if '%s' in jp else jp with open(f_name, 'w', encoding='utf8') as fp: json.dump(dump_api, fp, sort_keys=True) default_logger.info(f'API is exported to {f_name}')
def _write_optimization_parameter(executor_configurations, target_file, overwrite_parameter_file): output = [ parameter for config in executor_configurations.values() for parameter in config ] if os.path.exists(target_file) and not overwrite_parameter_file: logger.warning( f"{target_file} already exists. Skip writing. Please remove it before parameter discovery." ) else: with open(target_file, "w") as outfile: JAML.dump(output, outfile)
def test_dump_load_build(monkeypatch): f: Flow = Flow.load_config( ''' jtype: Flow with: name: abc port_expose: 12345 protocol: http executors: - name: executor1 port_in: 45678 shards: 2 - name: executor2 uses: docker://exec host: 1.2.3.4 - name: executor3 uses: docker://exec shards: 2 ''' ).build() f['gateway'].args.runs_in_docker = True f['executor1'].args.runs_in_docker = True f1: Flow = Flow.load_config(JAML.dump(f)).build() assert not f1[ 'gateway' ].args.runs_in_docker # gateway doesn't have custom args set, as env was not set assert f1['executor1'].args.runs_in_docker # these were passed by the user assert f.port_expose == f1.port_expose assert f.protocol == f1.protocol assert f['executor1'].args.port_in == f1['executor1'].args.port_in assert f['executor2'].args.host == f1['executor2'].args.host # this was set during `load_config` assert f['executor2'].args.port_in == f1['executor2'].args.port_in assert f['executor3'].args.port_out == f1['executor3'].args.port_out # gateway args are not set, if `JINA_FULL_CLI` is not set assert f['gateway'].args.port_in != f1['gateway'].args.port_in assert f['gateway'].args.port_out != f1['gateway'].args.port_out monkeypatch.setenv('JINA_FULL_CLI', 'true') f2: Flow = Flow.load_config(JAML.dump(f)).build() assert f2['gateway'].args.runs_in_docker assert f2['executor1'].args.runs_in_docker # these were passed by the user assert f.port_expose == f2.port_expose # validate gateway args (set during build) assert f['gateway'].args.port_in == f2['gateway'].args.port_in assert f['gateway'].args.port_out == f2['gateway'].args.port_out assert f['gateway'].args.port_ctrl == f2['gateway'].args.port_ctrl
def test_load_yaml1(tmpdir): with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp: a = JAML.load(fp) assert isinstance(a[0], KVSearchDriver) assert isinstance(a[1], ControlReqDriver) assert isinstance(a[2], BaseDriver) with open(os.path.join(tmpdir, 'test_driver.yml'), 'w', encoding='utf8') as fp: JAML.dump(a[0], fp) with open(os.path.join(tmpdir, 'test_driver.yml'), encoding='utf8') as fp: b = JAML.load(fp) assert isinstance(b, KVSearchDriver) assert b._executor_name == a[0]._executor_name
def export_api(args: 'Namespace'): from .export import api_to_dict from jina import __version__ from jina.logging import default_logger if args.yaml_path: for yp in args.yaml_path: f_name = (yp % __version__) if '%s' in yp else yp from jina.jaml import JAML with open(f_name, 'w', encoding='utf8') as fp: JAML.dump(api_to_dict(), fp) default_logger.info(f'API is exported to {f_name}') if args.json_path: for jp in args.json_path: f_name = (jp % __version__) if '%s' in jp else jp import json with open(f_name, 'w', encoding='utf8') as fp: json.dump(api_to_dict(), fp, sort_keys=True) default_logger.info(f'API is exported to {f_name}')
def test_enum_dump(): assert JAML.dump(SocketType.PUSH_CONNECT).strip() == '"PUSH_CONNECT"'
def test_export_api(tmpdir): with open(tmpdir / 'test.yml', 'w', encoding='utf8') as fp: JAML.dump(api_to_dict(), fp) with open(tmpdir / 'test.json', 'w', encoding='utf8') as fp: json.dump(api_to_dict(), fp)
def test_enum_yaml(): assert JAML.load(JAML.dump(SocketType.PAIR_BIND)) == SocketType.PAIR_BIND
def parse_config_source( path: Union[str, TextIO, Dict], allow_stream: bool = True, allow_yaml_file: bool = True, allow_raw_yaml_content: bool = True, allow_class_type: bool = True, allow_dict: bool = True, allow_json: bool = True, extra_search_paths: Optional[List[str]] = None, *args, **kwargs, ) -> Tuple[TextIO, Optional[str]]: """ Check if the text or text stream is valid. .. # noqa: DAR401 :param path: the multi-kind source of the configs. :param allow_stream: flag :param allow_yaml_file: flag :param allow_raw_yaml_content: flag :param allow_class_type: flag :param allow_dict: flag :param allow_json: flag :param extra_search_paths: extra paths to search for :param args: unused :param kwargs: unused :return: a tuple, the first element is the text stream, the second element is the file path associate to it if available. """ import io if not path: raise BadConfigSource elif allow_dict and isinstance(path, dict): from jina.jaml import JAML tmp = JAML.dump(path) return io.StringIO(tmp), None elif allow_stream and hasattr(path, 'read'): # already a readable stream return path, None elif allow_yaml_file and is_yaml_filepath(path): comp_path = complete_path(path, extra_search_paths) return open(comp_path, encoding='utf8'), comp_path elif allow_raw_yaml_content and path.lstrip().startswith(('!', 'jtype')): # possible YAML content path = path.replace('|', '\n with: ') return io.StringIO(path), None elif allow_class_type and path.isidentifier(): # possible class name return io.StringIO(f'!{path}'), None elif allow_json and isinstance(path, str): try: from jina.jaml import JAML tmp = json.loads(path) tmp = JAML.dump(tmp) return io.StringIO(tmp), None except json.JSONDecodeError: raise BadConfigSource(path) else: raise BadConfigSource( f'{path} can not be resolved, it should be a readable stream,' ' or a valid file path, or a supported class name.')