def parse_config_source(path: Union[str, TextIO],
allow_stream: bool = True,
allow_yaml_file: bool = True,
allow_builtin_resource: bool = True,
allow_raw_yaml_content: bool = True,
allow_raw_driver_yaml_content: bool = True,
allow_class_type: bool = True,
*args,
**kwargs) -> Tuple[TextIO, Optional[str]]:
""" Check if the text or text stream is valid
:return: a tuple, the first element is the text stream, the second element is the file path associate to it
if available.
"""
import io
from pkg_resources import resource_filename, resource_stream
if not path:
raise BadConfigSource
elif allow_stream and hasattr(path, 'read'):
# already a readable stream
return path, None
elif allow_yaml_file and (path.endswith('.yml') or path.endswith('.yaml')):
comp_path = _complete_path(path)
return open(comp_path, encoding='utf8'), comp_path
elif allow_builtin_resource and path.startswith('_') and os.path.exists(
resource_filename('jina', '/'.join(
('resources', f'executors.{path}.yml')))):
# NOTE: this returns a binary stream
comp_path = resource_filename(
'jina', '/'.join(('resources', f'executors.{path}.yml')))
return open(comp_path, encoding='utf8'), comp_path
elif allow_raw_yaml_content and path.startswith('!'):
# possible YAML content
path = path.replace('|', '\n with: ')
return io.StringIO(path), None
elif allow_raw_driver_yaml_content and path.startswith('- !'):
# possible driver YAML content, right now it is only used for debugging
with open(
resource_filename(
'jina', '/'.join(
('resources',
'executors.base.all.yml' if path.startswith('- !!')
else 'executors.base.yml')))) as fp:
_defaults = fp.read()
path = path.replace('- !!', '- !').replace(
'|', '\n with: ') # for indent, I know, its nasty
path = _defaults.replace('*', path)
return io.StringIO(path), None
elif allow_class_type and path.isidentifier():
# possible class name
return io.StringIO(f'!{path}'), None
else:
raise BadConfigSource(
f'{path} can not be resolved, it should be a readable stream,'
' or a valid file path, or a supported class name.')
def load_config(
cls,
source: Union[str, TextIO, Dict],
*,
allow_py_modules: bool = True,
substitute: bool = True,
context: Optional[Dict[str, Any]] = None,
uses_with: Optional[Dict] = None,
uses_metas: Optional[Dict] = None,
uses_requests: Optional[Dict] = None,
extra_search_paths: Optional[List[str]] = None,
py_modules: Optional[str] = None,
runtime_args: Optional[Dict[str, Any]] = None,
**kwargs,
) -> 'JAMLCompatible':
"""A high-level interface for loading configuration with features
of loading extra py_modules, substitute env & context variables. Any class that
implements :class:`JAMLCompatible` mixin can enjoy this feature, e.g. :class:`BaseFlow`,
:class:`BaseExecutor`, :class:`BaseDriver` and all their subclasses.
Support substitutions in YAML:
- Environment variables: ``${{ ENV.VAR }}`` (recommended), ``$VAR`` (deprecated).
- Context dict (``context``): ``${{ CONTEXT.VAR }}``(recommended), ``${{ VAR }}``.
- Internal reference via ``this`` and ``root``: ``${{this.same_level_key}}``, ``${{root.root_level_key}}``
Substitutions are carried in the order and multiple passes to resolve variables with best effort.
.. highlight:: yaml
.. code-block:: yaml
!BaseEncoder
metas:
name: ${{VAR_A}} # env or context variables
workspace: my-${{this.name}} # internal reference
.. highlight:: python
.. code-block:: python
# load Executor from yaml file
BaseExecutor.load_config('a.yml')
# load Executor from yaml file and substitute environment variables
os.environ['VAR_A'] = 'hello-world'
b = BaseExecutor.load_config('a.yml')
assert b.name == 'hello-world'
# load Executor from yaml file and substitute variables from a dict
b = BaseExecutor.load_config('a.yml', context={'VAR_A': 'hello-world'})
assert b.name == 'hello-world'
# disable substitute
b = BaseExecutor.load_config('a.yml', substitute=False)
.. # noqa: DAR401
:param source: the multi-kind source of the configs.
:param allow_py_modules: allow importing plugins specified by ``py_modules`` in YAML at any levels
:param substitute: substitute environment, internal reference and context variables.
:param context: context replacement variables in a dict, the value of the dict is the replacement.
:param uses_with: dictionary of parameters to overwrite from the default config's with field
:param uses_metas: dictionary of parameters to overwrite from the default config's metas field
:param uses_requests: dictionary of parameters to overwrite from the default config's requests field
:param extra_search_paths: extra paths used when looking for executor yaml files
:param py_modules: Optional py_module from which the object need to be loaded
:param runtime_args: Optional dictionary of parameters runtime_args to be directly passed without being parsed into a yaml config
:param : runtime_args that need to be passed to the yaml
:param kwargs: kwargs for parse_config_source
:return: :class:`JAMLCompatible` object
"""
if runtime_args:
kwargs['runtimes_args'] = (
dict()
) # when we have runtime args it is needed to have an empty runtime args session in the yam config
if py_modules:
kwargs['runtimes_args']['py_modules'] = py_modules
if isinstance(source, str) and os.path.exists(source):
extra_search_paths = (extra_search_paths
or []) + [os.path.dirname(source)]
stream, s_path = parse_config_source(
source, extra_search_paths=extra_search_paths, **kwargs)
with stream as fp:
# first load yml with no tag
no_tag_yml = JAML.load_no_tags(fp)
if no_tag_yml:
no_tag_yml.update(**kwargs)
# if there is `override_with` u should make sure that `uses_with` does not remain in the yaml
def _delitem(
obj,
key,
):
value = obj.get(key, None)
if value:
del obj[key]
return
for k, v in obj.items():
if isinstance(v, dict):
_delitem(v, key)
if uses_with is not None:
_delitem(no_tag_yml, key='uses_with')
if uses_metas is not None:
_delitem(no_tag_yml, key='uses_metas')
if uses_requests is not None:
_delitem(no_tag_yml, key='uses_requests')
cls._override_yml_params(no_tag_yml, 'with', uses_with)
cls._override_yml_params(no_tag_yml, 'metas', uses_metas)
cls._override_yml_params(no_tag_yml, 'requests', uses_requests)
else:
raise BadConfigSource(
f'can not construct {cls} from an empty {source}. nothing to read from there'
)
if substitute:
# expand variables
no_tag_yml = JAML.expand_dict(no_tag_yml, context)
if allow_py_modules:
_extra_search_paths = extra_search_paths or []
load_py_modules(
no_tag_yml,
extra_search_paths=(_extra_search_paths +
[os.path.dirname(s_path)])
if s_path else _extra_search_paths,
)
from jina.orchestrate.flow.base import Flow
if issubclass(cls, Flow):
no_tag_yml_copy = copy.copy(no_tag_yml)
# only needed for Flow
if no_tag_yml_copy.get('with') is None:
no_tag_yml_copy['with'] = {}
no_tag_yml_copy['with']['extra_search_paths'] = (
no_tag_yml_copy['with'].get('extra_search_paths')
or []) + (extra_search_paths or [])
if cls.is_valid_jaml(no_tag_yml_copy):
no_tag_yml = no_tag_yml_copy
tag_yml = JAML.unescape(
JAML.dump(no_tag_yml),
include_unknown_tags=False,
jtype_whitelist=('Flow', ),
)
else:
# revert yaml's tag and load again, this time with substitution
tag_yml = JAML.unescape(JAML.dump(no_tag_yml))
# load into object, no more substitute
obj = JAML.load(tag_yml,
substitute=False,
runtime_args=runtime_args)
if not isinstance(obj, cls):
raise BadConfigSource(
f'Can not construct {cls} object from {source}. Source might be an invalid configuration.'
)
return obj
def parse_config_source(path: Union[str, TextIO, Dict],
allow_stream: bool = True,
allow_yaml_file: bool = True,
allow_builtin_resource: bool = True,
allow_raw_yaml_content: bool = True,
allow_raw_driver_yaml_content: bool = True,
allow_class_type: bool = True,
allow_dict: bool = True,
allow_json: bool = True,
*args,
**kwargs) -> Tuple[TextIO, Optional[str]]:
"""Check if the text or text stream is valid.
# noqa: DAR401
:param path: the multi-kind source of the configs.
:param allow_stream: flag
:param allow_yaml_file: flag
:param allow_builtin_resource: flag
:param allow_raw_yaml_content: flag
:param allow_raw_driver_yaml_content: flag
:param allow_class_type: flag
:param allow_dict: flag
:param allow_json: flag
:param *args: *args
:param **kwargs: **kwargs
:return: a tuple, the first element is the text stream, the second element is the file path associate to it
if available.
"""
import io
from pkg_resources import resource_filename
if not path:
raise BadConfigSource
elif allow_dict and isinstance(path, dict):
from . import JAML
tmp = JAML.dump(path)
return io.StringIO(tmp), None
elif allow_stream and hasattr(path, 'read'):
# already a readable stream
return path, None
elif allow_yaml_file and is_yaml_filepath(path):
comp_path = complete_path(path)
return open(comp_path, encoding='utf8'), comp_path
elif allow_builtin_resource and path.lstrip(
).startswith('_') and os.path.exists(
resource_filename('jina', '/'.join(
('resources', f'executors.{path}.yml')))):
# NOTE: this returns a binary stream
comp_path = resource_filename(
'jina', '/'.join(('resources', f'executors.{path}.yml')))
return open(comp_path, encoding='utf8'), comp_path
elif allow_raw_yaml_content and path.lstrip().startswith('!'):
# possible YAML content
path = path.replace('|', '\n with: ')
return io.StringIO(path), None
elif allow_raw_driver_yaml_content and path.lstrip().startswith('- !'):
# possible driver YAML content, right now it is only used for debugging
with open(
resource_filename(
'jina', '/'.join(('resources', 'executors.base.all.yml'
if path.lstrip().startswith('- !!') else
'executors.base.yml')))) as fp:
_defaults = fp.read()
path = path.replace('- !!', '- !').replace(
'|', '\n with: ') # for indent, I know, its nasty
path = _defaults.replace('*', path)
return io.StringIO(path), None
elif allow_class_type and path.isidentifier():
# possible class name
return io.StringIO(f'!{path}'), None
elif allow_json and isinstance(path, str):
try:
from . import JAML
tmp = json.loads(path)
tmp = JAML.dump(tmp)
return io.StringIO(tmp), None
except json.JSONDecodeError:
raise BadConfigSource(path)
else:
raise BadConfigSource(
f'{path} can not be resolved, it should be a readable stream,'
' or a valid file path, or a supported class name.')
def parse_config_source(
path: Union[str, TextIO, Dict],
allow_stream: bool = True,
allow_yaml_file: bool = True,
allow_raw_yaml_content: bool = True,
allow_class_type: bool = True,
allow_dict: bool = True,
allow_json: bool = True,
extra_search_paths: Optional[List[str]] = None,
*args,
**kwargs,
) -> Tuple[TextIO, Optional[str]]:
"""
Check if the text or text stream is valid.
.. # noqa: DAR401
:param path: the multi-kind source of the configs.
:param allow_stream: flag
:param allow_yaml_file: flag
:param allow_raw_yaml_content: flag
:param allow_class_type: flag
:param allow_dict: flag
:param allow_json: flag
:param extra_search_paths: extra paths to search for
:param args: unused
:param kwargs: unused
:return: a tuple, the first element is the text stream, the second element is the file path associate to it
if available.
"""
import io
if not path:
raise BadConfigSource
elif allow_dict and isinstance(path, dict):
from jina.jaml import JAML
tmp = JAML.dump(path)
return io.StringIO(tmp), None
elif allow_stream and hasattr(path, 'read'):
# already a readable stream
return path, None
elif allow_yaml_file and is_yaml_filepath(path):
comp_path = complete_path(path, extra_search_paths)
return open(comp_path, encoding='utf8'), comp_path
elif allow_raw_yaml_content and path.lstrip().startswith(('!', 'jtype')):
# possible YAML content
path = path.replace('|', '\n with: ')
return io.StringIO(path), None
elif allow_class_type and path.isidentifier():
# possible class name
return io.StringIO(f'!{path}'), None
elif allow_json and isinstance(path, str):
try:
from jina.jaml import JAML
tmp = json.loads(path)
tmp = JAML.dump(tmp)
return io.StringIO(tmp), None
except json.JSONDecodeError:
raise BadConfigSource(path)
else:
raise BadConfigSource(
f'{path} can not be resolved, it should be a readable stream,'
' or a valid file path, or a supported class name.')
