def test_class_yaml():
class DummyClass:
pass
JAML.register(DummyClass)
a = JAML.load('!DummyClass {}')
assert type(a) == DummyClass
with open(
resource_filename(
'jina', '/'.join(
('resources',
'executors.requests.BaseExecutor.yml')))) as fp:
b = fp.read()
print(b)
c = JAML.load(b)
print(c)
args = set_pea_parser().parse_args([])
with BasePea(args):
pass
from jina.executors.requests import _defaults
assert _defaults is not None
def test_class_yaml2():
with open(
resource_filename(
'jina', '/'.join(
('resources',
'executors.requests.BaseExecutor.yml')))) as fp:
JAML.load(fp)
def add_handlers(self, config_path: Optional[str] = None, **kwargs):
"""
Add handlers from config file.
:param config_path: Path of config file.
:param kwargs: Extra parameters.
"""
self.logger.handlers = []
with open(config_path) as fp:
config = JAML.load(fp)
for h in config['handlers']:
cfg = config['configs'].get(h, None)
fmt = getattr(formatter, cfg.get('formatter', 'Formatter'))
if h not in self.supported or not cfg:
raise ValueError(
f'can not find configs for {h}, maybe it is not supported')
handler = None
if h == 'StreamHandler':
handler = logging.StreamHandler(sys.stdout)
handler.setFormatter(fmt(cfg['format'].format_map(kwargs)))
elif h == 'SysLogHandler' and not __windows__:
if cfg['host'] and cfg['port']:
handler = SysLogHandlerWrapper(address=(cfg['host'],
cfg['port']))
else:
# a UNIX socket is used
if platform.system() == 'Darwin':
handler = SysLogHandlerWrapper(
address='/var/run/syslog')
else:
handler = SysLogHandlerWrapper(address='/dev/log')
if handler:
handler.ident = cfg.get('ident', '')
handler.setFormatter(fmt(cfg['format'].format_map(kwargs)))
try:
handler._connect_unixsocket(handler.address)
except OSError:
handler = None
pass
elif h == 'FileHandler':
filename = cfg['output'].format_map(kwargs)
if __windows__:
# colons are not allowed in filenames
filename = filename.replace(':', '.')
handler = logging.FileHandler(filename, delay=True)
handler.setFormatter(fmt(cfg['format'].format_map(kwargs)))
if handler:
self.logger.addHandler(handler)
verbose_level = LogVerbosity.from_string(config['level'])
if 'JINA_LOG_LEVEL' in os.environ:
verbose_level = LogVerbosity.from_string(
os.environ['JINA_LOG_LEVEL'])
self.logger.setLevel(verbose_level.value)
def assert_bi():
b = BaseIndexer(1)
b.save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
b = JAML.load(fp)
assert b.a == 1
def test_load_yaml1(tmpdir):
with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp:
a = JAML.load(fp)
assert isinstance(a[0], KVSearchDriver)
assert isinstance(a[1], ControlReqDriver)
assert isinstance(a[2], BaseDriver)
with open(os.path.join(tmpdir, 'test_driver.yml'), 'w', encoding='utf8') as fp:
JAML.dump(a[0], fp)
with open(os.path.join(tmpdir, 'test_driver.yml'), encoding='utf8') as fp:
b = JAML.load(fp)
assert isinstance(b, KVSearchDriver)
assert b._executor_name == a[0]._executor_name
def test_hub_build_push(monkeypatch, mocker):
monkeypatch.setattr(Path, 'is_file', True)
mock_access_token = mocker.patch.object(hubapi,
'_fetch_access_token',
autospec=True)
mock_access_token.return_value = os.environ.get('GITHUB_TOKEN', None)
args = set_hub_build_parser().parse_args(
[str(cur_dir + '/hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir + '/hub-mwu' + '/manifest.yml') as fp:
manifest_jaml = JAML.load(fp, substitute=True)
manifest = expand_dict(manifest_jaml)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_push_push_again():
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp:
manifest = JAML.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'],
'--keywords', summary['manifest_info']['keywords'][0],
'--type', summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
with pytest.raises(ImageAlreadyExists):
# try and push same version again should fail with `--no-overwrite`
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info', '--no-overwrite'])
HubIO(args).build()
def test_yaml_expand3():
with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp:
a = JAML.load(fp)
b = expand_dict(a)
assert b['max_snapshot'] == 0
assert b['pea_workspace'] != '{root.workspace}/{root.name}-{this.pea_id}'
def test_hub_build_push():
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp:
manifest = JAML.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'],
'--keywords', summary['manifest_info']['keywords'][0],
'--type', summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_yaml_single_flow(tmpdir, config):
jsonlines_file = os.path.join(tmpdir, 'docs.jsonlines')
optimizer_yaml = f'''!FlowOptimizer
version: 1
with:
flow_runner: !SingleFlowRunner
with:
flow_yaml: '{os.path.join(cur_dir, 'flow.yml')}'
documents: {jsonlines_file}
request_size: 1
execution_endpoint: 'search'
evaluation_callback: !MeanEvaluationCallback {{}}
parameter_yaml: '{os.path.join(cur_dir, 'parameter.yml')}'
workspace_base_dir: {tmpdir}
n_trials: 5
'''
documents = document_generator(10)
with open(jsonlines_file, 'w') as f:
for document, groundtruth_doc in documents:
document.id = ""
groundtruth_doc.id = ""
json.dump(
{
'document': json.loads(MessageToJson(document).replace('\n', '')),
'groundtruth': json.loads(
MessageToJson(groundtruth_doc).replace('\n', '')
),
},
f,
)
f.write('\n')
optimizer = JAML.load(optimizer_yaml)
result = optimizer.optimize_flow()
validate_result(result, tmpdir)
def test_yaml_expand4():
os.environ['ENV1'] = 'a'
os.environ['ENV2'] = '{"1": "2"}'
with open(os.path.join(cur_dir, 'yaml/test-expand4.yml')) as fp:
b = JAML.load(
fp,
substitute=True,
context={
'context_var': 3.14,
'context_var2': 'hello-world'
},
)
assert b['components'][0]['metas']['bad_var'] == 'real-compound'
assert b['components'][1]['metas']['bad_var'] == 2
assert b['components'][1]['metas']['float_var'] == 0.232
assert b['components'][1]['metas']['mixed'] == '0.232-2-real-compound'
assert b['components'][1]['metas']['name_shortcut'] == 'test_numpy'
assert b['components'][1]['metas']['mixed_env'] == '0.232-a'
assert b['components'][1]['metas']['random_id'] == 3.14
assert b['components'][1]['metas']['config_str'] == 'hello-world'
assert b['components'][1]['metas']['bracket_env'] == '{"1": "2"}'
assert b['components'][1]['metas']['bracket_env'] == '{"1": "2"}'
assert b['components'][1]['metas']['context_dot'] == 3.14
def test_exec_type(tmpdir):
from jina.executors.indexers import BaseIndexer
assert 'BaseIndexer' in BaseExecutor._registered_class
# init from YAML should be okay as well
BaseExecutor.load_config('BaseIndexer')
BaseIndexer().save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
_ = JAML.load(fp)
def assert_bi():
b = BaseIndexer(1)
b.save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
b = JAML.load(fp)
assert b.a == 1
# we override BaseIndexer now, without force it shall not store all init values
class BaseIndexer(BaseExecutor):
def __init__(self, a=0):
super().__init__()
self.a = a
with pytest.raises(AssertionError):
assert_bi()
class BaseIndexer(BaseExecutor):
force_register = True
def __init__(self, a=0):
super().__init__()
self.a = a
assert_bi()
def test_class_yaml():
class DummyClass:
pass
JAML.register(DummyClass)
a = JAML.load('!DummyClass {}')
assert type(a) == DummyClass
def test_check_platform():
with resource_stream(
'jina', '/'.join(
('resources', 'hub-builder', 'platforms.yml'))) as fp:
platforms = JAML.load(fp)
check_platform(platforms)
with pytest.raises(ValueError):
check_platform(platforms + ['invalid'])
def test_check_licenses():
with resource_stream(
'jina', '/'.join(
('resources', 'hub-builder', 'osi-approved.yml'))) as fp:
licenses = JAML.load(fp)
for lic in licenses:
check_license(lic)
with pytest.raises(ValueError):
check_license('invalid')
def test_yaml_expand2():
with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp:
a = JAML.load(fp)
os.environ['ENV1'] = 'a'
b = expand_dict(a)
assert b['components'][0]['metas']['bad_var'] == 'real-compound'
assert b['components'][1]['metas']['bad_var'] == 2
assert b['components'][1]['metas']['float_var'] == 0.232
assert b['components'][1]['metas']['mixed'] == '0.232-2-real-compound'
assert b['components'][1]['metas']['mixed_env'] == '0.232-a'
assert b['components'][1]['metas']['name_shortcut'] == 'test_numpy'
def test_yaml_expand():
with open(os.path.join(cur_dir, 'yaml/test-expand.yml')) as fp:
a = JAML.load(fp)
b = expand_dict(a)
assert b['quote_dict'] == {}
assert b['quote_string'].startswith('{')
assert b['quote_string'].endswith('}')
assert b['nest']['quote_dict'] == {}
assert b['nest']['quote_string'].startswith('{')
assert b['nest']['quote_string'].endswith('}')
assert b['exist_env'] != '$PATH'
assert b['non_exist_env'] == '$JINA_WHATEVER_ENV'
def test_sse_client(tmpdir):
class WrapAssert:
def __init__(self):
self.count = 0
conf_path = os.path.join(tmpdir, 'log')
path = os.path.join(conf_path, GROUP_ID)
event = threading.Event()
feed_thread = threading.Thread(name='feed_path_logs',
target=feed_path_logs,
daemon=False,
kwargs={
'path': path,
'threading_event': event
})
with open(os.path.join(cur_dir, 'logserver_config.yml')) as fp:
log_config = JAML.load(fp)
log_config['files']['log'] = conf_path
log_config['port'] = RANDOM_PORT
sse_server_thread = threading.Thread(name='sentinel-sse-logger',
target=start_sse_logger,
daemon=False,
args=(log_config, GROUP_ID, None))
wrap = WrapAssert()
sse_client_thread = threading.Thread(name='sse-client',
target=sse_client,
daemon=False,
kwargs=({
'wrap': wrap
}))
feed_thread.start()
time.sleep(0.5)
sse_server_thread.start()
time.sleep(0.5)
sse_client_thread.start()
time.sleep(0.5)
event.set()
stop_log_server()
feed_thread.join()
sse_server_thread.join()
sse_client_thread.join()
assert wrap.count > 0
def test_base_ranking_evalutor():
evaluator = DummyRankingEvaluator()
actual_eval_driver = evaluator._drivers['SearchRequest'][-1]
assert isinstance(actual_eval_driver, RankEvaluateDriver)
default_eval_driver = RankEvaluateDriver()
assert list(default_eval_driver.fields) == actual_eval_driver.fields
# make sure the default value for fields in RankEvaluateDriver is no longer overwritten by `executors.requests.BaseRankingEvaluator.yml`
from jina.jaml import JAML
from pkg_resources import resource_filename
with open(
resource_filename(
'jina', '/'.join(
('resources',
'executors.requests.BaseRankingEvaluator.yml')))) as fp:
config_from_resources = JAML.load(fp)
assert default_eval_driver.fields == config_from_resources['on'][
'SearchRequest']['drivers'][-1].fields
def test_yaml(tmpdir):
jsonlines_file = os.path.join(tmpdir, 'docs.jsonlines')
optimizer_yaml = f'''!FlowOptimizer
version: 1
with:
flow_runner: !SingleFlowRunner
with:
flow_yaml: 'tests/integration/optimizers/flow.yml'
documents: {jsonlines_file}
request_size: 1
execution_method: 'search_lines'
documents_parameter_name: 'filepath'
evaluation_callback: !MeanEvaluationCallback {{}}
parameter_yaml: 'tests/integration/optimizers/parameter.yml'
workspace_base_dir: {tmpdir}
n_trials: 5
'''
documents = document_generator(10)
with open(jsonlines_file, 'w') as f:
for document, groundtruth_doc in documents:
document.id = ""
groundtruth_doc.id = ""
json.dump(
{
'document':
json.loads(MessageToJson(document).replace('\n', '')),
'groundtruth':
json.loads(
MessageToJson(groundtruth_doc).replace('\n', '')),
},
f,
)
f.write('\n')
optimizer = JAML.load(optimizer_yaml)
result = optimizer.optimize_flow()
result_path = str(tmpdir) + '/results/best_parameters.yml'
result.save_parameters(result_path)
parameters = result.best_parameters
assert parameters == BEST_PARAMETERS
assert yaml.load(open(result_path)) == BEST_PARAMETERS
def test_exec_type(tmpdir, f_register, unregister):
from jina.executors.indexers import BaseIndexer
assert 'jina.executors.indexers.BaseIndexer' in BaseExecutor._registered_class
# init from YAML should be okay as well
BaseExecutor.load_config('BaseIndexer')
BaseIndexer().save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
_ = JAML.load(fp)
def assert_bi():
b = BaseIndexer(1)
b.save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
b = JAML.load(fp)
assert b.a == 1
# By this point, BaseIndexer has not registered in reg_cls_set yet and store_init_kwargs will be executed
class BaseIndexer(BaseExecutor):
def __init__(self, a=0, *args, **kwargs):
super().__init__(*args, **kwargs)
self.a = a
assert_bi()
class BaseIndexer(BaseExecutor):
force_register = f_register
def __init__(self, a=0, *args, **kwargs):
super().__init__(*args, **kwargs)
self.a = a
if f_register:
assert_bi()
else:
with pytest.raises(AssertionError):
assert_bi()
def test_parsing_brackets_in_envvar():
flow_yaml = '''
jtype: Flow
executors:
- name: a
env:
var1: ${{ env.VAR1 }}
var4: -${{ env.VAR1 }}
var2: ${{root.executors[0].name}}
var3: ${{ env.VAR1 }}-${{root.executors[0].name}}
'''
with EnvironmentVarCtxtManager(envs={
'VAR1': '{"1": "2"}',
}):
b = JAML.load(flow_yaml, substitute=True)
assert b['executors'][0]['env']['var1'] == '{"1": "2"}'
assert b['executors'][0]['env']['var2'] == 'a'
assert b['executors'][0]['env']['var3'] == '{"1": "2"}-a'
assert b['executors'][0]['env']['var4'] == '-{"1": "2"}'
def test_hub_build_push_push_again(monkeypatch, access_token_github):
monkeypatch.setattr(Path, 'is_file', True)
monkeypatch.setattr(hubapi, '_fetch_access_token', access_token_github)
args = set_hub_build_parser().parse_args(
[str(cur_dir) + '/hub-mwu', '--push', '--host-info'])
summary = HubIO(args).build()
with open(str(cur_dir) + '/hub-mwu' + '/manifest.yml') as fp:
manifest_jaml = JAML.load(fp, substitute=True)
manifest = expand_dict(manifest_jaml)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
with pytest.raises(ImageAlreadyExists):
# try and push same version again should fail with `--no-overwrite`
args = set_hub_build_parser().parse_args([
str(cur_dir) + '/hub-mwu', '--push', '--host-info',
'--no-overwrite'
])
HubIO(args).build()
def test_yaml_fill():
with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp:
a = JAML.load(fp)
print(fill_metas_with_defaults(a))
def test_enum_yaml():
assert JAML.load(JAML.dump(SocketType.PAIR_BIND)) == SocketType.PAIR_BIND
def test_load_flow_from_empty_yaml():
with open(cur_dir / 'yaml' / 'dummy-flow.yml') as fp:
JAML.load(fp)
with open(cur_dir / 'yaml' / 'dummy-flow.yml') as fp:
Flow.load_config(fp)
def test_yaml_expand3():
with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp:
a = JAML.load(fp)
b = expand_dict(a)
assert b['max_snapshot'] == 0
def assert_bi():
b = BaseIndexer(1)
b.save_config(str(tmpdir / 'tmp.yml'))
with open(tmpdir / 'tmp.yml') as fp:
b = JAML.load(fp)
assert b.a == 1
