def test_class_yaml(): class DummyClass: pass JAML.register(DummyClass) a = JAML.load('!DummyClass {}') assert type(a) == DummyClass with open( resource_filename( 'jina', '/'.join( ('resources', 'executors.requests.BaseExecutor.yml')))) as fp: b = fp.read() print(b) c = JAML.load(b) print(c) args = set_pea_parser().parse_args([]) with BasePea(args): pass from jina.executors.requests import _defaults assert _defaults is not None
def test_class_yaml2(): with open( resource_filename( 'jina', '/'.join( ('resources', 'executors.requests.BaseExecutor.yml')))) as fp: JAML.load(fp)
def add_handlers(self, config_path: Optional[str] = None, **kwargs): """ Add handlers from config file. :param config_path: Path of config file. :param kwargs: Extra parameters. """ self.logger.handlers = [] with open(config_path) as fp: config = JAML.load(fp) for h in config['handlers']: cfg = config['configs'].get(h, None) fmt = getattr(formatter, cfg.get('formatter', 'Formatter')) if h not in self.supported or not cfg: raise ValueError( f'can not find configs for {h}, maybe it is not supported') handler = None if h == 'StreamHandler': handler = logging.StreamHandler(sys.stdout) handler.setFormatter(fmt(cfg['format'].format_map(kwargs))) elif h == 'SysLogHandler' and not __windows__: if cfg['host'] and cfg['port']: handler = SysLogHandlerWrapper(address=(cfg['host'], cfg['port'])) else: # a UNIX socket is used if platform.system() == 'Darwin': handler = SysLogHandlerWrapper( address='/var/run/syslog') else: handler = SysLogHandlerWrapper(address='/dev/log') if handler: handler.ident = cfg.get('ident', '') handler.setFormatter(fmt(cfg['format'].format_map(kwargs))) try: handler._connect_unixsocket(handler.address) except OSError: handler = None pass elif h == 'FileHandler': filename = cfg['output'].format_map(kwargs) if __windows__: # colons are not allowed in filenames filename = filename.replace(':', '.') handler = logging.FileHandler(filename, delay=True) handler.setFormatter(fmt(cfg['format'].format_map(kwargs))) if handler: self.logger.addHandler(handler) verbose_level = LogVerbosity.from_string(config['level']) if 'JINA_LOG_LEVEL' in os.environ: verbose_level = LogVerbosity.from_string( os.environ['JINA_LOG_LEVEL']) self.logger.setLevel(verbose_level.value)
def assert_bi(): b = BaseIndexer(1) b.save_config(os.path.join(tmpdir, 'tmp.yml')) with open(os.path.join(tmpdir, 'tmp.yml')) as fp: b = JAML.load(fp) assert b.a == 1
def test_load_yaml1(tmpdir): with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp: a = JAML.load(fp) assert isinstance(a[0], KVSearchDriver) assert isinstance(a[1], ControlReqDriver) assert isinstance(a[2], BaseDriver) with open(os.path.join(tmpdir, 'test_driver.yml'), 'w', encoding='utf8') as fp: JAML.dump(a[0], fp) with open(os.path.join(tmpdir, 'test_driver.yml'), encoding='utf8') as fp: b = JAML.load(fp) assert isinstance(b, KVSearchDriver) assert b._executor_name == a[0]._executor_name
def test_hub_build_push(monkeypatch, mocker): monkeypatch.setattr(Path, 'is_file', True) mock_access_token = mocker.patch.object(hubapi, '_fetch_access_token', autospec=True) mock_access_token.return_value = os.environ.get('GITHUB_TOKEN', None) args = set_hub_build_parser().parse_args( [str(cur_dir + '/hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir + '/hub-mwu' + '/manifest.yml') as fp: manifest_jaml = JAML.load(fp, substitute=True) manifest = expand_dict(manifest_jaml) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_push_push_again(): args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp: manifest = JAML.load(fp) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name'] with pytest.raises(ImageAlreadyExists): # try and push same version again should fail with `--no-overwrite` args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info', '--no-overwrite']) HubIO(args).build()
def test_yaml_expand3(): with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp: a = JAML.load(fp) b = expand_dict(a) assert b['max_snapshot'] == 0 assert b['pea_workspace'] != '{root.workspace}/{root.name}-{this.pea_id}'
def test_hub_build_push(): args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp: manifest = JAML.load(fp) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name']
def test_yaml_single_flow(tmpdir, config): jsonlines_file = os.path.join(tmpdir, 'docs.jsonlines') optimizer_yaml = f'''!FlowOptimizer version: 1 with: flow_runner: !SingleFlowRunner with: flow_yaml: '{os.path.join(cur_dir, 'flow.yml')}' documents: {jsonlines_file} request_size: 1 execution_endpoint: 'search' evaluation_callback: !MeanEvaluationCallback {{}} parameter_yaml: '{os.path.join(cur_dir, 'parameter.yml')}' workspace_base_dir: {tmpdir} n_trials: 5 ''' documents = document_generator(10) with open(jsonlines_file, 'w') as f: for document, groundtruth_doc in documents: document.id = "" groundtruth_doc.id = "" json.dump( { 'document': json.loads(MessageToJson(document).replace('\n', '')), 'groundtruth': json.loads( MessageToJson(groundtruth_doc).replace('\n', '') ), }, f, ) f.write('\n') optimizer = JAML.load(optimizer_yaml) result = optimizer.optimize_flow() validate_result(result, tmpdir)
def test_yaml_expand4(): os.environ['ENV1'] = 'a' os.environ['ENV2'] = '{"1": "2"}' with open(os.path.join(cur_dir, 'yaml/test-expand4.yml')) as fp: b = JAML.load( fp, substitute=True, context={ 'context_var': 3.14, 'context_var2': 'hello-world' }, ) assert b['components'][0]['metas']['bad_var'] == 'real-compound' assert b['components'][1]['metas']['bad_var'] == 2 assert b['components'][1]['metas']['float_var'] == 0.232 assert b['components'][1]['metas']['mixed'] == '0.232-2-real-compound' assert b['components'][1]['metas']['name_shortcut'] == 'test_numpy' assert b['components'][1]['metas']['mixed_env'] == '0.232-a' assert b['components'][1]['metas']['random_id'] == 3.14 assert b['components'][1]['metas']['config_str'] == 'hello-world' assert b['components'][1]['metas']['bracket_env'] == '{"1": "2"}' assert b['components'][1]['metas']['bracket_env'] == '{"1": "2"}' assert b['components'][1]['metas']['context_dot'] == 3.14
def test_exec_type(tmpdir): from jina.executors.indexers import BaseIndexer assert 'BaseIndexer' in BaseExecutor._registered_class # init from YAML should be okay as well BaseExecutor.load_config('BaseIndexer') BaseIndexer().save_config(os.path.join(tmpdir, 'tmp.yml')) with open(os.path.join(tmpdir, 'tmp.yml')) as fp: _ = JAML.load(fp) def assert_bi(): b = BaseIndexer(1) b.save_config(os.path.join(tmpdir, 'tmp.yml')) with open(os.path.join(tmpdir, 'tmp.yml')) as fp: b = JAML.load(fp) assert b.a == 1 # we override BaseIndexer now, without force it shall not store all init values class BaseIndexer(BaseExecutor): def __init__(self, a=0): super().__init__() self.a = a with pytest.raises(AssertionError): assert_bi() class BaseIndexer(BaseExecutor): force_register = True def __init__(self, a=0): super().__init__() self.a = a assert_bi()
def test_class_yaml(): class DummyClass: pass JAML.register(DummyClass) a = JAML.load('!DummyClass {}') assert type(a) == DummyClass
def test_check_platform(): with resource_stream( 'jina', '/'.join( ('resources', 'hub-builder', 'platforms.yml'))) as fp: platforms = JAML.load(fp) check_platform(platforms) with pytest.raises(ValueError): check_platform(platforms + ['invalid'])
def test_check_licenses(): with resource_stream( 'jina', '/'.join( ('resources', 'hub-builder', 'osi-approved.yml'))) as fp: licenses = JAML.load(fp) for lic in licenses: check_license(lic) with pytest.raises(ValueError): check_license('invalid')
def test_yaml_expand2(): with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp: a = JAML.load(fp) os.environ['ENV1'] = 'a' b = expand_dict(a) assert b['components'][0]['metas']['bad_var'] == 'real-compound' assert b['components'][1]['metas']['bad_var'] == 2 assert b['components'][1]['metas']['float_var'] == 0.232 assert b['components'][1]['metas']['mixed'] == '0.232-2-real-compound' assert b['components'][1]['metas']['mixed_env'] == '0.232-a' assert b['components'][1]['metas']['name_shortcut'] == 'test_numpy'
def test_yaml_expand(): with open(os.path.join(cur_dir, 'yaml/test-expand.yml')) as fp: a = JAML.load(fp) b = expand_dict(a) assert b['quote_dict'] == {} assert b['quote_string'].startswith('{') assert b['quote_string'].endswith('}') assert b['nest']['quote_dict'] == {} assert b['nest']['quote_string'].startswith('{') assert b['nest']['quote_string'].endswith('}') assert b['exist_env'] != '$PATH' assert b['non_exist_env'] == '$JINA_WHATEVER_ENV'
def test_sse_client(tmpdir): class WrapAssert: def __init__(self): self.count = 0 conf_path = os.path.join(tmpdir, 'log') path = os.path.join(conf_path, GROUP_ID) event = threading.Event() feed_thread = threading.Thread(name='feed_path_logs', target=feed_path_logs, daemon=False, kwargs={ 'path': path, 'threading_event': event }) with open(os.path.join(cur_dir, 'logserver_config.yml')) as fp: log_config = JAML.load(fp) log_config['files']['log'] = conf_path log_config['port'] = RANDOM_PORT sse_server_thread = threading.Thread(name='sentinel-sse-logger', target=start_sse_logger, daemon=False, args=(log_config, GROUP_ID, None)) wrap = WrapAssert() sse_client_thread = threading.Thread(name='sse-client', target=sse_client, daemon=False, kwargs=({ 'wrap': wrap })) feed_thread.start() time.sleep(0.5) sse_server_thread.start() time.sleep(0.5) sse_client_thread.start() time.sleep(0.5) event.set() stop_log_server() feed_thread.join() sse_server_thread.join() sse_client_thread.join() assert wrap.count > 0
def test_base_ranking_evalutor(): evaluator = DummyRankingEvaluator() actual_eval_driver = evaluator._drivers['SearchRequest'][-1] assert isinstance(actual_eval_driver, RankEvaluateDriver) default_eval_driver = RankEvaluateDriver() assert list(default_eval_driver.fields) == actual_eval_driver.fields # make sure the default value for fields in RankEvaluateDriver is no longer overwritten by `executors.requests.BaseRankingEvaluator.yml` from jina.jaml import JAML from pkg_resources import resource_filename with open( resource_filename( 'jina', '/'.join( ('resources', 'executors.requests.BaseRankingEvaluator.yml')))) as fp: config_from_resources = JAML.load(fp) assert default_eval_driver.fields == config_from_resources['on'][ 'SearchRequest']['drivers'][-1].fields
def test_yaml(tmpdir): jsonlines_file = os.path.join(tmpdir, 'docs.jsonlines') optimizer_yaml = f'''!FlowOptimizer version: 1 with: flow_runner: !SingleFlowRunner with: flow_yaml: 'tests/integration/optimizers/flow.yml' documents: {jsonlines_file} request_size: 1 execution_method: 'search_lines' documents_parameter_name: 'filepath' evaluation_callback: !MeanEvaluationCallback {{}} parameter_yaml: 'tests/integration/optimizers/parameter.yml' workspace_base_dir: {tmpdir} n_trials: 5 ''' documents = document_generator(10) with open(jsonlines_file, 'w') as f: for document, groundtruth_doc in documents: document.id = "" groundtruth_doc.id = "" json.dump( { 'document': json.loads(MessageToJson(document).replace('\n', '')), 'groundtruth': json.loads( MessageToJson(groundtruth_doc).replace('\n', '')), }, f, ) f.write('\n') optimizer = JAML.load(optimizer_yaml) result = optimizer.optimize_flow() result_path = str(tmpdir) + '/results/best_parameters.yml' result.save_parameters(result_path) parameters = result.best_parameters assert parameters == BEST_PARAMETERS assert yaml.load(open(result_path)) == BEST_PARAMETERS
def test_exec_type(tmpdir, f_register, unregister): from jina.executors.indexers import BaseIndexer assert 'jina.executors.indexers.BaseIndexer' in BaseExecutor._registered_class # init from YAML should be okay as well BaseExecutor.load_config('BaseIndexer') BaseIndexer().save_config(os.path.join(tmpdir, 'tmp.yml')) with open(os.path.join(tmpdir, 'tmp.yml')) as fp: _ = JAML.load(fp) def assert_bi(): b = BaseIndexer(1) b.save_config(os.path.join(tmpdir, 'tmp.yml')) with open(os.path.join(tmpdir, 'tmp.yml')) as fp: b = JAML.load(fp) assert b.a == 1 # By this point, BaseIndexer has not registered in reg_cls_set yet and store_init_kwargs will be executed class BaseIndexer(BaseExecutor): def __init__(self, a=0, *args, **kwargs): super().__init__(*args, **kwargs) self.a = a assert_bi() class BaseIndexer(BaseExecutor): force_register = f_register def __init__(self, a=0, *args, **kwargs): super().__init__(*args, **kwargs) self.a = a if f_register: assert_bi() else: with pytest.raises(AssertionError): assert_bi()
def test_parsing_brackets_in_envvar(): flow_yaml = ''' jtype: Flow executors: - name: a env: var1: ${{ env.VAR1 }} var4: -${{ env.VAR1 }} var2: ${{root.executors[0].name}} var3: ${{ env.VAR1 }}-${{root.executors[0].name}} ''' with EnvironmentVarCtxtManager(envs={ 'VAR1': '{"1": "2"}', }): b = JAML.load(flow_yaml, substitute=True) assert b['executors'][0]['env']['var1'] == '{"1": "2"}' assert b['executors'][0]['env']['var2'] == 'a' assert b['executors'][0]['env']['var3'] == '{"1": "2"}-a' assert b['executors'][0]['env']['var4'] == '-{"1": "2"}'
def test_hub_build_push_push_again(monkeypatch, access_token_github): monkeypatch.setattr(Path, 'is_file', True) monkeypatch.setattr(hubapi, '_fetch_access_token', access_token_github) args = set_hub_build_parser().parse_args( [str(cur_dir) + '/hub-mwu', '--push', '--host-info']) summary = HubIO(args).build() with open(str(cur_dir) + '/hub-mwu' + '/manifest.yml') as fp: manifest_jaml = JAML.load(fp, substitute=True) manifest = expand_dict(manifest_jaml) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name'] with pytest.raises(ImageAlreadyExists): # try and push same version again should fail with `--no-overwrite` args = set_hub_build_parser().parse_args([ str(cur_dir) + '/hub-mwu', '--push', '--host-info', '--no-overwrite' ]) HubIO(args).build()
def test_yaml_fill(): with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp: a = JAML.load(fp) print(fill_metas_with_defaults(a))
def test_enum_yaml(): assert JAML.load(JAML.dump(SocketType.PAIR_BIND)) == SocketType.PAIR_BIND
def test_load_flow_from_empty_yaml(): with open(cur_dir / 'yaml' / 'dummy-flow.yml') as fp: JAML.load(fp) with open(cur_dir / 'yaml' / 'dummy-flow.yml') as fp: Flow.load_config(fp)
def test_yaml_expand3(): with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp: a = JAML.load(fp) b = expand_dict(a) assert b['max_snapshot'] == 0
def assert_bi(): b = BaseIndexer(1) b.save_config(str(tmpdir / 'tmp.yml')) with open(tmpdir / 'tmp.yml') as fp: b = JAML.load(fp) assert b.a == 1