def test_class_yaml():
class DummyClass:
pass
yaml.register_class(DummyClass)
a = yaml.load('!DummyClass {}')
assert type(a) == DummyClass
with open(
resource_filename(
'jina', '/'.join(
('resources',
'executors.requests.BaseExecutor.yml')))) as fp:
b = fp.read()
print(b)
c = yaml.load(b)
print(c)
args = set_pea_parser().parse_args([])
with BasePea(args):
pass
from jina.executors.requests import _defaults
assert _defaults is not None
def test_class_yaml(self):
class DummyClass:
pass
yaml.register_class(DummyClass)
a = yaml.load('!DummyClass {}')
self.assertEqual(type(a), DummyClass)
with open(
resource_filename(
'jina', '/'.join(
('resources', 'executors.requests.%s.yml' %
'BaseExecutor')))) as fp:
b = fp.read()
print(b)
c = yaml.load(b)
print(c)
args = set_pea_parser().parse_args([])
with BasePea(args) as p:
pass
from jina.executors.requests import _defaults
self.assertIsNotNone(_defaults)
def test_load_yaml1(tmpdir):
with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp:
a = yaml.load(fp)
assert isinstance(a[0], KVSearchDriver)
assert isinstance(a[1], ControlReqDriver)
assert isinstance(a[2], BaseDriver)
with open(os.path.join(tmpdir, 'test_driver.yml'), 'w', encoding='utf8') as fp:
yaml.dump(a[0], fp)
with open(os.path.join(tmpdir, 'test_driver.yml'), encoding='utf8') as fp:
b = yaml.load(fp)
assert isinstance(b, KVSearchDriver)
assert b._executor_name == a[0]._executor_name
def test_hub_build_push_push_again():
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp:
manifest = yaml.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
# try and push same version again should fail
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
print(summary['is_build_success'])
def test_hub_build_push():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(os.path.join(cur_dir, 'hub-mwu', 'manifest.yml')) as fp:
manifest = yaml.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response.json()['manifest']
assert response.status_code == 200
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_exec_type(tmpdir):
from jina.executors.indexers import BaseIndexer
assert 'BaseIndexer' in BaseExecutor._registered_class
# init from YAML should be okay as well
BaseExecutor.load_config('BaseIndexer')
BaseIndexer().save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
s = yaml.load(fp)
def assert_bi():
b = BaseIndexer(1)
b.save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
b = yaml.load(fp)
assert b.a == 1
# we override BaseIndexer now, without force it shall not store all init values
class BaseIndexer(BaseExecutor):
def __init__(self, a=0):
super().__init__()
self.a = a
with pytest.raises(AssertionError):
assert_bi()
class BaseIndexer(BaseExecutor):
force_register = True
def __init__(self, a=0):
super().__init__()
self.a = a
assert_bi()
def get_default_login():
with resource_stream('jina', '/'.join(
('resources', 'hub-builder', 'login.yml'))) as fp:
login_info = yaml.load(fp)
for k, v in login_info.items():
login_info[k] = _decode(v)
return login_info
def test_load_yaml1(self):
with open(os.path.join(cur_dir, 'yaml/test-driver.yml'), encoding='utf8') as fp:
a = yaml.load(fp)
self.assertTrue(isinstance(a[0], KVSearchDriver))
self.assertTrue(isinstance(a[1], ControlReqDriver))
self.assertTrue(isinstance(a[2], BaseDriver))
with open('test_driver.yml', 'w', encoding='utf8') as fp:
yaml.dump(a[0], fp)
with open('test_driver.yml', encoding='utf8') as fp:
b = yaml.load(fp)
self.assertTrue(isinstance(b, KVSearchDriver))
assert b._executor_name == a[0]._executor_name
self.add_tmpfile('test_driver.yml')
def test_yaml_expand2():
with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp:
a = yaml.load(fp)
os.environ['ENV1'] = 'a'
b = expand_dict(a)
assert b['components'][0]['metas']['bad_var'] == 'real-compound'
assert b['components'][1]['metas']['bad_var'] == 2
assert b['components'][1]['metas']['float_var'] == 0.232
assert b['components'][1]['metas']['mixed'] == '0.232-2-real-compound'
assert b['components'][1]['metas']['mixed_env'] == '0.232-a'
assert b['components'][1]['metas']['name_shortcut'] == 'test_numpy'
def test_yaml_expand():
with open(os.path.join(cur_dir, 'yaml/test-expand.yml')) as fp:
a = yaml.load(fp)
b = expand_dict(a)
assert b['quote_dict'] == {}
assert b['quote_string'].startswith('{')
assert b['quote_string'].endswith('}')
assert b['nest']['quote_dict'] == {}
assert b['nest']['quote_string'].startswith('{')
assert b['nest']['quote_string'].endswith('}')
assert b['exist_env'] != '$PATH'
assert b['non_exist_env'] == '$JINA_WHATEVER_ENV'
def test_yaml_expand2(self):
with open('yaml/test-expand2.yml') as fp:
a = yaml.load(fp)
os.environ['ENV1'] = 'a'
b = expand_dict(a)
self.assertEqual(b['components'][0]['metas']['bad_var'],
'real-compound')
self.assertEqual(b['components'][1]['metas']['bad_var'], 2)
self.assertEqual(b['components'][1]['metas']['float_var'], 0.232)
self.assertEqual(b['components'][1]['metas']['mixed'],
'0.232-2-real-compound')
self.assertEqual(b['components'][1]['metas']['mixed_env'], '0.232-a')
self.assertEqual(b['components'][1]['metas']['name_shortcut'],
'test_numpy')
def test_hub_build_push():
args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(os.path.join(cur_dir, 'hub-mwu', 'manifest.yml')) as fp:
manifest = yaml.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
def test_sse_client(tmpdir):
class WrapAssert:
def __init__(self):
self.count = 0
conf_path = os.path.join(tmpdir, 'log')
path = os.path.join(conf_path, GROUP_ID)
event = threading.Event()
feed_thread = threading.Thread(name='feed_path_logs',
target=feed_path_logs,
daemon=False,
kwargs={
'path': path,
'threading_event': event
})
with open(os.path.join(cur_dir, 'logserver_config.yml')) as fp:
log_config = yaml.load(fp)
log_config['files']['log'] = conf_path
log_config['port'] = RANDOM_PORT
sse_server_thread = threading.Thread(name='sentinel-sse-logger',
target=start_sse_logger,
daemon=False,
args=(log_config, GROUP_ID, None))
wrap = WrapAssert()
sse_client_thread = threading.Thread(name='sse-client',
target=sse_client,
daemon=False,
kwargs=({
'wrap': wrap
}))
feed_thread.start()
time.sleep(0.5)
sse_server_thread.start()
time.sleep(0.5)
sse_client_thread.start()
time.sleep(0.5)
event.set()
stop_log_server()
feed_thread.join()
sse_server_thread.join()
sse_client_thread.join()
assert wrap.count > 0
def test_yaml_fill(self):
with open(os.path.join(cur_dir, 'yaml/test-expand2.yml')) as fp:
a = yaml.load(fp)
print(fill_metas_with_defaults(a))
def test_yaml_expand3(self):
with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp:
a = yaml.load(fp)
b = expand_dict(a)
print(b)
def test_yaml_expand3():
with open(os.path.join(cur_dir, 'yaml/test-expand3.yml')) as fp:
a = yaml.load(fp)
b = expand_dict(a)
assert b['pea_workspace'] != '{root.workspace}/{root.name}-{this.pea_id}'
def test_yaml_fill(self):
with open('yaml/test-expand2.yml') as fp:
a = yaml.load(fp)
print(fill_metas_with_defaults(a))
def assert_bi():
b = BaseIndexer(1)
b.save_config(os.path.join(tmpdir, 'tmp.yml'))
with open(os.path.join(tmpdir, 'tmp.yml')) as fp:
b = yaml.load(fp)
assert b.a == 1
def test_yaml_expand3(self):
with open('yaml/test-expand3.yml') as fp:
a = yaml.load(fp)
b = expand_dict(a)
print(b)
def _create(self,
config: Union[str, SpooledTemporaryFile,
List[PodModel]] = None,
files: List[UploadFile] = None):
""" Creates Flow using List[PodModel] or yaml spec """
# This makes sure `uses` & `py_modules` are created locally in `cwd`
# TODO: Handle file creation, deletion better
if files:
[
create_meta_files_from_upload(current_file)
for current_file in files
]
# FastAPI treats UploadFile as a tempfile.SpooledTemporaryFile
# I think this needs to be handled by some `FlowBuilder` class, transfrom a yaml_load to a config
if isinstance(config, str) or isinstance(config, SpooledTemporaryFile):
yamlspec = config.read().decode() if isinstance(
config, SpooledTemporaryFile) else config
try:
yaml.register_class(Flow)
flow = yaml.load(yamlspec)
except Exception as e:
self.logger.error(
f'Got error while loading from yaml {repr(e)}')
raise FlowYamlParseException
elif isinstance(config, list):
try:
flow = Flow()
# it is strange to build from a given flow, it seems like a lazy construction pattern could be used?
flow = self._build_with_pods(flow=flow, pod_args=config)
except Exception as e:
self.logger.error(
f'Got error while creating flows via pods: {repr(e)}')
raise FlowCreationException
else:
raise FlowBadInputException(
f'Not valid Flow config input {type(config)}')
try:
flow.args.log_id = flow.args.identity if 'identity' in flow.args else get_random_identity(
)
flow_id = uuid.UUID(flow.args.log_id)
flow = self._start(context=flow)
except PeaFailToStart as e:
self.logger.critical(
f'Flow couldn\'t get started - Invalid Pod {repr(e)} ')
self.logger.critical(
'Possible causes - invalid/not uploaded pod yamls & pymodules')
# TODO: Send correct error message
raise FlowStartException(repr(e))
except Exception as e:
self.logger.critical(
f'Got following error while starting the flow: {repr(e)}')
raise FlowStartException(repr(e))
self._store[flow_id] = {}
self._store[flow_id]['flow'] = flow
self._store[flow_id]['files'] = files
self.logger.info(
f'Started flow with flow_id {colored(flow_id, "cyan")}')
return flow_id, flow.host, flow.port_expose
import pytest
from jina.executors import BaseExecutor
from jina.executors.indexers import BaseIndexer
# BaseIndexer is already registered
from jina.helper import yaml
assert 'BaseIndexer' in BaseExecutor._registered_class
# init from YAML should be okay as well
BaseExecutor.load_config('BaseIndexer')
BaseIndexer().save_config('tmp.yml')
with open('tmp.yml') as fp:
s = yaml.load(fp)
def assert_bi():
b = BaseIndexer(1)
b.save_config('tmp.yml')
with open('tmp.yml') as fp:
b = yaml.load(fp)
assert b.a == 1
# we override BaseIndexer now, without force it shall not store all init values
class BaseIndexer(BaseExecutor):
def __init__(self, a=0):
super().__init__()
self.a = a
def assert_bi():
b = BaseIndexer(1)
b.save_config('tmp.yml')
with open('tmp.yml') as fp:
b = yaml.load(fp)
assert b.a == 1
