def test_hub_build_no_pymodules():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start')])
assert HubIO(args).build()['is_build_success']
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start'), '--test-uses'])
assert not HubIO(args).build()['is_build_success']
def test_hub_build_uses():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--raise-error'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--daemon', '--raise-error'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
def test_hub_build_uses():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--raise-error']
)
assert HubIO(args).build()['is_build_success']
# build again it shall not fail
assert HubIO(args).build()['is_build_success']
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--daemon', '--raise-error']
)
assert HubIO(args).build()['is_build_success']
# build again it shall not fail
assert HubIO(args).build()['is_build_success']
def docker_image():
from jina.parsers.hub import set_hub_build_parser
from jina.docker.hubio import HubIO
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'numpyhub')])
HubIO(args).build()
def test_hub_build_pull(mocker, monkeypatch, tmpdir, dummy_access_token):
mock = mocker.Mock()
def _mock_get(url, headers):
mock(url=url, headers=headers)
return MockResponse(response_code=requests.codes.ok)
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
return MockResponse(response_code=requests.codes.ok)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(Path, 'home', _mock_home)
monkeypatch.setattr(requests, 'get', _mock_get)
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(HubIO, '_docker_login', mock)
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--raise-error'])
HubIO(args).build()
args = set_hub_pushpull_parser().parse_args(
['jinahub/pod.dummy_mwu_encoder:0.0.6'])
HubIO(args).pull()
def test_use_executor_pretrained_model_except():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub_pretrained'), '--test-uses', '--raise-error']
)
with pytest.raises(HubBuilderError):
HubIO(args).build()
def test_hub_build_push(monkeypatch, mocker):
monkeypatch.setattr(Path, 'is_file', True)
mock_access_token = mocker.patch.object(hubapi,
'_fetch_access_token',
autospec=True)
mock_access_token.return_value = os.environ.get('GITHUB_TOKEN', None)
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir + '/hub-mwu' + '/manifest.yml') as fp:
manifest_jaml = JAML.load(fp, substitute=True)
manifest = expand_dict(manifest_jaml)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_build_timeout_ready():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub_slow'), '--timeout-ready', '20000', '--test-uses', '--raise-error'])
HubIO(args).build()
with Flow().add(uses=f'docker://jinahub/pod.crafter.dummyhubexecutorslow:0.0.0-{jina_version}',
timeout_ready=20000):
pass
def test_dry_run():
hub_mwu_path = os.path.join(cur_dir, 'hub-mwu')
args = set_hub_build_parser().parse_args([hub_mwu_path, '--dry-run'])
result = HubIO(args).build()
assert result['Dockerfile'] == os.path.join(hub_mwu_path, 'Dockerfile')
assert result['manifest.yml'] == os.path.join(hub_mwu_path, 'manifest.yml')
assert result['config.yml'] == os.path.join(hub_mwu_path, 'mwu_encoder.yml')
assert result['README.md'] == os.path.join(hub_mwu_path, 'README.md')
def test_hub_build_failures():
for j in [
'bad-dockerfile', 'bad-pythonfile', 'missing-dockerfile',
'missing-manifest'
]:
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', j)])
assert not HubIO(args).build()['is_build_success']
def test_use_from_local_dir_flow_container_level():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub'), '--test-uses', '--raise-error'])
HubIO(args).build()
with Flow(
).add(uses=
f'docker://jinahub/pod.crafter.dummyhubexecutor:0.0.0-{jina_version}'
):
pass
def test_hub_build_level_fail(monkeypatch, test_workspace, docker_image):
args = set_hub_build_parser().parse_args(['path/hub-mwu', '--push', '--host-info', '--test-level', 'FLOW'])
expected_failed_levels = [BuildTestLevel.POD_NONDOCKER, BuildTestLevel.POD_DOCKER, BuildTestLevel.FLOW]
_, failed_levels = HubIO(args)._test_build(docker_image, BuildTestLevel.FLOW,
os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True,
JinaLogger('unittest'))
assert expected_failed_levels == failed_levels
def test_jina_version_freeze_no_jina_dependency(requirements, tmpdir):
import pkg_resources
args = set_hub_build_parser().parse_args([str(tmpdir)])
hubio = HubIO(args)
hubio._freeze_jina_version()
requirements_file = os.path.join(tmpdir, 'requirements.txt')
with open(requirements_file, 'r') as fp:
requirements = pkg_resources.parse_requirements(fp)
assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 0
def test_hub_build_pull():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--test-uses', '--raise-error'])
HubIO(args).build()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder'])
HubIO(args).pull()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder:0.0.6'])
HubIO(args).pull()
def test_hub_build_level_pass(monkeypatch, test_workspace):
args = set_hub_build_parser().parse_args(
['path/hub-mwu', '--push', '--host-info', '--test-level', 'EXECUTOR'])
docker_image = cli.get_image('jinahub/pod.dummy_mwu_encoder')
expected_failed_levels = []
_, failed_levels = HubIO(args)._test_build(
docker_image, BuildTestLevel.EXECUTOR,
os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True)
assert expected_failed_levels == failed_levels
def test_dry_run(dockerfile, argument):
hub_mwu_path = os.path.join(cur_dir, 'hub-mwu')
_args_list = [hub_mwu_path, '--dry-run']
if dockerfile:
_args_list += [argument, dockerfile]
args = set_hub_build_parser().parse_args(_args_list)
result = HubIO(args).build()
assert result['Dockerfile'] == os.path.join(hub_mwu_path, dockerfile if dockerfile else 'Dockerfile')
assert result['manifest.yml'] == os.path.join(hub_mwu_path, 'manifest.yml')
assert result['config.yml'] == os.path.join(hub_mwu_path, 'mwu_encoder.yml')
assert result['README.md'] == os.path.join(hub_mwu_path, 'README.md')
def test_jina_version_freeze(requirements, tmpdir):
import pkg_resources
from jina import __version__
args = set_hub_build_parser().parse_args([str(tmpdir)])
hubio = HubIO(args)
hubio._freeze_jina_version()
requirements_file = os.path.join(tmpdir, 'requirements.txt')
with open(requirements_file, 'r') as fp:
requirements = pkg_resources.parse_requirements(fp)
assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 1
for req in requirements:
if 'jina' in str(req):
assert str(req) == f'jina=={__version__}'
def test_hub_build_level_pass(monkeypatch, test_workspace, docker_image):
args = set_hub_build_parser().parse_args(
['path/hub-mwu', '--push', '--host-info', '--test-level', 'EXECUTOR'])
expected_failed_levels = []
_, failed_levels = HubIO(args)._test_build(
docker_image,
BuildTestLevel.EXECUTOR,
os.path.join(cur_dir, 'yaml/test-joint.yml'),
60000,
True,
JinaLogger('unittest'),
)
assert expected_failed_levels == failed_levels
def test_labels():
class MockContainers:
def __init__(self):
pass
def build(self, *args, **kwargs):
labels = kwargs['labels']
assert all([isinstance(v, str) for k, v in labels.items()])
assert 'ai.jina.hub.version' in labels
assert 'ai.jina.hub.name' in labels
raise BaseException('labels all good')
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--raise-error'])
hubio = HubIO(args)
hubio._raw_client = MockContainers()
with pytest.raises(BaseException, match='labels all good'):
hubio.build()
def test_hub_build_multistage(dockerfile_path):
args = set_hub_build_parser().parse_args(
[dockerfile_path, '--raise-error'])
result = HubIO(args).build()
assert result['is_build_success']
def test_jina_version_freeze_no_jina_dependency_git_no_raise(requirements, tmpdir):
args = set_hub_build_parser().parse_args([str(tmpdir)])
hubio = HubIO(args)
hubio._freeze_jina_version()
