def test_hub_build_push_push_again(): args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp: manifest = JAML.load(fp) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name'] with pytest.raises(ImageAlreadyExists): # try and push same version again should fail with `--no-overwrite` args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info', '--no-overwrite']) HubIO(args).build()
def test_hub_build_push(monkeypatch, mocker): monkeypatch.setattr(Path, 'is_file', True) mock_access_token = mocker.patch.object(hubapi, '_fetch_access_token', autospec=True) mock_access_token.return_value = os.environ.get('GITHUB_TOKEN', None) args = set_hub_build_parser().parse_args( [str(cur_dir + '/hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir + '/hub-mwu' + '/manifest.yml') as fp: manifest_jaml = JAML.load(fp, substitute=True) manifest = expand_dict(manifest_jaml) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_pull(mocker, monkeypatch, tmpdir, dummy_access_token): mock = mocker.Mock() def _mock_get(url, headers): mock(url=url, headers=headers) return MockResponse(response_code=requests.codes.ok) def _mock_post(url, headers, data): mock(url=url, headers=headers, data=data) return MockResponse(response_code=requests.codes.ok) def _mock_home(): return Path(str(tmpdir)) monkeypatch.setattr(Path, 'home', _mock_home) monkeypatch.setattr(requests, 'get', _mock_get) monkeypatch.setattr(requests, 'post', _mock_post) monkeypatch.setattr(HubIO, '_docker_login', mock) args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu'), '--push', '--raise-error']) HubIO(args).build() args = set_hub_pushpull_parser().parse_args( ['jinahub/pod.dummy_mwu_encoder:0.0.6']) HubIO(args).pull()
def test_hub_build_push(): args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp: manifest = JAML.load(fp) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_list_keywords(mocker): mocker.return_value.__enter__.return_value.read.return_value = json.dumps( [{ 'name': 'foo' }, { 'name': 'bar' }]) args = set_hub_list_parser().parse_args(['--keywords', 'numeric']) response = HubIO(args).list() numeric_count = len(response) mocker.assert_called_once() assert numeric_count > 1 args = set_hub_list_parser().parse_args( ['--keywords', 'numeric', 'randjojd']) response = HubIO(args).list() combined_count = len(response) assert mocker.call_count == 2 assert combined_count > 1 # Making sure both arguments are --keywords are considered as `either or` assert combined_count >= numeric_count
def test_alias2path_transform(): # bad naming result to itself assert HubIO._alias_to_local_path('abcdefg') == 'abcdefg' # good name results to the local path assert HubIO._alias_to_local_path('MongoDBIndexer') != 'MongoDBIndexer' assert os.path.exists(HubIO._alias_to_local_path('MongoDBIndexer'))
def test_hub_build_push(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu'), '--push', '--host-info']) summary = HubIO(args).build() with open(os.path.join(cur_dir, 'hub-mwu', 'manifest.yml')) as fp: manifest = yaml.load(fp) assert summary['is_build_success'] assert manifest['version'] == summary['version'] assert manifest['description'] == summary['manifest_info']['description'] assert manifest['author'] == summary['manifest_info']['author'] assert manifest['kind'] == summary['manifest_info']['kind'] assert manifest['type'] == summary['manifest_info']['type'] assert manifest['vendor'] == summary['manifest_info']['vendor'] assert manifest['keywords'] == summary['manifest_info']['keywords'] args = set_hub_list_parser().parse_args([ '--name', summary['manifest_info']['name'], '--keywords', summary['manifest_info']['keywords'][0], '--type', summary['manifest_info']['type'] ]) response = HubIO(args).list() manifests = response.json()['manifest'] assert response.status_code == 200 assert len(manifests) >= 1 assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_no_pymodules(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start')]) assert HubIO(args).build()['is_build_success'] args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start'), '--test-uses']) assert not HubIO(args).build()['is_build_success']
def test_hub_build_pull(): args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--push']) HubIO(args).build() args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder']) HubIO(args).pull() args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder:0.0.6']) HubIO(args).pull()
def test_jina_version_freeze_no_jina_dependency(requirements, tmpdir): import pkg_resources args = set_hub_build_parser().parse_args([str(tmpdir)]) hubio = HubIO(args) hubio._freeze_jina_version() requirements_file = os.path.join(tmpdir, 'requirements.txt') with open(requirements_file, 'r') as fp: requirements = pkg_resources.parse_requirements(fp) assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 0
def test_hub_build_pull(): args = set_hub_build_parser().parse_args( [str(cur_dir / 'hub-mwu'), '--push', '--test-uses', '--raise-error']) HubIO(args).build() args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder']) HubIO(args).pull() args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder:0.0.6']) HubIO(args).pull()
def test_hub_build_uses(): args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--test-uses']) HubIO(args).build() # build again it shall not fail HubIO(args).build() args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--test-uses', '--daemon']) HubIO(args).build() # build again it shall not fail HubIO(args).build()
def test_alias2tag_transform(): # bad naming result to itself assert HubIO._alias_to_docker_image_name('abcdefg') == 'abcdefg' # good name results to the local path assert HubIO._alias_to_docker_image_name( 'MongoDBIndexer') != 'MongoDBIndexer' assert HubIO._alias_to_docker_image_name('MongoDBIndexer').startswith( 'jinahub/') assert HubIO._alias_to_docker_image_name('MongoDBIndexer').endswith( __version__)
def test_hub_build_uses(): args = set_hub_build_parser().parse_args( [str(cur_dir / 'hub-mwu'), '--test-uses', '--raise-error']) HubIO(args).build() # build again it shall not fail HubIO(args).build() args = set_hub_build_parser().parse_args( [str(cur_dir / 'hub-mwu'), '--test-uses', '--daemon', '--raise-error']) HubIO(args).build() # build again it shall not fail HubIO(args).build()
def test_jina_version_freeze(requirements, tmpdir): import pkg_resources from jina import __version__ args = set_hub_build_parser().parse_args([str(tmpdir)]) hubio = HubIO(args) hubio._freeze_jina_version() requirements_file = os.path.join(tmpdir, 'requirements.txt') with open(requirements_file, 'r') as fp: requirements = pkg_resources.parse_requirements(fp) assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 1 for req in requirements: if 'jina' in str(req): assert str(req) == f'jina=={__version__}'
def test_hub_build_uses(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--raise-error'] ) assert HubIO(args).build()['is_build_success'] # build again it shall not fail assert HubIO(args).build()['is_build_success'] args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--daemon', '--raise-error'] ) assert HubIO(args).build()['is_build_success'] # build again it shall not fail assert HubIO(args).build()['is_build_success']
def test_hub_list_keywords(): args = set_hub_list_parser().parse_args(['--keywords', 'numeric']) response = HubIO(args).list() numeric_count = len(response) assert numeric_count > 1 args = set_hub_list_parser().parse_args(['--keywords', 'numeric', 'randjojd']) response = HubIO(args).list() combined_count = len(response) assert combined_count > 1 # Making sure both arguments are --keywords are considered as `either or` assert combined_count >= numeric_count
def test_build_timeout_ready(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'dummyhub_slow'), '--timeout-ready', '20000', '--test-uses', '--raise-error']) HubIO(args).build() with Flow().add(uses=f'docker://jinahub/pod.crafter.dummyhubexecutorslow:0.0.0-{jina_version}', timeout_ready=20000): pass
def docker_image(): from jina.parsers.hub import set_hub_build_parser from jina.docker.hubio import HubIO args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'numpyhub')]) HubIO(args).build()
def test_hub_build_level_pass(monkeypatch, mock_load_config): monkeypatch.setattr(BaseExecutor, "load_config", mock_load_config) p_names, failed_levels = HubIO._test_build("jinahub/pod.dummy_mwu_encoder", BuildTestLevel.EXECUTOR, "sample/yaml.yaml", 10000, True) expected_failed_levels = [] assert expected_failed_levels == failed_levels
def test_use_executor_pretrained_model_except(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'dummyhub_pretrained'), '--test-uses', '--raise-error'] ) with pytest.raises(HubBuilderError): HubIO(args).build()
def test_use_from_local_dir_flow_container_level(): args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'dummyhub'), '--test-uses', '--raise-error']) HubIO(args).build() with Flow().add( uses=f'jinahub/pod.crafter.dummyhubexecutor:0.0.0-{jina_version}'): pass
def test_create_new(tmpdir, new_type): args = set_hub_new_parser().parse_args( ['--output-dir', str(tmpdir), '--type', new_type] ) HubIO(args).new(no_input=True) list_dir = os.listdir(str(tmpdir)) assert len(list_dir) == 1
def test_dry_run(): hub_mwu_path = os.path.join(cur_dir, 'hub-mwu') args = set_hub_build_parser().parse_args([hub_mwu_path, '--dry-run']) result = HubIO(args).build() assert result['Dockerfile'] == os.path.join(hub_mwu_path, 'Dockerfile') assert result['manifest.yml'] == os.path.join(hub_mwu_path, 'manifest.yml') assert result['config.yml'] == os.path.join(hub_mwu_path, 'mwu_encoder.yml') assert result['README.md'] == os.path.join(hub_mwu_path, 'README.md')
def test_hub_build_failures(): for j in [ 'bad-dockerfile', 'bad-pythonfile', 'missing-dockerfile', 'missing-manifest' ]: args = set_hub_build_parser().parse_args( [os.path.join(cur_dir, 'hub-mwu-bad', j), '--pull', '--push']) assert not HubIO(args).build()['is_build_success']
def test_hub_build_failures(): for j in [ 'bad-dockerfile', 'bad-pythonfile', 'missing-dockerfile', 'missing-manifest' ]: args = set_hub_build_parser().parse_args( [str(cur_dir / 'hub-mwu-bad' / j)]) assert not HubIO(args).build()['is_build_success']
def test_hub_list_nonexisting_kind(mocker): mocker.return_value.__enter__.return_value.read.return_value = json.dumps([]) args = set_hub_list_parser().parse_args(['--kind', 'blah']) response = HubIO(args).list() mocker.assert_called_once() assert not response
def hub(args: 'Namespace'): """ Start a hub builder for build, push, pull :param args: arguments coming from the CLI. """ from jina.docker.hubio import HubIO getattr(HubIO(args), args.hub)()
def test_hub_build_level_fail(monkeypatch, test_workspace, docker_image): args = set_hub_build_parser().parse_args(['path/hub-mwu', '--push', '--host-info', '--test-level', 'FLOW']) expected_failed_levels = [BuildTestLevel.POD_NONDOCKER, BuildTestLevel.POD_DOCKER, BuildTestLevel.FLOW] _, failed_levels = HubIO(args)._test_build(docker_image, BuildTestLevel.FLOW, os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True, JinaLogger('unittest')) assert expected_failed_levels == failed_levels
def test_login(tmpdir, monkeypatch, mocker): import requests import webbrowser from pathlib import Path class MockResponse: def __init__(self, content): self.content = content def json(self): return self.content def status_code(self): return requests.codes.ok mock = mocker.Mock() def _mock_post(url, headers, data): mock(url=url, headers=headers, data=data) resp = { 'device_code': 'device', 'user_code': 'user', 'verification_uri': 'verification', 'access_token': 'access', } return MockResponse(resp) def _mock_home(): return Path(str(tmpdir)) args = set_hub_pushpull_parser() monkeypatch.setattr(requests, 'post', _mock_post) monkeypatch.setattr(webbrowser, 'open', None) monkeypatch.setattr(Path, 'home', _mock_home) HubIO(args).login() device_request_kwargs = mock.call_args_list[0][1] assert device_request_kwargs[ 'url'] == 'https://github.com/login/device/code' assert device_request_kwargs['headers'] == {'Accept': 'application/json'} assert 'client_id' in device_request_kwargs['data'] assert 'scope' in device_request_kwargs['data'] access_request_kwargs = mock.call_args_list[1][1] assert access_request_kwargs[ 'url'] == 'https://github.com/login/oauth/access_token' assert access_request_kwargs['headers'] == {'Accept': 'application/json'} assert 'client_id' in access_request_kwargs['data'] assert 'device_code' in access_request_kwargs['data'] assert access_request_kwargs['data']['device_code'] == 'device' list_dir = os.listdir(str(tmpdir)) assert '.jina' in list_dir jina_subfolder = os.path.join(str(tmpdir), '.jina') assert Path(os.path.join(jina_subfolder, 'access.yml')).exists() with open(os.path.join(jina_subfolder, 'access.yml')) as fp: assert fp.read() == 'access_token: access\n'