def test_hub_build_push_push_again():
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp:
manifest = JAML.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'],
'--keywords', summary['manifest_info']['keywords'][0],
'--type', summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
with pytest.raises(ImageAlreadyExists):
# try and push same version again should fail with `--no-overwrite`
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info', '--no-overwrite'])
HubIO(args).build()
def test_hub_build_push(monkeypatch, mocker):
monkeypatch.setattr(Path, 'is_file', True)
mock_access_token = mocker.patch.object(hubapi,
'_fetch_access_token',
autospec=True)
mock_access_token.return_value = os.environ.get('GITHUB_TOKEN', None)
args = set_hub_build_parser().parse_args(
[str(cur_dir + '/hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir + '/hub-mwu' + '/manifest.yml') as fp:
manifest_jaml = JAML.load(fp, substitute=True)
manifest = expand_dict(manifest_jaml)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_pull(mocker, monkeypatch, tmpdir, dummy_access_token):
mock = mocker.Mock()
def _mock_get(url, headers):
mock(url=url, headers=headers)
return MockResponse(response_code=requests.codes.ok)
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
return MockResponse(response_code=requests.codes.ok)
def _mock_home():
return Path(str(tmpdir))
monkeypatch.setattr(Path, 'home', _mock_home)
monkeypatch.setattr(requests, 'get', _mock_get)
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(HubIO, '_docker_login', mock)
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--raise-error'])
HubIO(args).build()
args = set_hub_pushpull_parser().parse_args(
['jinahub/pod.dummy_mwu_encoder:0.0.6'])
HubIO(args).pull()
def test_hub_build_push():
args = set_hub_build_parser().parse_args([str(cur_dir / 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(cur_dir / 'hub-mwu' / 'manifest.yml') as fp:
manifest = JAML.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'],
'--keywords', summary['manifest_info']['keywords'][0],
'--type', summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_list_keywords(mocker):
mocker.return_value.__enter__.return_value.read.return_value = json.dumps(
[{
'name': 'foo'
}, {
'name': 'bar'
}])
args = set_hub_list_parser().parse_args(['--keywords', 'numeric'])
response = HubIO(args).list()
numeric_count = len(response)
mocker.assert_called_once()
assert numeric_count > 1
args = set_hub_list_parser().parse_args(
['--keywords', 'numeric', 'randjojd'])
response = HubIO(args).list()
combined_count = len(response)
assert mocker.call_count == 2
assert combined_count > 1
# Making sure both arguments are --keywords are considered as `either or`
assert combined_count >= numeric_count
def test_alias2path_transform():
# bad naming result to itself
assert HubIO._alias_to_local_path('abcdefg') == 'abcdefg'
# good name results to the local path
assert HubIO._alias_to_local_path('MongoDBIndexer') != 'MongoDBIndexer'
assert os.path.exists(HubIO._alias_to_local_path('MongoDBIndexer'))
def test_hub_build_push():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--push', '--host-info'])
summary = HubIO(args).build()
with open(os.path.join(cur_dir, 'hub-mwu', 'manifest.yml')) as fp:
manifest = yaml.load(fp)
assert summary['is_build_success']
assert manifest['version'] == summary['version']
assert manifest['description'] == summary['manifest_info']['description']
assert manifest['author'] == summary['manifest_info']['author']
assert manifest['kind'] == summary['manifest_info']['kind']
assert manifest['type'] == summary['manifest_info']['type']
assert manifest['vendor'] == summary['manifest_info']['vendor']
assert manifest['keywords'] == summary['manifest_info']['keywords']
args = set_hub_list_parser().parse_args([
'--name', summary['manifest_info']['name'], '--keywords',
summary['manifest_info']['keywords'][0], '--type',
summary['manifest_info']['type']
])
response = HubIO(args).list()
manifests = response.json()['manifest']
assert response.status_code == 200
assert len(manifests) >= 1
assert manifests[0]['name'] == summary['manifest_info']['name']
def test_hub_build_no_pymodules():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start')])
assert HubIO(args).build()['is_build_success']
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', 'fail-to-start'), '--test-uses'])
assert not HubIO(args).build()['is_build_success']
def test_hub_build_pull():
args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--push'])
HubIO(args).build()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder'])
HubIO(args).pull()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder:0.0.6'])
HubIO(args).pull()
def test_jina_version_freeze_no_jina_dependency(requirements, tmpdir):
import pkg_resources
args = set_hub_build_parser().parse_args([str(tmpdir)])
hubio = HubIO(args)
hubio._freeze_jina_version()
requirements_file = os.path.join(tmpdir, 'requirements.txt')
with open(requirements_file, 'r') as fp:
requirements = pkg_resources.parse_requirements(fp)
assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 0
def test_hub_build_pull():
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu'), '--push', '--test-uses', '--raise-error'])
HubIO(args).build()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder'])
HubIO(args).pull()
args = set_hub_pushpull_parser().parse_args(['jinahub/pod.dummy_mwu_encoder:0.0.6'])
HubIO(args).pull()
def test_hub_build_uses():
args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--test-uses'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
args = set_hub_build_parser().parse_args([os.path.join(cur_dir, 'hub-mwu'), '--pull', '--test-uses', '--daemon'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
def test_alias2tag_transform():
# bad naming result to itself
assert HubIO._alias_to_docker_image_name('abcdefg') == 'abcdefg'
# good name results to the local path
assert HubIO._alias_to_docker_image_name(
'MongoDBIndexer') != 'MongoDBIndexer'
assert HubIO._alias_to_docker_image_name('MongoDBIndexer').startswith(
'jinahub/')
assert HubIO._alias_to_docker_image_name('MongoDBIndexer').endswith(
__version__)
def test_hub_build_uses():
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu'), '--test-uses', '--raise-error'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu'), '--test-uses', '--daemon', '--raise-error'])
HubIO(args).build()
# build again it shall not fail
HubIO(args).build()
def test_jina_version_freeze(requirements, tmpdir):
import pkg_resources
from jina import __version__
args = set_hub_build_parser().parse_args([str(tmpdir)])
hubio = HubIO(args)
hubio._freeze_jina_version()
requirements_file = os.path.join(tmpdir, 'requirements.txt')
with open(requirements_file, 'r') as fp:
requirements = pkg_resources.parse_requirements(fp)
assert len(list(filter(lambda x: 'jina' in str(x), requirements))) == 1
for req in requirements:
if 'jina' in str(req):
assert str(req) == f'jina=={__version__}'
def test_hub_build_uses():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--raise-error']
)
assert HubIO(args).build()['is_build_success']
# build again it shall not fail
assert HubIO(args).build()['is_build_success']
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu'), '--test-uses', '--daemon', '--raise-error']
)
assert HubIO(args).build()['is_build_success']
# build again it shall not fail
assert HubIO(args).build()['is_build_success']
def test_hub_list_keywords():
args = set_hub_list_parser().parse_args(['--keywords', 'numeric'])
response = HubIO(args).list()
numeric_count = len(response)
assert numeric_count > 1
args = set_hub_list_parser().parse_args(['--keywords', 'numeric', 'randjojd'])
response = HubIO(args).list()
combined_count = len(response)
assert combined_count > 1
# Making sure both arguments are --keywords are considered as `either or`
assert combined_count >= numeric_count
def test_build_timeout_ready():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub_slow'), '--timeout-ready', '20000', '--test-uses', '--raise-error'])
HubIO(args).build()
with Flow().add(uses=f'docker://jinahub/pod.crafter.dummyhubexecutorslow:0.0.0-{jina_version}',
timeout_ready=20000):
pass
def docker_image():
from jina.parsers.hub import set_hub_build_parser
from jina.docker.hubio import HubIO
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'numpyhub')])
HubIO(args).build()
def test_hub_build_level_pass(monkeypatch, mock_load_config):
monkeypatch.setattr(BaseExecutor, "load_config", mock_load_config)
p_names, failed_levels = HubIO._test_build("jinahub/pod.dummy_mwu_encoder",
BuildTestLevel.EXECUTOR,
"sample/yaml.yaml", 10000, True)
expected_failed_levels = []
assert expected_failed_levels == failed_levels
def test_use_executor_pretrained_model_except():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub_pretrained'), '--test-uses', '--raise-error']
)
with pytest.raises(HubBuilderError):
HubIO(args).build()
def test_use_from_local_dir_flow_container_level():
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'dummyhub'), '--test-uses', '--raise-error'])
HubIO(args).build()
with Flow().add(
uses=f'jinahub/pod.crafter.dummyhubexecutor:0.0.0-{jina_version}'):
pass
def test_create_new(tmpdir, new_type):
args = set_hub_new_parser().parse_args(
['--output-dir', str(tmpdir), '--type', new_type]
)
HubIO(args).new(no_input=True)
list_dir = os.listdir(str(tmpdir))
assert len(list_dir) == 1
def test_dry_run():
hub_mwu_path = os.path.join(cur_dir, 'hub-mwu')
args = set_hub_build_parser().parse_args([hub_mwu_path, '--dry-run'])
result = HubIO(args).build()
assert result['Dockerfile'] == os.path.join(hub_mwu_path, 'Dockerfile')
assert result['manifest.yml'] == os.path.join(hub_mwu_path, 'manifest.yml')
assert result['config.yml'] == os.path.join(hub_mwu_path, 'mwu_encoder.yml')
assert result['README.md'] == os.path.join(hub_mwu_path, 'README.md')
def test_hub_build_failures():
for j in [
'bad-dockerfile', 'bad-pythonfile', 'missing-dockerfile',
'missing-manifest'
]:
args = set_hub_build_parser().parse_args(
[os.path.join(cur_dir, 'hub-mwu-bad', j), '--pull', '--push'])
assert not HubIO(args).build()['is_build_success']
def test_hub_build_failures():
for j in [
'bad-dockerfile', 'bad-pythonfile', 'missing-dockerfile',
'missing-manifest'
]:
args = set_hub_build_parser().parse_args(
[str(cur_dir / 'hub-mwu-bad' / j)])
assert not HubIO(args).build()['is_build_success']
def test_hub_list_nonexisting_kind(mocker):
mocker.return_value.__enter__.return_value.read.return_value = json.dumps([])
args = set_hub_list_parser().parse_args(['--kind', 'blah'])
response = HubIO(args).list()
mocker.assert_called_once()
assert not response
def hub(args: 'Namespace'):
"""
Start a hub builder for build, push, pull
:param args: arguments coming from the CLI.
"""
from jina.docker.hubio import HubIO
getattr(HubIO(args), args.hub)()
def test_hub_build_level_fail(monkeypatch, test_workspace, docker_image):
args = set_hub_build_parser().parse_args(['path/hub-mwu', '--push', '--host-info', '--test-level', 'FLOW'])
expected_failed_levels = [BuildTestLevel.POD_NONDOCKER, BuildTestLevel.POD_DOCKER, BuildTestLevel.FLOW]
_, failed_levels = HubIO(args)._test_build(docker_image, BuildTestLevel.FLOW,
os.path.join(cur_dir, 'yaml/test-joint.yml'), 60, True,
JinaLogger('unittest'))
assert expected_failed_levels == failed_levels
def test_login(tmpdir, monkeypatch, mocker):
import requests
import webbrowser
from pathlib import Path
class MockResponse:
def __init__(self, content):
self.content = content
def json(self):
return self.content
def status_code(self):
return requests.codes.ok
mock = mocker.Mock()
def _mock_post(url, headers, data):
mock(url=url, headers=headers, data=data)
resp = {
'device_code': 'device',
'user_code': 'user',
'verification_uri': 'verification',
'access_token': 'access',
}
return MockResponse(resp)
def _mock_home():
return Path(str(tmpdir))
args = set_hub_pushpull_parser()
monkeypatch.setattr(requests, 'post', _mock_post)
monkeypatch.setattr(webbrowser, 'open', None)
monkeypatch.setattr(Path, 'home', _mock_home)
HubIO(args).login()
device_request_kwargs = mock.call_args_list[0][1]
assert device_request_kwargs[
'url'] == 'https://github.com/login/device/code'
assert device_request_kwargs['headers'] == {'Accept': 'application/json'}
assert 'client_id' in device_request_kwargs['data']
assert 'scope' in device_request_kwargs['data']
access_request_kwargs = mock.call_args_list[1][1]
assert access_request_kwargs[
'url'] == 'https://github.com/login/oauth/access_token'
assert access_request_kwargs['headers'] == {'Accept': 'application/json'}
assert 'client_id' in access_request_kwargs['data']
assert 'device_code' in access_request_kwargs['data']
assert access_request_kwargs['data']['device_code'] == 'device'
list_dir = os.listdir(str(tmpdir))
assert '.jina' in list_dir
jina_subfolder = os.path.join(str(tmpdir), '.jina')
assert Path(os.path.join(jina_subfolder, 'access.yml')).exists()
with open(os.path.join(jina_subfolder, 'access.yml')) as fp:
assert fp.read() == 'access_token: access\n'
