def test_fetch(mocker, monkeypatch, rebuild_image):
mock = mocker.Mock()
def _mock_post(url, json, headers=None):
mock(url=url, json=json)
return FetchMetaMockResponse(response_code=200)
monkeypatch.setattr(requests, 'post', _mock_post)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder',
None,
rebuild_image=rebuild_image,
force=True)
assert executor.uuid == 'dummy_mwu_encoder'
assert executor.name == 'alias_dummy'
assert executor.tag == 'v0'
assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder'
assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f'
_, mock_kwargs = mock.call_args_list[0]
assert mock_kwargs['json']['rebuildImage'] is rebuild_image
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '', force=True)
assert executor.tag == 'v0'
_, mock_kwargs = mock.call_args_list[1]
assert mock_kwargs['json'][
'rebuildImage'] is True # default value must be True
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder',
'v0.1',
force=True)
assert executor.tag == 'v0.1'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile):
mock = mocker.Mock()
fail_meta_fetch = True
@disk_cache_offline(cache_file=str(tmpfile))
def _mock_fetch(name, tag=None, secret=None):
mock(name=name)
if fail_meta_fetch:
raise urllib.error.URLError('Failed fetching meta')
else:
return HubExecutor(
uuid='dummy_mwu_encoder',
alias='alias_dummy',
tag='v0',
image_name='jinahub/pod.dummy_mwu_encoder',
md5sum=None,
visibility=True,
archive_url=None,
)
def _gen_load_docker_client(fail_pull: bool):
def _load_docker_client(obj):
obj._raw_client = MockDockerClient(fail_pull=fail_pull)
return _load_docker_client
args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder'])
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=True),
)
monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch)
# Expect failure due to _fetch_meta
with pytest.raises(urllib.error.URLError):
HubIO(args).pull()
fail_meta_fetch = False
# Expect failure due to image pull
with pytest.raises(docker.errors.APIError):
HubIO(args).pull()
# expect successful pull
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=False),
)
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
# expect successful pull using cached _fetch_meta response and saved image
fail_meta_fetch = True
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=False),
)
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
def test_pull(test_envs, mocker, monkeypatch):
mock = mocker.Mock()
def _mock_fetch(name, tag=None, secret=None):
mock(name=name)
return HubExecutor(
uuid='dummy_mwu_encoder',
alias='alias_dummy',
tag='v0',
image_name='jinahub/pod.dummy_mwu_encoder',
md5sum=None,
visibility=True,
archive_url=None,
)
monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch)
def _mock_download(url, stream=True, headers=None):
mock(url=url)
return DownloadMockResponse(response_code=200)
def _mock_head(url):
from collections import namedtuple
HeadInfo = namedtuple('HeadInfo', ['headers'])
return HeadInfo(headers={})
monkeypatch.setattr(requests, 'get', _mock_download)
monkeypatch.setattr(requests, 'head', _mock_head)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
HubIO(args).pull()
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder:secret'])
HubIO(args).pull()
def update_runtime_cls(args, copy=False) -> 'Namespace':
"""Get runtime_cls as a string from args
:param args: pod/deployment namespace args
:param copy: True if args shouldn't be modified in-place
:return: runtime class as a string
"""
_args = deepcopy(args) if copy else args
gateway_runtime_dict = {
GatewayProtocolType.GRPC: 'GRPCGatewayRuntime',
GatewayProtocolType.WEBSOCKET: 'WebSocketGatewayRuntime',
GatewayProtocolType.HTTP: 'HTTPGatewayRuntime',
}
if _args.runtime_cls == 'WorkerRuntime' and is_valid_huburi(_args.uses):
_hub_args = deepcopy(_args)
_hub_args.uri = _args.uses
_hub_args.no_usage = True
_args.uses = HubIO(_hub_args).pull()
if hasattr(_args, 'protocol'):
_args.runtime_cls = gateway_runtime_dict[_args.protocol]
if _args.pod_role == PodRoleType.HEAD:
_args.runtime_cls = 'HeadRuntime'
return _args
def test_push(mocker, monkeypatch, path, mode, tmpdir, force, tag):
mock = mocker.Mock()
def _mock_post(url, data, headers=None, stream=True):
mock(url=url, data=data)
return PostMockResponse(response_code=requests.codes.created)
monkeypatch.setattr(requests, 'post', _mock_post)
# Second push will use --force --secret because of .jina/secret.key
# Then it will use put method
monkeypatch.setattr(requests, 'put', _mock_post)
exec_path = os.path.join(cur_dir, path)
_args_list = [exec_path, mode]
if force:
_args_list.extend(['--force', force])
if tag:
_args_list.append(tag)
args = set_hub_push_parser().parse_args(_args_list)
result = HubIO(args).push()
# remove .jina
exec_config_path = os.path.join(exec_path, '.jina')
shutil.rmtree(exec_config_path)
def test_push_wrong_dockerfile(
mocker, monkeypatch, path, mode, tmpdir, dockerfile, expected_error
):
dockerfile = os.path.join(cur_dir, path, dockerfile)
mock = mocker.Mock()
def _mock_post(url, data, headers=None, stream=True):
mock(url=url, data=data)
return PostMockResponse(response_code=requests.codes.created)
monkeypatch.setattr(requests, 'post', _mock_post)
# Second push will use --force --secret because of .jina/secret.key
# Then it will use put method
monkeypatch.setattr(requests, 'put', _mock_post)
exec_path = os.path.join(cur_dir, path)
_args_list = [exec_path, mode]
args = set_hub_push_parser().parse_args(_args_list)
args.dockerfile = dockerfile
with pytest.raises(Exception) as info:
HubIO(args).push()
assert expected_error.format(dockerfile=dockerfile, work_path=args.path) in str(
info.value
)
def build_pod(args: 'Namespace') -> Type['BasePod']:
"""Build an implementation of a `BasePod` interface
:param args: deployment arguments parsed from the CLI.
:return: the created BaseDeployment
"""
# copy to update but forward original
cargs = deepcopy(args)
if cargs.host != __default_host__ and not cargs.disable_remote:
cargs.timeout_ready = -1
return JinaDPod(cargs)
if is_valid_huburi(cargs.uses):
_hub_args = deepcopy(args)
_hub_args.uri = args.uses
_hub_args.no_usage = True
cargs.uses = HubIO(_hub_args).pull()
if (
cargs.pod_role != PodRoleType.HEAD
and cargs.uses
and cargs.uses.startswith('docker://')
):
return ContainerPod(cargs)
else:
return Pod(args)
def from_hub(
cls: Type[T],
uri: str,
context: Optional[Dict[str, Any]] = None,
uses_with: Optional[Dict] = None,
uses_metas: Optional[Dict] = None,
uses_requests: Optional[Dict] = None,
**kwargs,
) -> T:
"""Construct an Executor from Hub.
:param uri: a hub Executor scheme starts with `jinahub://`
:param context: context replacement variables in a dict, the value of the dict is the replacement.
:param uses_with: dictionary of parameters to overwrite from the default config's with field
:param uses_metas: dictionary of parameters to overwrite from the default config's metas field
:param uses_requests: dictionary of parameters to overwrite from the default config's requests field
:param kwargs: other kwargs accepted by the CLI ``jina hub pull``
:return: the Hub Executor object.
.. highlight:: python
.. code-block:: python
from jina import Executor
from docarray import Document, DocumentArray
executor = Executor.from_hub(
uri='jinahub://CLIPImageEncoder', install_requirements=True
)
"""
from jina.hubble.helper import is_valid_huburi
_source = None
if is_valid_huburi(uri):
from jina.hubble.hubio import HubIO
from jina.parsers.hubble import set_hub_pull_parser
_args = ArgNamespace.kwargs2namespace(
{
'no_usage': True,
**kwargs
},
set_hub_pull_parser(),
positional_args=(uri, ),
)
_source = HubIO(args=_args).pull()
if not _source or _source.startswith('docker://'):
raise ValueError(
f'Can not construct a native Executor from {uri}. Looks like you want to use it as a '
f'Docker container, you may want to use it in the Flow via `.add(uses={uri})` instead.'
)
return cls.load_config(
_source,
context=context,
uses_with=uses_with,
uses_metas=uses_metas,
uses_requests=uses_requests,
)
def hub(args: 'Namespace'):
"""
Start a hub builder for push, pull
:param args: arguments coming from the CLI.
"""
from jina.hubble.hubio import HubIO
getattr(HubIO(args), args.hub)()
def test_fetch(mocker, monkeypatch):
mock = mocker.Mock()
def _mock_get(url, headers=None):
mock(url=url)
return GetMockResponse(response_code=200)
monkeypatch.setattr(requests, 'get', _mock_get)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', None)
assert executor.uuid == 'dummy_mwu_encoder'
assert executor.name == 'alias_dummy'
assert executor.tag == 'v0'
assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder'
assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f'
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '')
assert executor.tag == 'v0'
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', 'v0.1')
assert executor.tag == 'v0.1'
def test_new(monkeypatch, tmpdir, add_dockerfile):
from rich.prompt import Prompt, Confirm
prompts = iter(
[
'DummyExecutor',
tmpdir / 'DummyExecutor',
'dummy description',
'dummy author',
'dummy tags',
'dummy docs',
]
)
confirms = iter([True, add_dockerfile])
def _mock_prompt_ask(*args, **kwargs):
return next(prompts)
def _mock_confirm_ask(*args, **kwargs):
return next(confirms)
monkeypatch.setattr(Prompt, 'ask', _mock_prompt_ask)
monkeypatch.setattr(Confirm, 'ask', _mock_confirm_ask)
args = argparse.Namespace(hub='new')
HubIO(args).new()
path = tmpdir / 'DummyExecutor'
pkg_files = [
'executor.py',
'manifest.yml',
'README.md',
'requirements.txt',
'config.yml',
]
if add_dockerfile:
pkg_files.append('Dockerfile')
for file in pkg_files:
assert (path / file).exists()
for file in [
'executor.py',
'manifest.yml',
'README.md',
'config.yml',
]:
with open(path / file, 'r') as fp:
assert 'DummyExecutor' in fp.read()
def test_pull_with_progress():
import json
args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder'])
def _log_stream_generator():
with open(os.path.join(cur_dir, 'docker_pull.logs')) as fin:
for line in fin:
if line.strip():
yield json.loads(line)
from rich.console import Console
console = Console()
HubIO(args)._pull_with_progress(_log_stream_generator(), console)
def test_push_with_authorization(mocker, monkeypatch, auth_token):
mock = mocker.Mock()
def _mock_post(url, data, headers, stream):
mock(url=url, headers=headers)
return PostMockResponse(response_code=200)
monkeypatch.setattr(requests, 'post', _mock_post)
exec_path = os.path.join(cur_dir, 'dummy_executor')
args = set_hub_push_parser().parse_args([exec_path])
HubIO(args).push()
# remove .jina
exec_config_path = os.path.join(exec_path, '.jina')
shutil.rmtree(exec_config_path)
assert mock.call_count == 1
_, kwargs = mock.call_args_list[0]
assert kwargs['headers'].get('Authorization') == f'token {auth_token}'
def test_new_with_arguments(
monkeypatch,
tmpdir,
add_dockerfile,
advance_configuration,
confirm_advance_configuration,
confirm_add_docker,
):
from rich.prompt import Confirm
path = os.path.join(tmpdir, 'DummyExecutor')
_args_list = [
'--name',
'argsExecutor',
'--description',
'args description',
'--keywords',
'args,keywords',
'--url',
'args url',
]
temp = []
_args_list.extend(['--path', path])
if advance_configuration:
_args_list.append('--advance-configuration')
else:
temp.append(confirm_advance_configuration)
if add_dockerfile:
_args_list.append('--add-dockerfile')
else:
temp.append(confirm_add_docker)
confirms = iter(temp)
def _mock_confirm_ask(*args, **kwargs):
return next(confirms)
monkeypatch.setattr(Confirm, 'ask', _mock_confirm_ask)
args = set_hub_new_parser().parse_args(_args_list)
HubIO(args).new()
# path = tmpdir / 'argsExecutor'
pkg_files = [
'executor.py',
'manifest.yml',
'README.md',
'requirements.txt',
'config.yml',
]
path = tmpdir / 'DummyExecutor'
if (advance_configuration
or confirm_advance_configuration) and (add_dockerfile
or confirm_add_docker):
pkg_files.append('Dockerfile')
for file in pkg_files:
assert (path / file).exists()
for file in ['executor.py', 'manifest.yml', 'README.md', 'config.yml']:
with open(path / file, 'r') as fp:
assert 'argsExecutor' in fp.read()
if advance_configuration or confirm_advance_configuration:
with open(path / 'manifest.yml') as fp:
temp = yaml.load(fp, Loader=yaml.FullLoader)
assert temp['name'] == 'argsExecutor'
assert temp['description'] == 'args description'
assert temp['keywords'] == ['args', 'keywords']
assert temp['url'] == 'args url'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile):
mock = mocker.Mock()
fail_meta_fetch = True
version = 'v0'
@disk_cache_offline(cache_file=str(tmpfile))
def _mock_fetch(
name,
tag=None,
secret=None,
image_required=True,
rebuild_image=True,
force=False,
):
mock(name=name)
if fail_meta_fetch:
raise urllib.error.URLError('Failed fetching meta')
else:
return HubExecutor(
uuid='dummy_mwu_encoder',
name='alias_dummy',
tag='v0',
image_name=f'jinahub/pod.dummy_mwu_encoder:{version}',
md5sum=None,
visibility=True,
archive_url=None,
)
def _gen_load_docker_client(fail_pull: bool):
def _load_docker_client(obj):
obj._raw_client = MockDockerClient(fail_pull=fail_pull)
obj._client = MockDockerClient(fail_pull=fail_pull)
return _load_docker_client
args = set_hub_pull_parser().parse_args(
['--force', 'jinahub+docker://dummy_mwu_encoder'])
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=True))
monkeypatch.setattr(HubIO, 'fetch_meta', _mock_fetch)
# Expect failure due to fetch_meta
with pytest.raises(urllib.error.URLError):
HubIO(args).pull()
fail_meta_fetch = False
# Expect failure due to image pull
with pytest.raises(AttributeError):
HubIO(args).pull()
# expect successful pull
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=False))
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v0'
version = 'v1'
# expect successful forced pull because force == True
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
# expect successful pull using cached fetch_meta response and saved image
fail_meta_fetch = True
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=False))
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
args.force_update = False
fail_meta_fetch = False
version = 'v2'
# expect successful but outdated pull because force == False
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
def test_push(mocker, monkeypatch, path, mode, tmpdir, force, tag, no_cache):
mock = mocker.Mock()
def _mock_post(url, data, headers=None, stream=True):
mock(url=url, data=data, headers=headers)
return PostMockResponse(response_code=requests.codes.created)
monkeypatch.setattr(requests, 'post', _mock_post)
# Second push will use --force --secret because of .jina/secret.key
# Then it will use put method
monkeypatch.setattr(requests, 'put', _mock_post)
exec_path = os.path.join(cur_dir, path)
_args_list = [exec_path, mode]
if force:
_args_list.extend(['--force', force])
if tag:
_args_list.append(tag)
if no_cache:
_args_list.append('--no-cache')
args = set_hub_push_parser().parse_args(_args_list)
result = HubIO(args).push()
# remove .jina
exec_config_path = os.path.join(exec_path, '.jina')
shutil.rmtree(exec_config_path)
_, mock_kwargs = mock.call_args_list[0]
c_type, c_data = cgi.parse_header(mock_kwargs['headers']['Content-Type'])
assert c_type == 'multipart/form-data'
form_data = cgi.parse_multipart(BytesIO(mock_kwargs['data']),
{'boundary': c_data['boundary'].encode()})
assert 'file' in form_data
assert 'md5sum' in form_data
if force:
assert form_data['id'] == ['UUID8']
else:
assert form_data.get('id') is None
if mode == '--private':
assert form_data['private'] == ['True']
assert form_data['public'] == ['False']
else:
assert form_data['private'] == ['False']
assert form_data['public'] == ['True']
if tag:
assert form_data['tags'] == [' v0']
else:
assert form_data.get('tags') is None
if no_cache:
assert form_data['buildWithNoCache'] == ['True']
else:
assert form_data.get('buildWithNoCache') is None