def test_pull(test_envs, mocker, monkeypatch):
mock = mocker.Mock()
def _mock_fetch(name, tag=None, secret=None):
mock(name=name)
return HubExecutor(
uuid='dummy_mwu_encoder',
alias='alias_dummy',
tag='v0',
image_name='jinahub/pod.dummy_mwu_encoder',
md5sum=None,
visibility=True,
archive_url=None,
)
monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch)
def _mock_download(url, stream=True, headers=None):
mock(url=url)
return DownloadMockResponse(response_code=200)
def _mock_head(url):
from collections import namedtuple
HeadInfo = namedtuple('HeadInfo', ['headers'])
return HeadInfo(headers={})
monkeypatch.setattr(requests, 'get', _mock_download)
monkeypatch.setattr(requests, 'head', _mock_head)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
HubIO(args).pull()
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder:secret'])
HubIO(args).pull()
def test_fetch(mocker, monkeypatch, rebuild_image):
mock = mocker.Mock()
def _mock_post(url, json, headers=None):
mock(url=url, json=json)
return FetchMetaMockResponse(response_code=200)
monkeypatch.setattr(requests, 'post', _mock_post)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder',
None,
rebuild_image=rebuild_image,
force=True)
assert executor.uuid == 'dummy_mwu_encoder'
assert executor.name == 'alias_dummy'
assert executor.tag == 'v0'
assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder'
assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f'
_, mock_kwargs = mock.call_args_list[0]
assert mock_kwargs['json']['rebuildImage'] is rebuild_image
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '', force=True)
assert executor.tag == 'v0'
_, mock_kwargs = mock.call_args_list[1]
assert mock_kwargs['json'][
'rebuildImage'] is True # default value must be True
executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder',
'v0.1',
force=True)
assert executor.tag == 'v0.1'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile):
mock = mocker.Mock()
fail_meta_fetch = True
@disk_cache_offline(cache_file=str(tmpfile))
def _mock_fetch(name, tag=None, secret=None):
mock(name=name)
if fail_meta_fetch:
raise urllib.error.URLError('Failed fetching meta')
else:
return HubExecutor(
uuid='dummy_mwu_encoder',
alias='alias_dummy',
tag='v0',
image_name='jinahub/pod.dummy_mwu_encoder',
md5sum=None,
visibility=True,
archive_url=None,
)
def _gen_load_docker_client(fail_pull: bool):
def _load_docker_client(obj):
obj._raw_client = MockDockerClient(fail_pull=fail_pull)
return _load_docker_client
args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder'])
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=True),
)
monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch)
# Expect failure due to _fetch_meta
with pytest.raises(urllib.error.URLError):
HubIO(args).pull()
fail_meta_fetch = False
# Expect failure due to image pull
with pytest.raises(docker.errors.APIError):
HubIO(args).pull()
# expect successful pull
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=False),
)
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
# expect successful pull using cached _fetch_meta response and saved image
fail_meta_fetch = True
monkeypatch.setattr(
HubIO,
'_load_docker_client',
_gen_load_docker_client(fail_pull=False),
)
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
def from_hub(
cls: Type[T],
uri: str,
context: Optional[Dict[str, Any]] = None,
uses_with: Optional[Dict] = None,
uses_metas: Optional[Dict] = None,
uses_requests: Optional[Dict] = None,
**kwargs,
) -> T:
"""Construct an Executor from Hub.
:param uri: a hub Executor scheme starts with `jinahub://`
:param context: context replacement variables in a dict, the value of the dict is the replacement.
:param uses_with: dictionary of parameters to overwrite from the default config's with field
:param uses_metas: dictionary of parameters to overwrite from the default config's metas field
:param uses_requests: dictionary of parameters to overwrite from the default config's requests field
:param kwargs: other kwargs accepted by the CLI ``jina hub pull``
:return: the Hub Executor object.
.. highlight:: python
.. code-block:: python
from jina import Executor
from docarray import Document, DocumentArray
executor = Executor.from_hub(
uri='jinahub://CLIPImageEncoder', install_requirements=True
)
"""
from jina.hubble.helper import is_valid_huburi
_source = None
if is_valid_huburi(uri):
from jina.hubble.hubio import HubIO
from jina.parsers.hubble import set_hub_pull_parser
_args = ArgNamespace.kwargs2namespace(
{
'no_usage': True,
**kwargs
},
set_hub_pull_parser(),
positional_args=(uri, ),
)
_source = HubIO(args=_args).pull()
if not _source or _source.startswith('docker://'):
raise ValueError(
f'Can not construct a native Executor from {uri}. Looks like you want to use it as a '
f'Docker container, you may want to use it in the Flow via `.add(uses={uri})` instead.'
)
return cls.load_config(
_source,
context=context,
uses_with=uses_with,
uses_metas=uses_metas,
uses_requests=uses_requests,
)
def test_pull_with_progress():
import json
args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder'])
def _log_stream_generator():
with open(os.path.join(cur_dir, 'docker_pull.logs')) as fin:
for line in fin:
if line.strip():
yield json.loads(line)
from rich.console import Console
console = Console()
HubIO(args)._pull_with_progress(_log_stream_generator(), console)
def test_fetch(mocker, monkeypatch):
mock = mocker.Mock()
def _mock_get(url, headers=None):
mock(url=url)
return GetMockResponse(response_code=200)
monkeypatch.setattr(requests, 'get', _mock_get)
args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder'])
executor = HubIO(args)._fetch_meta('dummy_mwu_encoder')
assert executor.uuid == 'dummy_mwu_encoder'
assert executor.alias == 'alias_dummy'
assert executor.tag == 'v0'
assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder'
assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile):
mock = mocker.Mock()
fail_meta_fetch = True
version = 'v0'
@disk_cache_offline(cache_file=str(tmpfile))
def _mock_fetch(
name,
tag=None,
secret=None,
image_required=True,
rebuild_image=True,
force=False,
):
mock(name=name)
if fail_meta_fetch:
raise urllib.error.URLError('Failed fetching meta')
else:
return HubExecutor(
uuid='dummy_mwu_encoder',
name='alias_dummy',
tag='v0',
image_name=f'jinahub/pod.dummy_mwu_encoder:{version}',
md5sum=None,
visibility=True,
archive_url=None,
)
def _gen_load_docker_client(fail_pull: bool):
def _load_docker_client(obj):
obj._raw_client = MockDockerClient(fail_pull=fail_pull)
obj._client = MockDockerClient(fail_pull=fail_pull)
return _load_docker_client
args = set_hub_pull_parser().parse_args(
['--force', 'jinahub+docker://dummy_mwu_encoder'])
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=True))
monkeypatch.setattr(HubIO, 'fetch_meta', _mock_fetch)
# Expect failure due to fetch_meta
with pytest.raises(urllib.error.URLError):
HubIO(args).pull()
fail_meta_fetch = False
# Expect failure due to image pull
with pytest.raises(AttributeError):
HubIO(args).pull()
# expect successful pull
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=False))
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v0'
version = 'v1'
# expect successful forced pull because force == True
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
# expect successful pull using cached fetch_meta response and saved image
fail_meta_fetch = True
monkeypatch.setattr(HubIO, '_load_docker_client',
_gen_load_docker_client(fail_pull=False))
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
args.force_update = False
fail_meta_fetch = False
version = 'v2'
# expect successful but outdated pull because force == False
assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
