def test_pull(test_envs, mocker, monkeypatch): mock = mocker.Mock() def _mock_fetch(name, tag=None, secret=None): mock(name=name) return HubExecutor( uuid='dummy_mwu_encoder', alias='alias_dummy', tag='v0', image_name='jinahub/pod.dummy_mwu_encoder', md5sum=None, visibility=True, archive_url=None, ) monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch) def _mock_download(url, stream=True, headers=None): mock(url=url) return DownloadMockResponse(response_code=200) def _mock_head(url): from collections import namedtuple HeadInfo = namedtuple('HeadInfo', ['headers']) return HeadInfo(headers={}) monkeypatch.setattr(requests, 'get', _mock_download) monkeypatch.setattr(requests, 'head', _mock_head) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) HubIO(args).pull() args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder:secret']) HubIO(args).pull()
def test_fetch(mocker, monkeypatch, rebuild_image): mock = mocker.Mock() def _mock_post(url, json, headers=None): mock(url=url, json=json) return FetchMetaMockResponse(response_code=200) monkeypatch.setattr(requests, 'post', _mock_post) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', None, rebuild_image=rebuild_image, force=True) assert executor.uuid == 'dummy_mwu_encoder' assert executor.name == 'alias_dummy' assert executor.tag == 'v0' assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder' assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f' _, mock_kwargs = mock.call_args_list[0] assert mock_kwargs['json']['rebuildImage'] is rebuild_image executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '', force=True) assert executor.tag == 'v0' _, mock_kwargs = mock.call_args_list[1] assert mock_kwargs['json'][ 'rebuildImage'] is True # default value must be True executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', 'v0.1', force=True) assert executor.tag == 'v0.1'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile): mock = mocker.Mock() fail_meta_fetch = True @disk_cache_offline(cache_file=str(tmpfile)) def _mock_fetch(name, tag=None, secret=None): mock(name=name) if fail_meta_fetch: raise urllib.error.URLError('Failed fetching meta') else: return HubExecutor( uuid='dummy_mwu_encoder', alias='alias_dummy', tag='v0', image_name='jinahub/pod.dummy_mwu_encoder', md5sum=None, visibility=True, archive_url=None, ) def _gen_load_docker_client(fail_pull: bool): def _load_docker_client(obj): obj._raw_client = MockDockerClient(fail_pull=fail_pull) return _load_docker_client args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder']) monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=True), ) monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch) # Expect failure due to _fetch_meta with pytest.raises(urllib.error.URLError): HubIO(args).pull() fail_meta_fetch = False # Expect failure due to image pull with pytest.raises(docker.errors.APIError): HubIO(args).pull() # expect successful pull monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False), ) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder' # expect successful pull using cached _fetch_meta response and saved image fail_meta_fetch = True monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False), ) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
def from_hub( cls: Type[T], uri: str, context: Optional[Dict[str, Any]] = None, uses_with: Optional[Dict] = None, uses_metas: Optional[Dict] = None, uses_requests: Optional[Dict] = None, **kwargs, ) -> T: """Construct an Executor from Hub. :param uri: a hub Executor scheme starts with `jinahub://` :param context: context replacement variables in a dict, the value of the dict is the replacement. :param uses_with: dictionary of parameters to overwrite from the default config's with field :param uses_metas: dictionary of parameters to overwrite from the default config's metas field :param uses_requests: dictionary of parameters to overwrite from the default config's requests field :param kwargs: other kwargs accepted by the CLI ``jina hub pull`` :return: the Hub Executor object. .. highlight:: python .. code-block:: python from jina import Executor from docarray import Document, DocumentArray executor = Executor.from_hub( uri='jinahub://CLIPImageEncoder', install_requirements=True ) """ from jina.hubble.helper import is_valid_huburi _source = None if is_valid_huburi(uri): from jina.hubble.hubio import HubIO from jina.parsers.hubble import set_hub_pull_parser _args = ArgNamespace.kwargs2namespace( { 'no_usage': True, **kwargs }, set_hub_pull_parser(), positional_args=(uri, ), ) _source = HubIO(args=_args).pull() if not _source or _source.startswith('docker://'): raise ValueError( f'Can not construct a native Executor from {uri}. Looks like you want to use it as a ' f'Docker container, you may want to use it in the Flow via `.add(uses={uri})` instead.' ) return cls.load_config( _source, context=context, uses_with=uses_with, uses_metas=uses_metas, uses_requests=uses_requests, )
def test_pull_with_progress(): import json args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder']) def _log_stream_generator(): with open(os.path.join(cur_dir, 'docker_pull.logs')) as fin: for line in fin: if line.strip(): yield json.loads(line) from rich.console import Console console = Console() HubIO(args)._pull_with_progress(_log_stream_generator(), console)
def test_fetch(mocker, monkeypatch): mock = mocker.Mock() def _mock_get(url, headers=None): mock(url=url) return GetMockResponse(response_code=200) monkeypatch.setattr(requests, 'get', _mock_get) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) executor = HubIO(args)._fetch_meta('dummy_mwu_encoder') assert executor.uuid == 'dummy_mwu_encoder' assert executor.alias == 'alias_dummy' assert executor.tag == 'v0' assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder' assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile): mock = mocker.Mock() fail_meta_fetch = True version = 'v0' @disk_cache_offline(cache_file=str(tmpfile)) def _mock_fetch( name, tag=None, secret=None, image_required=True, rebuild_image=True, force=False, ): mock(name=name) if fail_meta_fetch: raise urllib.error.URLError('Failed fetching meta') else: return HubExecutor( uuid='dummy_mwu_encoder', name='alias_dummy', tag='v0', image_name=f'jinahub/pod.dummy_mwu_encoder:{version}', md5sum=None, visibility=True, archive_url=None, ) def _gen_load_docker_client(fail_pull: bool): def _load_docker_client(obj): obj._raw_client = MockDockerClient(fail_pull=fail_pull) obj._client = MockDockerClient(fail_pull=fail_pull) return _load_docker_client args = set_hub_pull_parser().parse_args( ['--force', 'jinahub+docker://dummy_mwu_encoder']) monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=True)) monkeypatch.setattr(HubIO, 'fetch_meta', _mock_fetch) # Expect failure due to fetch_meta with pytest.raises(urllib.error.URLError): HubIO(args).pull() fail_meta_fetch = False # Expect failure due to image pull with pytest.raises(AttributeError): HubIO(args).pull() # expect successful pull monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False)) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v0' version = 'v1' # expect successful forced pull because force == True assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1' # expect successful pull using cached fetch_meta response and saved image fail_meta_fetch = True monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False)) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1' args.force_update = False fail_meta_fetch = False version = 'v2' # expect successful but outdated pull because force == False assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'