def test_pull(test_envs, mocker, monkeypatch): mock = mocker.Mock() def _mock_fetch(name, tag=None, secret=None): mock(name=name) return HubExecutor( uuid='dummy_mwu_encoder', alias='alias_dummy', tag='v0', image_name='jinahub/pod.dummy_mwu_encoder', md5sum=None, visibility=True, archive_url=None, ) monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch) def _mock_download(url, stream=True, headers=None): mock(url=url) return DownloadMockResponse(response_code=200) def _mock_head(url): from collections import namedtuple HeadInfo = namedtuple('HeadInfo', ['headers']) return HeadInfo(headers={}) monkeypatch.setattr(requests, 'get', _mock_download) monkeypatch.setattr(requests, 'head', _mock_head) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) HubIO(args).pull() args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder:secret']) HubIO(args).pull()
def test_fetch(mocker, monkeypatch, rebuild_image): mock = mocker.Mock() def _mock_post(url, json, headers=None): mock(url=url, json=json) return FetchMetaMockResponse(response_code=200) monkeypatch.setattr(requests, 'post', _mock_post) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', None, rebuild_image=rebuild_image, force=True) assert executor.uuid == 'dummy_mwu_encoder' assert executor.name == 'alias_dummy' assert executor.tag == 'v0' assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder' assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f' _, mock_kwargs = mock.call_args_list[0] assert mock_kwargs['json']['rebuildImage'] is rebuild_image executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '', force=True) assert executor.tag == 'v0' _, mock_kwargs = mock.call_args_list[1] assert mock_kwargs['json'][ 'rebuildImage'] is True # default value must be True executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', 'v0.1', force=True) assert executor.tag == 'v0.1'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile): mock = mocker.Mock() fail_meta_fetch = True @disk_cache_offline(cache_file=str(tmpfile)) def _mock_fetch(name, tag=None, secret=None): mock(name=name) if fail_meta_fetch: raise urllib.error.URLError('Failed fetching meta') else: return HubExecutor( uuid='dummy_mwu_encoder', alias='alias_dummy', tag='v0', image_name='jinahub/pod.dummy_mwu_encoder', md5sum=None, visibility=True, archive_url=None, ) def _gen_load_docker_client(fail_pull: bool): def _load_docker_client(obj): obj._raw_client = MockDockerClient(fail_pull=fail_pull) return _load_docker_client args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder']) monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=True), ) monkeypatch.setattr(HubIO, '_fetch_meta', _mock_fetch) # Expect failure due to _fetch_meta with pytest.raises(urllib.error.URLError): HubIO(args).pull() fail_meta_fetch = False # Expect failure due to image pull with pytest.raises(docker.errors.APIError): HubIO(args).pull() # expect successful pull monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False), ) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder' # expect successful pull using cached _fetch_meta response and saved image fail_meta_fetch = True monkeypatch.setattr( HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False), ) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder'
def from_hub( cls: Type[T], uri: str, context: Optional[Dict[str, Any]] = None, uses_with: Optional[Dict] = None, uses_metas: Optional[Dict] = None, uses_requests: Optional[Dict] = None, **kwargs, ) -> T: """Construct an Executor from Hub. :param uri: a hub Executor scheme starts with `jinahub://` :param context: context replacement variables in a dict, the value of the dict is the replacement. :param uses_with: dictionary of parameters to overwrite from the default config's with field :param uses_metas: dictionary of parameters to overwrite from the default config's metas field :param uses_requests: dictionary of parameters to overwrite from the default config's requests field :param kwargs: other kwargs accepted by the CLI ``jina hub pull`` :return: the Hub Executor object. .. highlight:: python .. code-block:: python from jina import Executor from docarray import Document, DocumentArray executor = Executor.from_hub( uri='jinahub://CLIPImageEncoder', install_requirements=True ) """ from jina.hubble.helper import is_valid_huburi _source = None if is_valid_huburi(uri): from jina.hubble.hubio import HubIO from jina.parsers.hubble import set_hub_pull_parser _args = ArgNamespace.kwargs2namespace( { 'no_usage': True, **kwargs }, set_hub_pull_parser(), positional_args=(uri, ), ) _source = HubIO(args=_args).pull() if not _source or _source.startswith('docker://'): raise ValueError( f'Can not construct a native Executor from {uri}. Looks like you want to use it as a ' f'Docker container, you may want to use it in the Flow via `.add(uses={uri})` instead.' ) return cls.load_config( _source, context=context, uses_with=uses_with, uses_metas=uses_metas, uses_requests=uses_requests, )
def test_fetch_with_authorization(mocker, monkeypatch, auth_token): mock = mocker.Mock() def _mock_post(url, json, headers): mock(url=url, json=json, headers=headers) return FetchMetaMockResponse(response_code=200) monkeypatch.setattr(requests, 'post', _mock_post) HubIO.fetch_meta('dummy_mwu_encoder', tag=None, force=True) assert mock.call_count == 1 _, kwargs = mock.call_args_list[0] assert kwargs['headers'].get('Authorization') == f'token {auth_token}'
def test_push_wrong_dockerfile( mocker, monkeypatch, path, mode, tmpdir, dockerfile, expected_error ): dockerfile = os.path.join(cur_dir, path, dockerfile) mock = mocker.Mock() def _mock_post(url, data, headers=None, stream=True): mock(url=url, data=data) return PostMockResponse(response_code=requests.codes.created) monkeypatch.setattr(requests, 'post', _mock_post) # Second push will use --force --secret because of .jina/secret.key # Then it will use put method monkeypatch.setattr(requests, 'put', _mock_post) exec_path = os.path.join(cur_dir, path) _args_list = [exec_path, mode] args = set_hub_push_parser().parse_args(_args_list) args.dockerfile = dockerfile with pytest.raises(Exception) as info: HubIO(args).push() assert expected_error.format(dockerfile=dockerfile, work_path=args.path) in str( info.value )
def update_runtime_cls(args, copy=False) -> 'Namespace': """Get runtime_cls as a string from args :param args: pod/deployment namespace args :param copy: True if args shouldn't be modified in-place :return: runtime class as a string """ _args = deepcopy(args) if copy else args gateway_runtime_dict = { GatewayProtocolType.GRPC: 'GRPCGatewayRuntime', GatewayProtocolType.WEBSOCKET: 'WebSocketGatewayRuntime', GatewayProtocolType.HTTP: 'HTTPGatewayRuntime', } if _args.runtime_cls == 'WorkerRuntime' and is_valid_huburi(_args.uses): _hub_args = deepcopy(_args) _hub_args.uri = _args.uses _hub_args.no_usage = True _args.uses = HubIO(_hub_args).pull() if hasattr(_args, 'protocol'): _args.runtime_cls = gateway_runtime_dict[_args.protocol] if _args.pod_role == PodRoleType.HEAD: _args.runtime_cls = 'HeadRuntime' return _args
def test_deploy_public_sandbox_existing(mocker, monkeypatch): mock = mocker.Mock() def _mock_post(url, json, headers=None): mock(url=url, json=json) return SandboxGetMockResponse(response_code=200) monkeypatch.setattr(requests, "post", _mock_post) args = Namespace( uses='jinahub+sandbox://dummy_mwu_encoder', uses_with={'foo': 'bar'}, test_string='text', test_number=1, ) host, port = HubIO.deploy_public_sandbox(args) assert host == 'http://test_existing_deployment.com' assert port == 4321 _, kwargs = mock.call_args assert kwargs['json']['args'] == { 'uses_with': { 'foo': 'bar' }, 'test_number': 1, 'test_string': 'text', }
def test_push(mocker, monkeypatch, path, mode, tmpdir, force, tag): mock = mocker.Mock() def _mock_post(url, data, headers=None, stream=True): mock(url=url, data=data) return PostMockResponse(response_code=requests.codes.created) monkeypatch.setattr(requests, 'post', _mock_post) # Second push will use --force --secret because of .jina/secret.key # Then it will use put method monkeypatch.setattr(requests, 'put', _mock_post) exec_path = os.path.join(cur_dir, path) _args_list = [exec_path, mode] if force: _args_list.extend(['--force', force]) if tag: _args_list.append(tag) args = set_hub_push_parser().parse_args(_args_list) result = HubIO(args).push() # remove .jina exec_config_path = os.path.join(exec_path, '.jina') shutil.rmtree(exec_config_path)
def get_image_name(uses: str) -> str: """The image can be provided in different formats by the user. This function converts it to an image name which can be understood by k8s. It uses the Hub api to get the image name and the latest tag on Docker Hub. If you don't want to rebuild image on Jina Hub, you can set `JINA_HUB_NO_IMAGE_REBUILD` environment variable. :param uses: image name :return: normalized image name """ try: rebuild_image = 'JINA_HUB_NO_IMAGE_REBUILD' not in os.environ scheme, name, tag, secret = parse_hub_uri(uses) meta_data, _ = HubIO.fetch_meta(name, tag, secret=secret, rebuild_image=rebuild_image, force=True) image_name = meta_data.image_name return image_name except Exception: if uses.startswith('docker'): # docker:// is a valid requirement and user may want to put its own image return uses.replace('docker://', '') raise
def build_pod(args: 'Namespace') -> Type['BasePod']: """Build an implementation of a `BasePod` interface :param args: deployment arguments parsed from the CLI. :return: the created BaseDeployment """ # copy to update but forward original cargs = deepcopy(args) if cargs.host != __default_host__ and not cargs.disable_remote: cargs.timeout_ready = -1 return JinaDPod(cargs) if is_valid_huburi(cargs.uses): _hub_args = deepcopy(args) _hub_args.uri = args.uses _hub_args.no_usage = True cargs.uses = HubIO(_hub_args).pull() if ( cargs.pod_role != PodRoleType.HEAD and cargs.uses and cargs.uses.startswith('docker://') ): return ContainerPod(cargs) else: return Pod(args)
def hub(args: 'Namespace'): """ Start a hub builder for push, pull :param args: arguments coming from the CLI. """ from jina.hubble.hubio import HubIO getattr(HubIO(args), args.hub)()
def update_sandbox_args(self): """Update args of all its pods based on the host and port returned by Hubble""" if self.is_sandbox: host, port = HubIO.deploy_public_sandbox(self.args) self._sandbox_deployed = True self.first_pod_args.host = host self.first_pod_args.port = port if self.head_args: self.pod_args['head'].host = host self.pod_args['head'].port = port
def test_fetch_with_no_image(mocker, monkeypatch): mock = mocker.Mock() def _mock_post(url, json, headers=None): mock(url=url, json=json) return FetchMetaMockResponse(response_code=200, no_image=True) monkeypatch.setattr(requests, 'post', _mock_post) with pytest.raises(Exception) as exc_info: HubIO.fetch_meta('dummy_mwu_encoder', tag=None, force=True) assert exc_info.match('No image found for executor "dummy_mwu_encoder"') executor, _ = HubIO.fetch_meta( 'dummy_mwu_encoder', tag=None, image_required=False, force=True ) assert executor.image_name is None assert mock.call_count == 2
def test_fetch_with_retry(mocker, monkeypatch): mock = mocker.Mock() mock_response = FetchMetaMockResponse(response_code=200, fail_count=3) def _mock_post(url, json, headers=None): mock(url=url, json=json) return mock_response monkeypatch.setattr(requests, 'post', _mock_post) with pytest.raises(Exception) as exc_info: # failing 3 times, so it should raise an error HubIO.fetch_meta('dummy_mwu_encoder', tag=None, force=True) assert exc_info.match('{"message": "Internal server error"}') mock_response = FetchMetaMockResponse(response_code=200, fail_count=2) # failing 2 times, it must succeed on 3rd time executor, _ = HubIO.fetch_meta('dummy_mwu_encoder', tag=None, force=True) assert executor.uuid == 'dummy_mwu_encoder' assert mock.call_count == 6 # mock must be called 3+3
def test_fetch(mocker, monkeypatch): mock = mocker.Mock() def _mock_get(url, headers=None): mock(url=url) return GetMockResponse(response_code=200) monkeypatch.setattr(requests, 'get', _mock_get) args = set_hub_pull_parser().parse_args(['jinahub://dummy_mwu_encoder']) executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', None) assert executor.uuid == 'dummy_mwu_encoder' assert executor.name == 'alias_dummy' assert executor.tag == 'v0' assert executor.image_name == 'jinahub/pod.dummy_mwu_encoder' assert executor.md5sum == 'ecbe3fdd9cbe25dbb85abaaf6c54ec4f' executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', '') assert executor.tag == 'v0' executor, _ = HubIO(args).fetch_meta('dummy_mwu_encoder', 'v0.1') assert executor.tag == 'v0.1'
def test_new(monkeypatch, tmpdir, add_dockerfile): from rich.prompt import Prompt, Confirm prompts = iter( [ 'DummyExecutor', tmpdir / 'DummyExecutor', 'dummy description', 'dummy author', 'dummy tags', 'dummy docs', ] ) confirms = iter([True, add_dockerfile]) def _mock_prompt_ask(*args, **kwargs): return next(prompts) def _mock_confirm_ask(*args, **kwargs): return next(confirms) monkeypatch.setattr(Prompt, 'ask', _mock_prompt_ask) monkeypatch.setattr(Confirm, 'ask', _mock_confirm_ask) args = argparse.Namespace(hub='new') HubIO(args).new() path = tmpdir / 'DummyExecutor' pkg_files = [ 'executor.py', 'manifest.yml', 'README.md', 'requirements.txt', 'config.yml', ] if add_dockerfile: pkg_files.append('Dockerfile') for file in pkg_files: assert (path / file).exists() for file in [ 'executor.py', 'manifest.yml', 'README.md', 'config.yml', ]: with open(path / file, 'r') as fp: assert 'DummyExecutor' in fp.read()
def update_pod_args(self): """ Update args of all its pods based on Deployment args. Including head/tail""" if isinstance(self.args, Dict): # This is used when a Deployment is created in a remote context, where pods & their connections are already given. self.pod_args = self.args else: self.pod_args = self._parse_args(self.args) if self.is_sandbox: host, port = HubIO.deploy_public_sandbox( getattr(self.args, 'uses', '')) self.first_pod_args.host = host self.first_pod_args.port_in = port self.pod_args['head'].host = host self.pod_args['head'].port_in = port
def test_pull_with_progress(): import json args = set_hub_pull_parser().parse_args(['jinahub+docker://dummy_mwu_encoder']) def _log_stream_generator(): with open(os.path.join(cur_dir, 'docker_pull.logs')) as fin: for line in fin: if line.strip(): yield json.loads(line) from rich.console import Console console = Console() HubIO(args)._pull_with_progress(_log_stream_generator(), console)
def get_image_name(uses: str) -> str: """The image can be provided in different formats by the user. This function converts it to an image name which can be understood by k8s. It uses the Hub api to get the image name and the latest tag on Docker Hub. :param uses: image name :return: normalized image name """ try: scheme, name, tag, secret = parse_hub_uri(uses) meta_data = HubIO.fetch_meta(name, tag, secret=secret) image_name = meta_data.image_name return image_name except Exception: return uses.replace('docker://', '')
def test_deploy_public_sandbox_create_new(mocker, monkeypatch): mock = mocker.Mock() def _mock_post(url, json, headers=None): mock(url=url, json=json) if url.endswith('/sandbox.get'): return SandboxGetMockResponse(response_code=404) else: return SandboxCreateMockResponse(response_code=requests.codes.created) monkeypatch.setattr(requests, 'post', _mock_post) args = Namespace(uses='jinahub+sandbox://dummy_mwu_encoder') host, port = HubIO.deploy_public_sandbox(args) assert host == 'http://test_new_deployment.com' assert port == 4322
def get_image_name(uses: str) -> str: """The image can be provided in different formats by the user. This function converts it to an image name which can be understood by k8s. It uses the Hub api to get the image name and the latest tag on Docker Hub. :param uses: image name :return: normalized image name """ try: scheme, name, tag, secret = parse_hub_uri(uses) meta_data, _ = HubIO.fetch_meta(name, tag, secret=secret, force=True) image_name = meta_data.image_name return image_name except Exception: if uses.startswith('docker'): # docker:// is a valid requirement and user may want to put its own image return uses.replace('docker://', '') raise
def test_push_with_authorization(mocker, monkeypatch, auth_token): mock = mocker.Mock() def _mock_post(url, data, headers, stream): mock(url=url, headers=headers) return PostMockResponse(response_code=200) monkeypatch.setattr(requests, 'post', _mock_post) exec_path = os.path.join(cur_dir, 'dummy_executor') args = set_hub_push_parser().parse_args([exec_path]) HubIO(args).push() # remove .jina exec_config_path = os.path.join(exec_path, '.jina') shutil.rmtree(exec_config_path) assert mock.call_count == 1 _, kwargs = mock.call_args_list[0] assert kwargs['headers'].get('Authorization') == f'token {auth_token}'
def test_new_with_arguments( monkeypatch, tmpdir, add_dockerfile, advance_configuration, confirm_advance_configuration, confirm_add_docker, ): from rich.prompt import Confirm path = os.path.join(tmpdir, 'DummyExecutor') _args_list = [ '--name', 'argsExecutor', '--description', 'args description', '--keywords', 'args,keywords', '--url', 'args url', ] temp = [] _args_list.extend(['--path', path]) if advance_configuration: _args_list.append('--advance-configuration') else: temp.append(confirm_advance_configuration) if add_dockerfile: _args_list.append('--add-dockerfile') else: temp.append(confirm_add_docker) confirms = iter(temp) def _mock_confirm_ask(*args, **kwargs): return next(confirms) monkeypatch.setattr(Confirm, 'ask', _mock_confirm_ask) args = set_hub_new_parser().parse_args(_args_list) HubIO(args).new() # path = tmpdir / 'argsExecutor' pkg_files = [ 'executor.py', 'manifest.yml', 'README.md', 'requirements.txt', 'config.yml', ] path = tmpdir / 'DummyExecutor' if (advance_configuration or confirm_advance_configuration) and (add_dockerfile or confirm_add_docker): pkg_files.append('Dockerfile') for file in pkg_files: assert (path / file).exists() for file in ['executor.py', 'manifest.yml', 'README.md', 'config.yml']: with open(path / file, 'r') as fp: assert 'argsExecutor' in fp.read() if advance_configuration or confirm_advance_configuration: with open(path / 'manifest.yml') as fp: temp = yaml.load(fp, Loader=yaml.FullLoader) assert temp['name'] == 'argsExecutor' assert temp['description'] == 'args description' assert temp['keywords'] == ['args', 'keywords'] assert temp['url'] == 'args url'
def test_offline_pull(test_envs, mocker, monkeypatch, tmpfile): mock = mocker.Mock() fail_meta_fetch = True version = 'v0' @disk_cache_offline(cache_file=str(tmpfile)) def _mock_fetch( name, tag=None, secret=None, image_required=True, rebuild_image=True, force=False, ): mock(name=name) if fail_meta_fetch: raise urllib.error.URLError('Failed fetching meta') else: return HubExecutor( uuid='dummy_mwu_encoder', name='alias_dummy', tag='v0', image_name=f'jinahub/pod.dummy_mwu_encoder:{version}', md5sum=None, visibility=True, archive_url=None, ) def _gen_load_docker_client(fail_pull: bool): def _load_docker_client(obj): obj._raw_client = MockDockerClient(fail_pull=fail_pull) obj._client = MockDockerClient(fail_pull=fail_pull) return _load_docker_client args = set_hub_pull_parser().parse_args( ['--force', 'jinahub+docker://dummy_mwu_encoder']) monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=True)) monkeypatch.setattr(HubIO, 'fetch_meta', _mock_fetch) # Expect failure due to fetch_meta with pytest.raises(urllib.error.URLError): HubIO(args).pull() fail_meta_fetch = False # Expect failure due to image pull with pytest.raises(AttributeError): HubIO(args).pull() # expect successful pull monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False)) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v0' version = 'v1' # expect successful forced pull because force == True assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1' # expect successful pull using cached fetch_meta response and saved image fail_meta_fetch = True monkeypatch.setattr(HubIO, '_load_docker_client', _gen_load_docker_client(fail_pull=False)) assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1' args.force_update = False fail_meta_fetch = False version = 'v2' # expect successful but outdated pull because force == False assert HubIO(args).pull() == 'docker://jinahub/pod.dummy_mwu_encoder:v1'
def test_push(mocker, monkeypatch, path, mode, tmpdir, force, tag, no_cache): mock = mocker.Mock() def _mock_post(url, data, headers=None, stream=True): mock(url=url, data=data, headers=headers) return PostMockResponse(response_code=requests.codes.created) monkeypatch.setattr(requests, 'post', _mock_post) # Second push will use --force --secret because of .jina/secret.key # Then it will use put method monkeypatch.setattr(requests, 'put', _mock_post) exec_path = os.path.join(cur_dir, path) _args_list = [exec_path, mode] if force: _args_list.extend(['--force', force]) if tag: _args_list.append(tag) if no_cache: _args_list.append('--no-cache') args = set_hub_push_parser().parse_args(_args_list) result = HubIO(args).push() # remove .jina exec_config_path = os.path.join(exec_path, '.jina') shutil.rmtree(exec_config_path) _, mock_kwargs = mock.call_args_list[0] c_type, c_data = cgi.parse_header(mock_kwargs['headers']['Content-Type']) assert c_type == 'multipart/form-data' form_data = cgi.parse_multipart(BytesIO(mock_kwargs['data']), {'boundary': c_data['boundary'].encode()}) assert 'file' in form_data assert 'md5sum' in form_data if force: assert form_data['id'] == ['UUID8'] else: assert form_data.get('id') is None if mode == '--private': assert form_data['private'] == ['True'] assert form_data['public'] == ['False'] else: assert form_data['private'] == ['False'] assert form_data['public'] == ['True'] if tag: assert form_data['tags'] == [' v0'] else: assert form_data.get('tags') is None if no_cache: assert form_data['buildWithNoCache'] == ['True'] else: assert form_data.get('buildWithNoCache') is None