def test_install_local(test_envs, executor_zip_file, test_executor):
assert not hubapi.exist_local(test_executor.uuid, test_executor.tag)
hubapi.install_local(executor_zip_file, test_executor)
assert hubapi.exist_local(test_executor.uuid, test_executor.tag)
hubapi.uninstall_local(test_executor.uuid)
assert not hubapi.exist_local(test_executor.uuid, test_executor.tag)
def local_hub_executor(tmpdir, test_envs):
from jina.hubble import hubapi, helper, HubExecutor
hubapi._hub_root = Path(os.environ.get('JINA_HUB_ROOT'))
pkg_path = Path(__file__).parent / 'dummyhub'
stream_data = helper.archive_package(pkg_path)
with open(tmpdir / 'dummy_test.zip', 'wb') as temp_zip_file:
temp_zip_file.write(stream_data.getvalue())
hubapi.install_local(
Path(tmpdir) / 'dummy_test.zip', HubExecutor(uuid='hello', tag='v0'))
def test_install_local(test_envs, executor_zip_file, test_executor, install_deps):
assert not hubapi.exist_local(test_executor.uuid, test_executor.tag)
hubapi.install_local(executor_zip_file, test_executor, install_deps=install_deps)
assert hubapi.exist_local(test_executor.uuid, test_executor.tag)
assert any(
str(path).endswith(
f'{os.path.join(test_executor.uuid, test_executor.tag)}.dist-info'
)
for path in list_local()
)
hubapi.uninstall_local(test_executor.uuid)
assert not hubapi.exist_local(test_executor.uuid, test_executor.tag)
def pull(self) -> str:
"""Pull the executor package from Jina Hub.
:return: the `uses` string
"""
from rich.console import Console
console = Console()
cached_zip_file = None
executor_name = None
usage_kind = None
try:
need_pull = self.args.force_update
with console.status(f'Pulling {self.args.uri}...') as st:
scheme, name, tag, secret = parse_hub_uri(self.args.uri)
st.update(f'Fetching [bold]{name}[/bold] from Jina Hub ...')
executor, from_cache = HubIO.fetch_meta(name,
tag,
secret=secret,
force=need_pull)
presented_id = getattr(executor, 'name', executor.uuid)
executor_name = (
f'{presented_id}' if executor.visibility == 'public' else
f'{presented_id}:{secret}') + (f'/{tag}' if tag else '')
if scheme == 'jinahub+docker':
self._load_docker_client()
import docker
try:
self._client.images.get(executor.image_name)
except docker.errors.ImageNotFound:
need_pull = True
if need_pull:
st.update(f'Pulling image ...')
log_stream = self._raw_client.pull(executor.image_name,
stream=True,
decode=True)
st.stop()
self._pull_with_progress(
log_stream,
console,
)
usage_kind = 'docker'
return f'docker://{executor.image_name}'
elif scheme == 'jinahub':
import filelock
with filelock.FileLock(get_lockfile(), timeout=-1):
try:
pkg_path, pkg_dist_path = get_dist_path_of_executor(
executor)
# check serial number to upgrade
sn_file_path = pkg_dist_path / f'PKG-SN-{executor.sn or 0}'
if (not sn_file_path.exists()) and any(
pkg_dist_path.glob('PKG-SN-*')):
raise FileNotFoundError(
f'{pkg_path} need to be upgraded')
st.update(
'Installing [bold]requirements.txt[/bold]...')
install_package_dependencies(
install_deps=self.args.install_requirements,
pkg_dist_path=pkg_dist_path,
pkg_path=pkg_dist_path,
)
except FileNotFoundError:
need_pull = True
if need_pull:
# pull the latest executor meta, as the cached data would expire
if from_cache:
executor, _ = HubIO.fetch_meta(name,
tag,
secret=secret,
force=True)
cache_dir = Path(
os.environ.get(
'JINA_HUB_CACHE_DIR',
Path.home().joinpath('.cache', 'jina'),
))
cache_dir.mkdir(parents=True, exist_ok=True)
st.update(f'Downloading {name} ...')
cached_zip_file = download_with_resume(
executor.archive_url,
cache_dir,
f'{executor.uuid}-{executor.md5sum}.zip',
md5sum=executor.md5sum,
)
st.update(f'Unpacking {name} ...')
install_local(
cached_zip_file,
executor,
install_deps=self.args.install_requirements,
)
pkg_path, _ = get_dist_path_of_executor(executor)
usage_kind = 'source'
return f'{pkg_path / "config.yml"}'
else:
raise ValueError(f'{self.args.uri} is not a valid scheme')
except KeyboardInterrupt:
executor_name = None
except Exception:
executor_name = None
raise
finally:
# delete downloaded zip package if existed
if cached_zip_file is not None:
cached_zip_file.unlink()
if not self.args.no_usage and executor_name:
self._get_prettyprint_usage(console, executor_name, usage_kind)