def local_hub_executor(tmpdir, test_envs):
from jina.hubble import hubapi, helper, HubExecutor
hubapi._hub_root = Path(os.environ.get('JINA_HUB_ROOT'))
pkg_path = Path(__file__).parent / 'dummyhub'
stream_data = helper.archive_package(pkg_path)
with open(tmpdir / 'dummy_test.zip', 'wb') as temp_zip_file:
temp_zip_file.write(stream_data.getvalue())
hubapi.install_local(
Path(tmpdir) / 'dummy_test.zip', HubExecutor(uuid='hello', tag='v0'))
def push(self) -> None:
"""Push the executor pacakge to Jina Hub."""
from rich.console import Console
work_path = Path(self.args.path)
exec_tags = None
if self.args.tag:
exec_tags = ','.join(self.args.tag)
dockerfile = None
if self.args.dockerfile:
dockerfile = Path(self.args.dockerfile)
if not dockerfile.exists():
raise Exception(
f'The given Dockerfile `{dockerfile}` does not exist!')
if dockerfile.parent != work_path:
raise Exception(
f'The Dockerfile must be placed at the given folder `{work_path}`'
)
dockerfile = dockerfile.relative_to(work_path)
console = Console()
with console.status(f'Pushing `{self.args.path}` ...') as st:
req_header = get_request_header()
try:
st.update(f'Packaging {self.args.path} ...')
md5_hash = hashlib.md5()
bytesio = archive_package(work_path)
content = bytesio.getvalue()
md5_hash.update(content)
md5_digest = md5_hash.hexdigest()
# upload the archived package
form_data = {
'public':
'True' if getattr(self.args, 'public', None) else 'False',
'private':
'True' if getattr(self.args, 'private', None) else 'False',
'md5sum': md5_digest,
}
if exec_tags:
form_data['tags'] = exec_tags
if dockerfile:
form_data['dockerfile'] = str(dockerfile)
uuid8, secret = load_secret(work_path)
if self.args.force_update or uuid8:
form_data['force'] = self.args.force_update or uuid8
if self.args.secret or secret:
form_data['secret'] = self.args.secret or secret
method = 'put' if ('force' in form_data) else 'post'
st.update(f'Connecting to Jina Hub ...')
hubble_url = get_hubble_url_v1() + '/executors'
# upload the archived executor to Jina Hub
st.update(f'Uploading...')
resp = upload_file(
hubble_url,
'filename',
content,
dict_data=form_data,
headers=req_header,
stream=True,
method=method,
)
result = None
for stream_line in resp.iter_lines():
stream_msg = json.loads(stream_line)
if 'stream' in stream_msg:
st.update(
f'Cloud building ... [dim]{stream_msg["stream"]}[/dim]'
)
elif 'status' in stream_msg:
st.update(
f'Cloud building ... [dim]{stream_msg["status"]}[/dim]'
)
elif 'result' in stream_msg:
result = stream_msg['result']
break
if result is None:
raise Exception('Unknown Error')
elif not result.get('data', None):
msg = result.get('message', 'Unknown Error')
if 'Process(docker) exited on non-zero code' in msg:
self.logger.error('''
Failed on building Docker image. Potential solutions:
- If you haven't provide a Dockerfile in the executor bundle, you may want to provide one,
as the auto-generated one on the cloud did not work.
- If you have provided a Dockerfile, you may want to check the validity of this Dockerfile.
''')
raise Exception(msg)
elif 200 <= result['statusCode'] < 300:
new_uuid8, new_secret = self._prettyprint_result(
console, result)
if new_uuid8 != uuid8 or new_secret != secret:
dump_secret(work_path, new_uuid8, new_secret)
elif result['message']:
raise Exception(result['message'])
elif resp.text:
# NOTE: sometimes resp.text returns empty
raise Exception(resp.text)
else:
resp.raise_for_status()
except KeyboardInterrupt:
pass
except Exception as e: # IO related errors
self.logger.error(
f'''Please report this session_id: {colored(req_header["jinameta-session-id"], color="yellow", attrs="bold")} to https://github.com/jina-ai/jina/issues'
{e!r}''')
raise e
def test_archive_package(tmpdir):
pkg_path = Path(__file__).parent / 'dummy_executor'
stream_data = helper.archive_package(pkg_path)
with open(tmpdir / 'dummy_test.zip', 'wb') as temp_zip_file:
temp_zip_file.write(stream_data.getvalue())
def push(self) -> None:
"""Push the executor package to Jina Hub."""
work_path = Path(self.args.path)
exec_tags = None
exec_immutable_tags = None
if self.args.tag:
exec_tags = ','.join(self.args.tag)
if self.args.protected_tag:
exec_immutable_tags = ','.join(self.args.protected_tag)
dockerfile = None
if self.args.dockerfile:
dockerfile = Path(self.args.dockerfile)
if not dockerfile.exists():
raise Exception(
f'The given Dockerfile `{dockerfile}` does not exist!')
if dockerfile.parent != work_path:
raise Exception(
f'The Dockerfile must be placed at the given folder `{work_path}`'
)
dockerfile = dockerfile.relative_to(work_path)
console = get_rich_console()
with console.status(f'Pushing `{self.args.path}` ...') as st:
req_header = get_request_header()
try:
st.update(f'Packaging {self.args.path} ...')
md5_hash = hashlib.md5()
bytesio = archive_package(work_path)
content = bytesio.getvalue()
md5_hash.update(content)
md5_digest = md5_hash.hexdigest()
# upload the archived package
form_data = {
'public':
'True' if getattr(self.args, 'public', None) else 'False',
'private':
'True' if getattr(self.args, 'private', None) else 'False',
'md5sum': md5_digest,
}
if self.args.verbose:
form_data['verbose'] = 'True'
if self.args.no_cache:
form_data['buildWithNoCache'] = 'True'
if exec_tags:
form_data['tags'] = exec_tags
if exec_immutable_tags:
form_data['immutableTags'] = exec_immutable_tags
if dockerfile:
form_data['dockerfile'] = str(dockerfile)
uuid8, secret = load_secret(work_path)
if self.args.force_update or uuid8:
form_data['id'] = self.args.force_update or uuid8
if self.args.secret or secret:
form_data['secret'] = self.args.secret or secret
st.update(f'Connecting to Jina Hub ...')
if form_data.get('id') and form_data.get('secret'):
hubble_url = get_hubble_url_v2() + '/rpc/executor.update'
else:
hubble_url = get_hubble_url_v2() + '/rpc/executor.create'
# upload the archived executor to Jina Hub
st.update(f'Uploading...')
resp = upload_file(
hubble_url,
'filename',
content,
dict_data=form_data,
headers=req_header,
stream=True,
method='post',
)
image = None
session_id = req_header.get('jinameta-session-id')
for stream_line in resp.iter_lines():
stream_msg = json.loads(stream_line)
t = stream_msg.get('type')
subject = stream_msg.get('subject')
payload = stream_msg.get('payload', '')
if t == 'error':
msg = stream_msg.get('message')
hubble_err = payload
overridden_msg = ''
detail_msg = ''
if isinstance(hubble_err, dict):
(overridden_msg,
detail_msg) = get_hubble_error_message(hubble_err)
if not msg:
msg = detail_msg
if overridden_msg and overridden_msg != detail_msg:
self.logger.warning(overridden_msg)
raise Exception(
f'{overridden_msg or msg or "Unknown Error"} session_id: {session_id}'
)
if t == 'progress' and subject == 'buildWorkspace':
legacy_message = stream_msg.get('legacyMessage', {})
status = legacy_message.get('status', '')
st.update(
f'Cloud building ... [dim]{subject}: {t} ({status})[/dim]'
)
elif t == 'complete':
image = stream_msg['payload']
st.update(
f'Cloud building ... [dim]{subject}: {t} ({stream_msg["message"]})[/dim]'
)
break
elif t and subject:
if self.args.verbose and t == 'console':
console.log(
f'Cloud building ... [dim]{subject}: {payload}[/dim]'
)
else:
st.update(
f'Cloud building ... [dim]{subject}: {t} {payload}[/dim]'
)
if image:
new_uuid8, new_secret = self._prettyprint_result(
console, image)
if new_uuid8 != uuid8 or new_secret != secret:
dump_secret(work_path, new_uuid8, new_secret)
else:
raise Exception(f'Unknown Error, session_id: {session_id}')
except KeyboardInterrupt:
pass
except Exception as e: # IO related errors
self.logger.error(
f'''Please report this session_id: [yellow bold]{req_header["jinameta-session-id"]}[/] to https://github.com/jina-ai/jina/issues'''
)
raise e