def _get_help_string(self, action):
help_string = ''
if '%(default)' not in action.help:
if action.default is not argparse.SUPPRESS:
from jina.helper import colored
defaulting_nargs = [argparse.OPTIONAL, argparse.ZERO_OR_MORE]
if isinstance(action, argparse._StoreTrueAction):
help_string = colored(
'default: %s' %
('enabled' if action.default else
f'disabled, use "{action.option_strings[0]}" to enable it'
),
attrs=['dark'],
)
elif action.choices:
choices_str = f'{{{", ".join([str(c) for c in action.choices])}}}'
help_string = colored(
'choose from: ' + choices_str +
'; default: %(default)s',
attrs=['dark'],
)
elif action.option_strings or action.nargs in defaulting_nargs:
help_string = colored(
'type: %(type)s; default: %(default)s', attrs=['dark'])
return f'''
def lookup_and_print(query: str):
"""Lookup argument name in Jina API and prettyprint the result.
:param query: the argument (fuzzy) name
"""
nkw2kw, kw2info = _build_lookup_table()
if query not in nkw2kw:
from jina.helper import colored
print(
f'Can not find argument {colored(query, attrs="bold")}, '
f'maybe it\'s a misspelling or Jina does not have this argument.')
else:
helps = kw2info[nkw2kw[query]] # type: list
if len(helps) == 1:
_prettyprint_help(helps[0])
elif len(helps) > 1 and len(set(h['help'] for h in helps)) == 1:
_prettyprint_help(helps[0], also_in=helps)
elif len(helps) > 1:
from collections import defaultdict
from jina.helper import colored
help_group = defaultdict(list)
for h in helps:
help_group[h['help']].append(h)
print(
colored(f'Found {len(help_group)} mentions in Jina API.',
attrs='dark'))
for hg in help_group.values():
_prettyprint_help(hg[0], also_in=hg)
print(colored('─' * 40, attrs='dark'))
def _check_completeness(self):
self.dockerfile_path = get_exist_path(self.args.path, 'Dockerfile')
self.manifest_path = get_exist_path(self.args.path, 'manifest.yml')
self.readme_path = get_exist_path(self.args.path, 'README.md')
yaml_glob = glob.glob(os.path.join(self.args.path, '*.yml'))
if yaml_glob:
yaml_glob.remove(self.manifest_path)
py_glob = glob.glob(os.path.join(self.args.path, '*.py'))
completeness = {
'Dockerfile': self.dockerfile_path,
'manifest.yml': self.manifest_path,
'README.md': os.path.exists(self.readme_path),
'*.yml': yaml_glob,
'*.py': py_glob
}
self.logger.info(
f'a completeness check\n' +
'\n'.join('%4s %-20s %s' % (colored('✓', 'green') if v else colored('✗', 'red'), k, v) for k, v in
completeness.items()) + '\n')
if completeness['Dockerfile'] and completeness['manifest.yml']:
pass
else:
self.logger.critical('Dockerfile or manifest.yml is not given, can not build')
raise FileNotFoundError('Dockerfile or manifest.yml is not given, can not build')
tmp = self._read_manifest(self.manifest_path)
self.dockerfile_path_revised = self._get_revised_dockerfile(self.dockerfile_path, tmp)
self.canonical_name = safe_url_name(f'{_repo_prefix}' + '{type}.{name}:{version}'.format(**tmp))
def __exit__(self, exc_type, exc_val, traceback):
if exc_type == ModuleNotFoundError:
missing_module = self._pkg_name or exc_val.name
with open(
os.path.join(__resources_path__,
'extra-requirements.txt')) as fp:
for v in fp:
if (v.strip() and not v.startswith('#')
and v.startswith(missing_module) and ':' in v):
missing_module, install_tags = v.split(':')
self._tags.append(missing_module)
self._tags.extend(vv.strip()
for vv in install_tags.split(','))
break
if self._tags:
from jina.helper import colored
req_msg = colored('fallback to default behavior',
color='yellow')
if self._required:
req_msg = colored('and it is required', color='red')
err_msg = f'''Python package "{colored(missing_module, attrs='bold')}" is not installed, {req_msg}.
You are trying to use a feature not enabled by your current Jina installation.'''
avail_tags = ' '.join(
colored(f'[{tag}]', attrs='bold') for tag in self._tags)
err_msg += (
f'\n\nTo enable this feature, use {colored("pip install jina[TAG]", attrs="bold")}, '
f'where {colored("[TAG]", attrs="bold")} is one of {avail_tags}.\n'
)
else:
err_msg = f'{exc_val.msg}'
if self._required:
if self._verbose:
if self._logger:
self._logger.critical(err_msg)
if self._help_text:
self._logger.error(self._help_text)
else:
warnings.warn(err_msg, RuntimeWarning, stacklevel=2)
raise exc_val
else:
if self._verbose:
if self._logger:
self._logger.warning(err_msg)
if self._help_text:
self._logger.info(self._help_text)
else:
warnings.warn(err_msg, RuntimeWarning, stacklevel=2)
return True # suppress the error
def _prettyprint_help(d, also_in=None):
from jina.helper import colored
if d['type'] == 'command':
print(
f'''
{colored(d['name'], attrs='bold')} is a CLI command of Jina.
{colored(d['help'], attrs='bold')}
More info: {d['usage']} --help
'''
)
else:
availables = ' '.join(
colored(v, attrs='underline')
for v in (set(h['usage'] for h in also_in) if also_in else {d['usage']})
)
option_str = ' '.join(colored(v, attrs='bold') for v in d['option_strings'])
if option_str:
option_str = f'as {option_str}'
table = {}
table['Type'] = d['type']
table['Required'] = d['required']
if d['choices']:
table['Choices'] = ' | '.join(d['choices'])
if not d['default_random'] and d['default'] is not None:
table['Default'] = d['default']
if d['default_random']:
table['Remark'] = colored(
'This argument has a random default value!', 'on yellow'
)
table_str = '\n '.join(
f'{k + ": " + colored(v, attrs="bold")}' for k, v in table.items()
)
lb = '\033[F'
import argparse
print(
f'''
{colored(d['name'], attrs='bold')} is {colored('an internal CLI of Jina, should not be used directly', color='yellow') if d['help'] == argparse.SUPPRESS else 'a CLI argument of Jina.'}.
It is available in {availables} {option_str}
{colored(d['help'], attrs='bold') if d['help'] != argparse.SUPPRESS else lb * 2}
{table_str}
'''
)
def format_help(self):
# format the indented section
if self.parent is not None:
self.formatter._indent()
join = self.formatter._join_parts
item_help = join([func(*args) for func, args in self.items])
if self.parent is not None:
self.formatter._dedent()
# return nothing if the section was empty
if not item_help.strip():
return ''
# add the heading if the section was non-empty
if self.heading is not argparse.SUPPRESS and self.heading is not None:
from jina.helper import colored
current_indent = self.formatter._current_indent
captial_heading = ' '.join(v[0].upper() + v[1:]
for v in self.heading.split(' '))
heading = '%*s%s\n' % (
current_indent,
'',
colored(f'▮ {captial_heading}', 'cyan', attrs=['bold']),
)
else:
heading = ''
# join the section-initial newline, the heading and the help
return join(['\n', heading, item_help, '\n'])
def _get_run_args(print_args: bool = True):
from jina.helper import colored
from . import daemon_logger
parser = get_main_parser()
from argparse import _StoreAction, _StoreTrueAction
args = parser.parse_args()
if print_args:
default_args = {
a.dest: a.default
for a in parser._actions
if isinstance(a, (_StoreAction, _StoreTrueAction))
}
with open(os.path.join(__resources_path__, 'jina.logo')) as fp:
logo_str = fp.read()
param_str = []
for k, v in sorted(vars(args).items()):
j = f'{k.replace("_", "-"): >30.30} = {str(v):30.30}'
if default_args.get(k, None) == v:
param_str.append(' ' + j)
else:
param_str.append('🔧️ ' + colored(j, 'blue', 'on_yellow'))
param_str = '\n'.join(param_str)
daemon_logger.info(
f'\n{logo_str}\n▶️ {" ".join(sys.argv)}\n{param_str}\n')
return args
def _get_run_args(print_args: bool = True):
from jina.logging import default_logger
from jina.parser import get_main_parser
from jina.helper import colored
parser = get_main_parser()
if len(sys.argv) > 1:
from argparse import _StoreAction, _StoreTrueAction
args = parser.parse_args()
if print_args:
from pkg_resources import resource_filename
p = parser._actions[-1].choices[sys.argv[1]]
default_args = {a.dest: a.default for a in p._actions if
isinstance(a, _StoreAction) or isinstance(a, _StoreTrueAction)}
with open(resource_filename('jina', '/'.join(('resources', 'jina.logo')))) as fp:
logo_str = fp.read()
param_str = []
for k, v in sorted(vars(args).items()):
j = f'{k.replace("_", "-"): >30.30} = {str(v):30.30}'
if default_args.get(k, None) == v:
param_str.append(' ' + j)
else:
param_str.append('🔧️ ' + colored(j, 'blue', 'on_yellow'))
param_str = '\n'.join(param_str)
default_logger.info(f'\n{logo_str}\n▶️ {" ".join(sys.argv)}\n{param_str}\n')
return args
else:
parser.print_help()
exit()
def write_html(html_path: str):
global num_docs_evaluated
global evaluation_value
with open(resource_filename('jina', '/'.join(('resources', 'helloworld.html'))), 'r') as fp, \
open(html_path, 'w') as fw:
t = fp.read()
t = t.replace('{% RESULT %}', '\n'.join(result_html))
t = t.replace(
'{% PRECISION_EVALUATION %}',
'{:.2f}%'.format(evaluation_value['PrecisionEvaluator'] * 100.0))
t = t.replace(
'{% RECALL_EVALUATION %}',
'{:.2f}%'.format(evaluation_value['RecallEvaluator'] * 100.0))
t = t.replace('{% TOP_K %}', str(TOP_K))
fw.write(t)
url_html_path = 'file://' + os.path.abspath(html_path)
try:
webbrowser.open(url_html_path, new=2)
except:
pass
finally:
default_logger.success(
f'You should see a "hello-world.html" opened in your browser, '
f'if not you may open {url_html_path} manually')
colored_url = colored('https://opensource.jina.ai',
color='cyan',
attrs='underline')
default_logger.success(
f'🤩 Intrigued? Play with "jina hello-world --help" and learn more about Jina at {colored_url}'
)
async def startup():
daemon_logger.success(
f'\tWelcome to Jina daemon - the manager of distributed Jina')
daemon_logger.success(
f'\tUvicorn + FastAPI running on {server_config.HOST}:{server_config.PORT}'
)
daemon_logger.success(
f'\t🌐 Private address:\t' +
colored(f'http://{get_internal_ip()}:{server_config.PORT}',
'cyan',
attrs='underline'))
daemon_logger.success(
f'\t🌐 Public address:\t' +
colored(f'http://{get_public_ip()}:{server_config.PORT}',
'cyan',
attrs='underline'))
def _get_run_args(print_args: bool = True):
"""Fetch run args for jinad
:param print_args: True if we want to print args to console
:return: jinad args
"""
from jina.helper import colored
from daemon import daemon_logger
parser = get_main_parser()
from argparse import _StoreAction, _StoreTrueAction
args, argv = parser.parse_known_args()
# avoid printing for partial daemon (args.mode is set)
if print_args and args.mode is None:
default_args = {
a.dest: a.default
for a in parser._actions
if isinstance(a, (_StoreAction, _StoreTrueAction))
}
with open(os.path.join(__resources_path__, 'jina.logo')) as fp:
logo_str = fp.read()
param_str = []
for k, v in sorted(vars(args).items()):
j = f'{k.replace("_", "-"): >30.30} = {str(v):30.30}'
if default_args.get(k, None) == v:
param_str.append(' ' + j)
else:
param_str.append('🔧️ ' + colored(j, 'blue', 'on_yellow'))
param_str = '\n'.join(param_str)
daemon_logger.info(
f'\n{logo_str}\n▶️ {" ".join(sys.argv)}\n{param_str}\n')
return args
def print_embed(req):
for d in req.docs:
for c in d.chunks:
embed = pb2array(c.embedding)
text = colored(f'{c.text[:10]}...' if len(c.text) > 10 else c.text,
'blue')
print(
f'{text} embed to {embed.shape} [{embed[0]:.3f}, {embed[1]:.3f}...]'
)
def _exit_msg(self):
if self._logger:
self._logger.info(
f'{self.task_name} takes {self.readable_duration} ({self.duration:.2f}s)'
)
else:
print(
colored(
f'{self.task_name} takes {self.readable_duration} ({self.duration:.2f}s)'
),
flush=True,
)
def _print_bar(self, bar_info):
time_str = str(
datetime.timedelta(seconds=time.perf_counter() -
self.start)).split('.')[0]
sys.stdout.write(self.clear_line)
sys.stdout.write('{} {:>10} {:<}{:<} {} {} {}'.format(
colored(next(self.spinner), 'green'),
bar_info['description_str'],
colored('━' * bar_info['num_fullbars'], bar_info['bar_color']) +
(colored(
'╸',
bar_info['bar_color'] if bar_info['num_halfbars'] else
bar_info['unfinished_bar_color'],
)),
colored(
'━' * (self._bars_on_row - bar_info['num_fullbars']),
bar_info['unfinished_bar_color'],
attrs=['dark'],
),
colored(time_str, 'cyan'),
bar_info['speed_str'],
bar_info['msg_str'],
))
sys.stdout.flush()
def _get_run_args(print_args: bool = True):
from jina.parsers import get_main_parser
silent_print = {'help', 'hub'}
parser = get_main_parser()
if len(sys.argv) > 1:
from argparse import _StoreAction, _StoreTrueAction
args, unknown = parser.parse_known_args()
if unknown:
from jina.helper import warn_unknown_args
warn_unknown_args(unknown)
if args.cli not in silent_print and print_args:
from jina.helper import colored
from jina import __resources_path__
p = parser._actions[-1].choices[sys.argv[1]]
default_args = {
a.dest: a.default
for a in p._actions
if isinstance(a, (_StoreAction, _StoreTrueAction))
}
with open(os.path.join(__resources_path__, 'jina.logo')) as fp:
logo_str = fp.read()
param_str = []
for k, v in sorted(vars(args).items()):
j = f'{k.replace("_", "-"): >30.30} = {str(v):30.30}'
if default_args.get(k, None) == v:
param_str.append(' ' + j)
else:
param_str.append('🔧️ ' + colored(j, 'blue', 'on_yellow'))
param_str = '\n'.join(param_str)
print(f'\n{logo_str}\n▶️ {" ".join(sys.argv)}\n{param_str}\n')
return args
else:
parser.print_help()
exit()
def write_html(html_path):
with open(resource_filename('jina', '/'.join(('resources', 'helloworld.html'))), 'r') as fp, \
open(html_path, 'w') as fw:
t = fp.read()
t = t.replace('{% RESULT %}', '\n'.join(result_html))
fw.write(t)
url_html_path = 'file://' + os.path.abspath(html_path)
try:
webbrowser.open(url_html_path, new=2)
except:
pass
finally:
default_logger.success(f'You should see a "hello-world.html" opened in your browser, '
f'if not you may open {url_html_path} manually')
colored_url = colored('https://opensource.jina.ai', color='cyan', attrs='underline')
default_logger.success(
f'🤩 Intrigued? Play with "jina hello-world --help" and learn more about Jina at {colored_url}')
def write_html(html_path):
"""
Method to present results in browser.
:param html_path: path of the written html
"""
with open(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'demo.html')) as fp, open(html_path, 'w') as fw:
t = fp.read()
t = t.replace('{% RESULT %}', '\n'.join(result_html))
t = t.replace(
'{% PRECISION_EVALUATION %}',
'{:.2f}%'.format(evaluation_value['Precision'] * 100.0),
)
t = t.replace(
'{% RECALL_EVALUATION %}',
'{:.2f}%'.format(evaluation_value['Recall'] * 100.0),
)
t = t.replace('{% TOP_K %}', str(top_k))
fw.write(t)
url_html_path = 'file://' + os.path.abspath(html_path)
try:
webbrowser.open(url_html_path, new=2)
except:
pass # intentional pass, browser support isn't cross-platform
finally:
default_logger.success(
f'You should see a "hello-world.html" opened in your browser, '
f'if not you may open {url_html_path} manually')
colored_url = colored('https://opensource.jina.ai',
color='cyan',
attrs='underline')
default_logger.success(
f'🤩 Intrigued? Play with "jina hello fashion --help" and learn more about Jina at {colored_url}'
)
