def _is_latest_version(suppress_on_error=True):
try:
from urllib.request import Request, urlopen
import json
from pkg_resources import parse_version
from jina import __version__
from jina.logging import default_logger
req = Request(
'https://api.jina.ai/latest', headers={'User-Agent': 'Mozilla/5.0'}
)
with urlopen(
req, timeout=1
) as resource: # 'with' is important to close the resource after use
latest_ver = json.load(resource)['version']
latest_ver = parse_version(latest_ver)
cur_ver = parse_version(__version__)
if cur_ver < latest_ver:
default_logger.warning(
f'WARNING: You are using Jina version {cur_ver}, however version {latest_ver} is available. '
f'You should consider upgrading via the "pip install --upgrade jina" command.'
)
return False
return True
except:
# no network, two slow, api.jina.ai is down
if not suppress_on_error:
raise
def _extract_parameters(executor_yml):
try:
with BaseExecutor.load_config(executor_yml) as executor:
if hasattr(executor, "DEFAULT_OPTIMIZATION_PARAMETER"):
default_config = executor.DEFAULT_OPTIMIZATION_PARAMETER
else:
default_config = {}
return default_config
except TypeError:
logger.warning(
f"Failing building from {executor_yml}. All environment variables in {executor_yml} must be defined!"
)
def _write_optimization_parameter(executor_configurations, target_file,
overwrite_parameter_file):
output = [
parameter for config in executor_configurations.values()
for parameter in config
]
if os.path.exists(target_file) and not overwrite_parameter_file:
logger.warning(
f"{target_file} already exists. Skip writing. Please remove it before parameter discovery."
)
else:
with open(target_file, "w") as outfile:
JAML.dump(output, outfile)
def _replace_parameters(executor_yml, default_parameters):
for parameter in default_parameters:
if "\nwith:\n" not in executor_yml:
executor_yml = executor_yml + "\nwith:\n"
if f"{parameter.parameter_name}:" in executor_yml:
logger.warning(
f"Skipping the following parameter, since it is already defined: {parameter.parameter_name}"
)
continue
executor_yml = executor_yml.replace(
"\nwith:\n",
f"\nwith:\n {parameter.parameter_name}: ${parameter.jaml_variable}\n",
)
return executor_yml
def auto_reduce(model_outputs: 'np.ndarray', mask_2d: 'np.ndarray',
model_name: str) -> 'np.ndarray':
"""
Automatically creates a sentence embedding from its token embeddings.
* For BERT-like models (BERT, RoBERTa, DistillBERT, Electra ...) uses embedding of first token
* For XLM and XLNet models uses embedding of last token
* Assumes that other models are language-model like and uses embedding of last token
"""
if 'bert' in model_name or 'electra' in model_name:
return reduce_cls(model_outputs, mask_2d)
if 'xlnet' in model_name:
return reduce_cls(model_outputs, mask_2d, cls_pos='tail')
default_logger.warning(
'Using embedding of a last token as a sequence embedding. '
'If that is not desirable, change `pooling_strategy`')
return reduce_cls(model_outputs, mask_2d, cls_pos='tail')
def new_doc_id(doc: 'DocumentProto') -> str:
""" Generate a new hexdigest based on the content of the document.
.. note::
Always use it AFTER you fill in the content of the document
:param doc: a non-empty document
:return: the hexdigest based on :meth:`blake2b`
"""
global _warned_deprecation
if not _warned_deprecation:
default_logger.warning(
'This function name is deprecated and will be renamed to `get_content_hash` latest with Jina 1.0.0. Please already use the updated name.'
)
_warned_deprecation = True
return get_content_hash(doc)
def clean_workdir():
if os.path.exists(os.environ['JINA_WORKSPACE']):
shutil.rmtree(os.environ['JINA_WORKSPACE'])
logger.warning('Workspace deleted')
"""Module for helper functions in the parser"""
import argparse
import os
import uuid
from typing import Tuple
_SHOW_ALL_ARGS = 'JINA_FULL_CLI' in os.environ
if _SHOW_ALL_ARGS:
from jina.logging import default_logger
default_logger.warning(
f'Setting {_SHOW_ALL_ARGS} will make remote Peas with sharding not work when using JinaD'
)
def add_arg_group(parser, title):
"""Add the arguments for a specific group to the parser
:param parser: the parser configure
:param title: the group name
:return: the new parser
"""
return parser.add_argument_group(f'{title} arguments')
def UUIDString(astring) -> str:
"""argparse type to check if a string is a valid UUID string
:param astring: the string to check
:return: the string
"""
uuid.UUID(astring)
