def _is_latest_version(suppress_on_error=True): try: from urllib.request import Request, urlopen import json from pkg_resources import parse_version from jina import __version__ from jina.logging import default_logger req = Request( 'https://api.jina.ai/latest', headers={'User-Agent': 'Mozilla/5.0'} ) with urlopen( req, timeout=1 ) as resource: # 'with' is important to close the resource after use latest_ver = json.load(resource)['version'] latest_ver = parse_version(latest_ver) cur_ver = parse_version(__version__) if cur_ver < latest_ver: default_logger.warning( f'WARNING: You are using Jina version {cur_ver}, however version {latest_ver} is available. ' f'You should consider upgrading via the "pip install --upgrade jina" command.' ) return False return True except: # no network, two slow, api.jina.ai is down if not suppress_on_error: raise
def _extract_parameters(executor_yml): try: with BaseExecutor.load_config(executor_yml) as executor: if hasattr(executor, "DEFAULT_OPTIMIZATION_PARAMETER"): default_config = executor.DEFAULT_OPTIMIZATION_PARAMETER else: default_config = {} return default_config except TypeError: logger.warning( f"Failing building from {executor_yml}. All environment variables in {executor_yml} must be defined!" )
def _write_optimization_parameter(executor_configurations, target_file, overwrite_parameter_file): output = [ parameter for config in executor_configurations.values() for parameter in config ] if os.path.exists(target_file) and not overwrite_parameter_file: logger.warning( f"{target_file} already exists. Skip writing. Please remove it before parameter discovery." ) else: with open(target_file, "w") as outfile: JAML.dump(output, outfile)
def _replace_parameters(executor_yml, default_parameters): for parameter in default_parameters: if "\nwith:\n" not in executor_yml: executor_yml = executor_yml + "\nwith:\n" if f"{parameter.parameter_name}:" in executor_yml: logger.warning( f"Skipping the following parameter, since it is already defined: {parameter.parameter_name}" ) continue executor_yml = executor_yml.replace( "\nwith:\n", f"\nwith:\n {parameter.parameter_name}: ${parameter.jaml_variable}\n", ) return executor_yml
def auto_reduce(model_outputs: 'np.ndarray', mask_2d: 'np.ndarray', model_name: str) -> 'np.ndarray': """ Automatically creates a sentence embedding from its token embeddings. * For BERT-like models (BERT, RoBERTa, DistillBERT, Electra ...) uses embedding of first token * For XLM and XLNet models uses embedding of last token * Assumes that other models are language-model like and uses embedding of last token """ if 'bert' in model_name or 'electra' in model_name: return reduce_cls(model_outputs, mask_2d) if 'xlnet' in model_name: return reduce_cls(model_outputs, mask_2d, cls_pos='tail') default_logger.warning( 'Using embedding of a last token as a sequence embedding. ' 'If that is not desirable, change `pooling_strategy`') return reduce_cls(model_outputs, mask_2d, cls_pos='tail')
def new_doc_id(doc: 'DocumentProto') -> str: """ Generate a new hexdigest based on the content of the document. .. note:: Always use it AFTER you fill in the content of the document :param doc: a non-empty document :return: the hexdigest based on :meth:`blake2b` """ global _warned_deprecation if not _warned_deprecation: default_logger.warning( 'This function name is deprecated and will be renamed to `get_content_hash` latest with Jina 1.0.0. Please already use the updated name.' ) _warned_deprecation = True return get_content_hash(doc)
def clean_workdir(): if os.path.exists(os.environ['JINA_WORKSPACE']): shutil.rmtree(os.environ['JINA_WORKSPACE']) logger.warning('Workspace deleted')
"""Module for helper functions in the parser""" import argparse import os import uuid from typing import Tuple _SHOW_ALL_ARGS = 'JINA_FULL_CLI' in os.environ if _SHOW_ALL_ARGS: from jina.logging import default_logger default_logger.warning( f'Setting {_SHOW_ALL_ARGS} will make remote Peas with sharding not work when using JinaD' ) def add_arg_group(parser, title): """Add the arguments for a specific group to the parser :param parser: the parser configure :param title: the group name :return: the new parser """ return parser.add_argument_group(f'{title} arguments') def UUIDString(astring) -> str: """argparse type to check if a string is a valid UUID string :param astring: the string to check :return: the string """ uuid.UUID(astring)